3 |
3 |
4 from lxml import etree |
4 from lxml import etree |
5 from models import * |
5 from models import * |
6 from optparse import OptionParser |
6 from optparse import OptionParser |
7 from sqlalchemy import Table, Column, Integer, BigInteger, String, MetaData, \ |
7 from sqlalchemy import Table, Column, Integer, BigInteger, String, MetaData, \ |
8 ForeignKey, create_engine |
8 ForeignKey |
9 from sqlalchemy.orm import sessionmaker, mapper |
9 from sqlalchemy.orm import sessionmaker, mapper |
10 from sqlalchemy.sql import select |
10 from sqlalchemy.sql import select |
11 import datetime |
11 import datetime |
12 import time |
|
13 import email.utils |
12 import email.utils |
14 import logging |
13 import logging |
15 import os |
14 import os |
16 import os.path |
15 import os.path |
17 import re |
16 import re |
18 import sys |
17 import sys |
|
18 import time |
19 import uuid |
19 import uuid |
20 |
20 |
21 #class TweetExclude(object): |
21 #class TweetExclude(object): |
22 # def __init__(self, id): |
22 # def __init__(self, id): |
23 # self.id = id |
23 # self.id = id |
27 |
27 |
28 def parse_date(date_str): |
28 def parse_date(date_str): |
29 ts = email.utils.parsedate_tz(date_str) |
29 ts = email.utils.parsedate_tz(date_str) |
30 return datetime.datetime(*ts[0:7]) |
30 return datetime.datetime(*ts[0:7]) |
31 |
31 |
32 |
32 def get_options(): |
33 if __name__ == "__main__" : |
|
34 |
|
35 parser = OptionParser() |
33 parser = OptionParser() |
36 parser.add_option("-f", "--file", dest="filename", |
34 parser.add_option("-f", "--file", dest="filename", |
37 help="write export to file", metavar="FILE", default="project_enmi.ldt") |
35 help="write export to file", metavar="FILE", default="project_enmi.ldt") |
38 parser.add_option("-d", "--database", dest="database", |
36 parser.add_option("-d", "--database", dest="database", |
39 help="Input database", metavar="DATABASE") |
37 help="Input database", metavar="DATABASE") |
61 help="Cutting name", metavar="NAME", default=u"Tweets") |
59 help="Cutting name", metavar="NAME", default=u"Tweets") |
62 parser.add_option("-R", "--replace", dest="replace", action="store_true", |
60 parser.add_option("-R", "--replace", dest="replace", action="store_true", |
63 help="Replace tweet ensemble", metavar="REPLACE", default=False) |
61 help="Replace tweet ensemble", metavar="REPLACE", default=False) |
64 parser.add_option("-l", "--log", dest="logfile", |
62 parser.add_option("-l", "--log", dest="logfile", |
65 help="log to file", metavar="LOG", default="stderr") |
63 help="log to file", metavar="LOG", default="stderr") |
66 parser.add_option("-v", dest="verbose", action="count", |
|
67 help="verbose", metavar="VERBOSE", default=0) |
|
68 parser.add_option("-q", dest="quiet", action="count", |
|
69 help="quiet", metavar="QUIET", default=0) |
|
70 parser.add_option("-L", dest="listconf", |
|
71 help="file containing the list of file to process", metavar="LIST", default=0) |
|
72 |
|
73 |
|
74 |
64 |
75 (options, args) = parser.parse_args() |
65 set_logging_options(parser) |
|
66 |
76 |
67 |
77 logging_config = {} |
68 return parser.parse_args() |
|
69 |
|
70 |
|
71 if __name__ == "__main__" : |
|
72 |
|
73 (options, args) = get_options() |
|
74 |
|
75 set_logging(options) |
|
76 |
|
77 logging.debug("OPTIONS : " + repr(options)) |
|
78 |
|
79 engine, metadata = setup_database('sqlite:///'+options.database, echo=((options.verbose-options.quiet)>0), create_all = False) |
78 |
80 |
79 if options.logfile == "stdout": |
|
80 logging_config["stream"] = sys.stdout |
|
81 elif options.logfile == "stderr": |
|
82 logging_config["stream"] = sys.stderr |
|
83 else: |
|
84 logging_config["filename"] = options.logfile |
|
85 |
|
86 logging_config["level"] = max(logging.NOTSET, min(logging.CRITICAL, logging.WARNING - 10 * options.verbose + 10 * options.quiet)) |
|
87 |
|
88 logging.basicConfig(**logging_config) |
|
89 |
|
90 logging.debug("OPTIONS : " + repr(options)) |
|
91 |
|
92 |
|
93 engine = create_engine('sqlite:///'+options.database, echo=((options.verbose-options.quiet)>0)) |
|
94 Session = sessionmaker() |
81 Session = sessionmaker() |
95 |
|
96 conn = engine.connect() |
82 conn = engine.connect() |
97 try : |
83 try : |
98 session = Session(bind=conn) |
84 session = Session(bind=conn) |
99 try : |
85 try : |
100 |
86 |