script/rest/export_twitter.py
changeset 888 6fc6637d8403
parent 9 bb44692e09ee
child 957 e4d0094f097b
equal deleted inserted replaced
887:503f9a7b7d6c 888:6fc6637d8403
     1 #!/usr/bin/env python
     1 #!/usr/bin/env python
     2 # coding=utf-8
     2 # coding=utf-8
     3 
     3 
     4 from sqlite3 import *
     4 from sqlite3 import register_adapter, register_converter, connect, Row
     5 import datetime, time
     5 import datetime, time
     6 import email.utils
     6 import email.utils
     7 from optparse import OptionParser
     7 from optparse import OptionParser
     8 import os.path
     8 import os.path
     9 import os
       
    10 import sys
       
    11 from lxml import etree
     9 from lxml import etree
    12 import uuid
    10 import uuid
    13 import re
    11 import re
       
    12 import simplejson
    14 
    13 
    15 def parse_date(date_str):
    14 def parse_date(date_str):
    16     ts = email.utils.parsedate_tz(date_str)
    15     ts = email.utils.parsedate_tz(date_str)
    17     return time.mktime(ts[0:9]) - 60 * ts[9]
    16     return time.mktime(ts[0:9]) - 60 * ts[9]
    18 
    17 
    19 def adapt_datetime(ts):
    18 def adapt_datetime(ts):
    20     return time.mktime(ts.timetuple())
    19     return time.mktime(ts.timetuple())
    21     
    20     
    22 def adapt_geo(geo):
    21 def adapt_geo(geo):
    23 	return simplejson.dumps(geo)
    22     return simplejson.dumps(geo)
    24 	
    23 
    25 def convert_geo(s):
    24 def convert_geo(s):
    26 	return simplejson.loads(s)
    25     return simplejson.loads(s)
    27 
    26 
    28 
    27 
    29 register_adapter(datetime.datetime, adapt_datetime)
    28 register_adapter(datetime.datetime, adapt_datetime)
    30 register_converter("geo", convert_geo)
    29 register_converter("geo", convert_geo)
    31 
    30 
    71     
    70     
    72         
    71         
    73     ts = int(parse_date(options.start_date))
    72     ts = int(parse_date(options.start_date))
    74 
    73 
    75     if options.end_date:
    74     if options.end_date:
    76     	te = int(parse_date(options.end_date))
    75         te = int(parse_date(options.end_date))
    77     else:
    76     else:
    78         te = ts + options.duration
    77         te = ts + options.duration
    79     
    78     
    80     conn = connect(options.database)
    79     conn = connect(options.database)
    81     conn.row_factory = Row
    80     conn.row_factory = Row