1 from models import *

     2 from utils import *

     3 from optparse import OptionParser

     4 from sqlalchemy.orm import sessionmaker

     5 import logging

     6 import sqlite3

     7 import sys

     8 

     9 

    10 #    'entities': "tweet_entity",     

    11 #    'user': "tweet_user"

    12 

    13 def get_option():

    14     

    15     parser = OptionParser()

    16 

    17     set_logging_options(parser)

    18 

    19     return parser.parse_args()

    20 

    21 if __name__ == "__main__":

    22     

    23     (options, args) = get_option()

    24         

    25     set_logging(options)

    26     

    27     with sqlite3.connect(args[0]) as conn_in:

    28         engine, metadata = setup_database('sqlite:///'+args[1], echo=((options.verbose-options.quiet)>0))

    29         Session = sessionmaker(bind=engine)

    30         session = Session()

    31         try:

    32             curs_in = conn_in.cursor()

    33             fields_mapping = {}

    34             for i,res in enumerate(curs_in.execute("select json from tweet_tweet;")):

    35                 logging.debug("main loop %d : %s" % (i, res[0]))

    36                 processor = TwitterProcessor(eval(res[0]), res[0], session)

    37                 processor.process()

    38                 session.commit()

    39             logging.debug("main : %d tweet processed" % (i+1))

    40         except Exception, e:

    41             session.rollback()

    42             raise e

    43         finally:

    44             session.close()

    45             

    46     

    47