1 from models import setup_database

     2 from optparse import OptionParser #@UnresolvedImport

     3 from sqlalchemy.orm import sessionmaker

     4 from utils import set_logging_options, set_logging, TwitterProcessor, logger

     5 import sqlite3 #@UnresolvedImport

     6 

     7 

     8 #    'entities': "tweet_entity",     

     9 #    'user': "tweet_user"

    10 

    11 def get_option():

    12     

    13     parser = OptionParser()

    14 

    15     parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",

    16                       help="Token file name")

    17 

    18     set_logging_options(parser)

    19 

    20     return parser.parse_args()

    21 

    22 if __name__ == "__main__":

    23     

    24     (options, args) = get_option()

    25         

    26     set_logging(options)

    27     

    28     with sqlite3.connect(args[0]) as conn_in:

    29         engine, metadata, Session = setup_database('sqlite:///'+args[1], echo=((options.verbose-options.quiet)>0))

    30         session = Session()

    31         try:

    32             curs_in = conn_in.cursor()

    33             fields_mapping = {}

    34             for i,res in enumerate(curs_in.execute("select json from tweet_tweet;")):

    35                 logger.debug("main loop %d : %s" % (i, res[0])) #@UndefinedVariable

    36                 processor = TwitterProcessor(eval(res[0]), res[0], None, session, options.token_filename)

    37                 processor.process()

    38                 session.commit()

    39             logger.debug("main : %d tweet processed" % (i+1)) #@UndefinedVariable

    40         except Exception, e:

    41             session.rollback()

    42             raise e

    43         finally:

    44             session.close()

    45             

    46     

    47