diff -r 7fb5a7b0d35c -r 99215db3da25 script/utils/export_tweet_db.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/utils/export_tweet_db.py Tue Feb 21 10:47:06 2012 +0100 @@ -0,0 +1,47 @@ +from models import setup_database +from optparse import OptionParser #@UnresolvedImport +from sqlalchemy.orm import sessionmaker +from utils import set_logging_options, set_logging, TwitterProcessor, logger +import sqlite3 #@UnresolvedImport + + +# 'entities': "tweet_entity", +# 'user': "tweet_user" + +def get_option(): + + parser = OptionParser() + + parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", + help="Token file name") + + set_logging_options(parser) + + return parser.parse_args() + +if __name__ == "__main__": + + (options, args) = get_option() + + set_logging(options) + + with sqlite3.connect(args[0]) as conn_in: + engine, metadata, Session = setup_database('sqlite:///'+args[1], echo=((options.verbose-options.quiet)>0)) + session = Session() + try: + curs_in = conn_in.cursor() + fields_mapping = {} + for i,res in enumerate(curs_in.execute("select json from tweet_tweet;")): + logger.debug("main loop %d : %s" % (i, res[0])) #@UndefinedVariable + processor = TwitterProcessor(eval(res[0]), res[0], None, session, options.token_filename) + processor.process() + session.commit() + logger.debug("main : %d tweet processed" % (i+1)) #@UndefinedVariable + except Exception, e: + session.rollback() + raise e + finally: + session.close() + + + \ No newline at end of file