diff -r 9374709c7647 -r d92b7a46e5c1 script/rest/user_timeline.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/rest/user_timeline.py Wed Dec 03 14:54:02 2014 +0100 @@ -0,0 +1,98 @@ +import argparse +import re + +import anyjson +import twitter + +from iri_tweet import models, processor, utils + + +def get_options(): + + usage = "usage: %(prog)s [options] " + + parser = argparse.ArgumentParser(usage=usage) + + parser.add_argument(dest="conn_str", + help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR") + parser.add_argument("-S", dest="screen_name", + help="Screen name", metavar="SCREENNAME", default=None) + parser.add_argument("-U", dest="user_id", + help="User id", metavar="USERID", default=None) + parser.add_argument("-P", dest="rpp", metavar="RPP", default="200", + help="Result per page") + parser.add_argument("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", + help="Token file name") + parser.add_argument("-k", "--key", dest="consumer_key", + help="Twitter consumer key", metavar="CONSUMER_KEY") + parser.add_argument("-s", "--secret", dest="consumer_secret", + help="Twitter consumer secret", metavar="CONSUMER_SECRET") + + utils.set_logging_options(parser) + + return parser.parse_args() + +def get_auth(options, access_token): + consumer_key = options.consumer_key + consumer_secret = options.consumer_secret + auth = twitter.OAuth(token=access_token[0], token_secret=access_token[1], consumer_key=consumer_key, consumer_secret=consumer_secret) + return auth + + +if __name__ == "__main__": + + options = get_options() + + access_token = utils.get_oauth_token(consumer_key=options.consumer_key, consumer_secret=options.consumer_secret, token_file_path=options.token_filename) + auth = get_auth(options, access_token) + + t = twitter.Twitter(domain="api.twitter.com",api_version="1.1",secure=True, auth=auth) + + conn_str = options.conn_str.strip() + if not re.match("^\w+://.+", conn_str): + conn_str = 'sqlite:///' + conn_str + + engine, metadata, Session = models.setup_database(conn_str, echo=((options.verbose-options.quiet)>0), create_all=True) + session = None + try: + session = Session() + #conn.row_factory = sqlite3.Row + #curs = conn.cursor() + #curs.execute("create table if not exists tweet_tweet (json);") + #conn.commit() + + results = None + page = 1 + print "screen_name: %r - userid: %r" % (options.screen_name, options.user_id) + + rargs = {'count': options.rpp} + + if options.screen_name: + rargs['screen_name'] = options.screen_name + if options.user_id: + rargs['user_id'] = options.user_id + + max_id = -1 + while page <= int(3200/int(options.rpp)) and ( results is None or len(results) > 0) and (max_id > 0 or max_id==-1): + + + if max_id>0: + rargs['max_id'] = max_id + results = t.statuses.user_timeline(**rargs) + + + for tweet in results: + print tweet + max_id = tweet.get('id', 0) + tweet_str = anyjson.serialize(tweet) + p = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=(options.consumer_key, options.consumer_secret), access_token=access_token, token_filename=options.token_filename, user_query_twitter=False, logger=None) + p.process() + session.flush() + session.commit() + page += 1 + + finally: + if session: + session.close() + +