diff -r 5378e8a44db1 -r 11c1322cffe6 script/utils/search_topsy.py --- a/script/utils/search_topsy.py Tue Nov 05 11:57:19 2013 +0100 +++ b/script/utils/search_topsy.py Wed Nov 06 02:42:13 2013 +0100 @@ -1,25 +1,26 @@ -from blessings import Terminal -from iri_tweet import models, utils -from iri_tweet.processor import TwitterProcessorStatus -from optparse import OptionParser +import argparse import logging import math import re +import time + +from blessings import Terminal import requests -import time import twitter +from iri_tweet import models, utils +from iri_tweet.processor import TwitterProcessorStatus + + logger = logging.getLogger(__name__) APPLICATION_NAME = "Tweet recorder user" -CONSUMER_KEY = "Vdr5ZcsjI1G3esTPI8yDg" -CONSUMER_SECRET = "LMhNrY99R6a7E0YbZZkRFpUZpX5EfB1qATbDk1sIVLs" class TopsyResource(object): def __init__(self, query, **kwargs): - + self.options = kwargs self.options['q'] = query self.url = kwargs.get("url", "http://otter.topsy.com/search.json") @@ -32,7 +33,7 @@ params = {} params.update(self.options) self.req = requests.get(self.url, params=params) - self.res = self.req.json + self.res = self.req.json() def __next_page(self): page = self.res.get("response").get("page") + 1 @@ -40,7 +41,7 @@ params.update(self.options) params['page'] = page self.req = requests.get(self.url, params=params) - self.res = self.req.json + self.res = self.req.json() def __iter__(self): if not self.req: @@ -58,19 +59,25 @@ -def get_option(): +def get_options(): - parser = OptionParser() + usage = "usage: %(prog)s [options] " + + parser = argparse.ArgumentParser(usage=usage) - parser.add_option("-d", "--database", dest="database", - help="Input database", metavar="DATABASE") - parser.add_option("-Q", dest="query", + parser.add_argument(dest="conn_str", + help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR") + parser.add_argument("-Q", dest="query", help="query", metavar="QUERY") - parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", + parser.add_argument("-k", "--key", dest="consumer_key", + help="Twitter consumer key", metavar="CONSUMER_KEY") + parser.add_argument("-s", "--secret", dest="consumer_secret", + help="Twitter consumer secret", metavar="CONSUMER_SECRET") + parser.add_argument("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", help="Token file name") - parser.add_option("-T", dest="topsy_apikey", metavar="TOPSY_APIKEY", default=None, + parser.add_argument("-T", dest="topsy_apikey", metavar="TOPSY_APIKEY", default=None, help="Topsy apikey") - + utils.set_logging_options(parser) return parser.parse_args() @@ -79,17 +86,17 @@ if __name__ == "__main__": - (options, args) = get_option() + options = get_options() utils.set_logging(options); - acess_token_key, access_token_secret = utils.get_oauth_token(consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET, options.token_filename, application_name=APPLICATION_NAME) + acess_token_key, access_token_secret = utils.get_oauth_token(consumer_key=options.consumer_key, consumer_secret=options.consumer_secret, token_file_path=options.token_filename, application_name=APPLICATION_NAME) - t = twitter.Twitter(domain="api.twitter.com", auth=twitter.OAuth(acess_token_key, access_token_secret, CONSUMER_KEY, CONSUMER_SECRET), secure=True) + t = twitter.Twitter(domain="api.twitter.com", auth=twitter.OAuth(acess_token_key, access_token_secret, options.consumer_key, options.consumer_secret), secure=True) t.secure = True - conn_str = options.database.strip() + conn_str = options.conn_str.strip() if not re.match("^\w+://.+", conn_str): conn_str = 'sqlite:///' + conn_str @@ -146,10 +153,10 @@ processor.process() session.flush() session.commit() - + time_to_sleep = int(math.ceil((tweet.rate_limit_reset - time.mktime(time.gmtime())) / tweet.rate_limit_remaining)) - print "rate limit remaining %s of %s" % (str(tweet.rate_limit_remaining), str(tweet.headers.getheader('x-ratelimit-limit'))) + term.clear_eol() + print("rate limit remaining %s of %s" % (str(tweet.rate_limit_remaining), str(tweet.headers.getheader('X-Rate-Limit-Limit'))) + term.clear_eol()) move_up += 1 for i in xrange(time_to_sleep): if i: