--- a/script/rest/search_twitter.py Tue Nov 05 11:57:19 2013 +0100
+++ b/script/rest/search_twitter.py Wed Nov 06 02:42:13 2013 +0100
@@ -1,31 +1,34 @@
-from iri_tweet import models, processor, utils
-from optparse import OptionParser
-import anyjson
+import argparse
import re
import sys
+
+import anyjson
import twitter
-def get_option():
+from iri_tweet import models, processor, utils
+
+
+def get_options():
- parser = OptionParser()
+ usage = "usage: %(prog)s [options] <connection_str_or_filepath>"
+
+ parser = argparse.ArgumentParser(usage=usage)
- parser.add_option("-l", "--log", dest="logfile",
- help="log to file", metavar="LOG", default="stderr")
- parser.add_option("-v", dest="verbose", action="count",
- help="verbose", metavar="VERBOSE", default=0)
- parser.add_option("-q", dest="quiet", action="count",
- help="quiet", metavar="QUIET", default=0)
- parser.add_option("-Q", dest="query",
+ parser.add_argument(dest="conn_str",
+ help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR")
+ parser.add_argument("-Q", dest="query",
help="query", metavar="QUERY")
- parser.add_option("-P", dest="rpp", metavar="RPP", default="50",
+ parser.add_argument("-P", dest="rpp", metavar="RPP", default="50",
help="Result per page")
- parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",
+ parser.add_argument("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",
help="Token file name")
- parser.add_option("-k", "--key", dest="consumer_key",
+ parser.add_argument("-k", "--key", dest="consumer_key",
help="Twitter consumer key", metavar="CONSUMER_KEY")
- parser.add_option("-s", "--secret", dest="consumer_secret",
+ parser.add_argument("-s", "--secret", dest="consumer_secret",
help="Twitter consumer secret", metavar="CONSUMER_SECRET")
+ utils.set_logging_options(parser)
+
return parser.parse_args()
def get_auth(options, access_token):
@@ -36,18 +39,17 @@
if __name__ == "__main__":
- (options, args) = get_option()
+ (options, args) = get_options()
access_token = utils.get_oauth_token(consumer_key=options.consumer_key, consumer_secret=options.consumer_secret, token_file_path=options.token_filename)
auth = get_auth(options, access_token)
t = twitter.Twitter(domain="api.twitter.com",api_version="1.1",secure=True, auth=auth)
-
- conn_str = args[0].strip()
+
+ conn_str = options.conn_str.strip()
if not re.match("^\w+://.+", conn_str):
conn_str = 'sqlite:///' + conn_str
-
engine, metadata, Session = models.setup_database(conn_str, echo=((options.verbose-options.quiet)>0), create_all=True)
session = None
try:
--- a/script/utils/search_topsy.py Tue Nov 05 11:57:19 2013 +0100
+++ b/script/utils/search_topsy.py Wed Nov 06 02:42:13 2013 +0100
@@ -1,25 +1,26 @@
-from blessings import Terminal
-from iri_tweet import models, utils
-from iri_tweet.processor import TwitterProcessorStatus
-from optparse import OptionParser
+import argparse
import logging
import math
import re
+import time
+
+from blessings import Terminal
import requests
-import time
import twitter
+from iri_tweet import models, utils
+from iri_tweet.processor import TwitterProcessorStatus
+
+
logger = logging.getLogger(__name__)
APPLICATION_NAME = "Tweet recorder user"
-CONSUMER_KEY = "Vdr5ZcsjI1G3esTPI8yDg"
-CONSUMER_SECRET = "LMhNrY99R6a7E0YbZZkRFpUZpX5EfB1qATbDk1sIVLs"
class TopsyResource(object):
def __init__(self, query, **kwargs):
-
+
self.options = kwargs
self.options['q'] = query
self.url = kwargs.get("url", "http://otter.topsy.com/search.json")
@@ -32,7 +33,7 @@
params = {}
params.update(self.options)
self.req = requests.get(self.url, params=params)
- self.res = self.req.json
+ self.res = self.req.json()
def __next_page(self):
page = self.res.get("response").get("page") + 1
@@ -40,7 +41,7 @@
params.update(self.options)
params['page'] = page
self.req = requests.get(self.url, params=params)
- self.res = self.req.json
+ self.res = self.req.json()
def __iter__(self):
if not self.req:
@@ -58,19 +59,25 @@
-def get_option():
+def get_options():
- parser = OptionParser()
+ usage = "usage: %(prog)s [options] <connection_str_or_filepath>"
+
+ parser = argparse.ArgumentParser(usage=usage)
- parser.add_option("-d", "--database", dest="database",
- help="Input database", metavar="DATABASE")
- parser.add_option("-Q", dest="query",
+ parser.add_argument(dest="conn_str",
+ help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR")
+ parser.add_argument("-Q", dest="query",
help="query", metavar="QUERY")
- parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",
+ parser.add_argument("-k", "--key", dest="consumer_key",
+ help="Twitter consumer key", metavar="CONSUMER_KEY")
+ parser.add_argument("-s", "--secret", dest="consumer_secret",
+ help="Twitter consumer secret", metavar="CONSUMER_SECRET")
+ parser.add_argument("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",
help="Token file name")
- parser.add_option("-T", dest="topsy_apikey", metavar="TOPSY_APIKEY", default=None,
+ parser.add_argument("-T", dest="topsy_apikey", metavar="TOPSY_APIKEY", default=None,
help="Topsy apikey")
-
+
utils.set_logging_options(parser)
return parser.parse_args()
@@ -79,17 +86,17 @@
if __name__ == "__main__":
- (options, args) = get_option()
+ options = get_options()
utils.set_logging(options);
- acess_token_key, access_token_secret = utils.get_oauth_token(consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET, options.token_filename, application_name=APPLICATION_NAME)
+ acess_token_key, access_token_secret = utils.get_oauth_token(consumer_key=options.consumer_key, consumer_secret=options.consumer_secret, token_file_path=options.token_filename, application_name=APPLICATION_NAME)
- t = twitter.Twitter(domain="api.twitter.com", auth=twitter.OAuth(acess_token_key, access_token_secret, CONSUMER_KEY, CONSUMER_SECRET), secure=True)
+ t = twitter.Twitter(domain="api.twitter.com", auth=twitter.OAuth(acess_token_key, access_token_secret, options.consumer_key, options.consumer_secret), secure=True)
t.secure = True
- conn_str = options.database.strip()
+ conn_str = options.conn_str.strip()
if not re.match("^\w+://.+", conn_str):
conn_str = 'sqlite:///' + conn_str
@@ -146,10 +153,10 @@
processor.process()
session.flush()
session.commit()
-
+
time_to_sleep = int(math.ceil((tweet.rate_limit_reset - time.mktime(time.gmtime())) / tweet.rate_limit_remaining))
- print "rate limit remaining %s of %s" % (str(tweet.rate_limit_remaining), str(tweet.headers.getheader('x-ratelimit-limit'))) + term.clear_eol()
+ print("rate limit remaining %s of %s" % (str(tweet.rate_limit_remaining), str(tweet.headers.getheader('X-Rate-Limit-Limit'))) + term.clear_eol())
move_up += 1
for i in xrange(time_to_sleep):
if i: