diff -r c428119e8399 -r 5dce89631093 script/rest/search_twitter.py --- a/script/rest/search_twitter.py Sat Jun 01 01:24:36 2013 +0200 +++ b/script/rest/search_twitter.py Sat Jun 01 01:32:22 2013 +0200 @@ -21,10 +21,10 @@ help="Result per page") parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", help="Token file name") - parser.add_argument("-k", "--key", dest="consumer_key", - help="Twitter consumer key", metavar="CONSUMER_KEY", required=True) - parser.add_argument("-s", "--secret", dest="consumer_secret", - help="Twitter consumer secret", metavar="CONSUMER_SECRET", required=True) + parser.add_option("-k", "--key", dest="consumer_key", + help="Twitter consumer key", metavar="CONSUMER_KEY") + parser.add_option("-s", "--secret", dest="consumer_secret", + help="Twitter consumer secret", metavar="CONSUMER_SECRET") return parser.parse_args() @@ -60,16 +60,26 @@ results = None page = 1 print options.query + + #get current_maxid + results = t.search.tweets(q=options.query, result_type="recent") + max_id = results.get('search_metadata',{}).get('max_id',0) + if max_id==0: + print("No results, exit") + sys.exit(0) - while page <= int(1500/int(options.rpp)) and ( results is None or len(results) > 0): - results = t.search.tweets(q=options.query, rpp=options.rpp, page=page, include_entities=True) + while page <= int(1500/int(options.rpp)) and \ + ( results is None or len(results.get('statuses',0)) > 0) and \ + max_id > 0: + results = t.search.tweets(q=options.query, count=options.rpp, max_id=max_id, include_entities=True, result_type='recent') + max_id = results.get('search_metadata',{}).get('since_id',1) - 1 - for tweet in results["results"]: + for tweet in results["statuses"]: print tweet tweet_str = anyjson.serialize(tweet) #invalidate user id - processor = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=None, access_token=None, token_filename=options.token_filename, user_query_twitter=False, logger=None) - processor.process() + p = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=(options.consumer_key, options.consumer_secret), access_token=access_token, token_filename=options.token_filename, user_query_twitter=False, logger=None) + p.process() session.flush() session.commit() page += 1