19 help="query", metavar="QUERY") |
19 help="query", metavar="QUERY") |
20 parser.add_option("-P", dest="rpp", metavar="RPP", default="50", |
20 parser.add_option("-P", dest="rpp", metavar="RPP", default="50", |
21 help="Result per page") |
21 help="Result per page") |
22 parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", |
22 parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", |
23 help="Token file name") |
23 help="Token file name") |
24 parser.add_argument("-k", "--key", dest="consumer_key", |
24 parser.add_option("-k", "--key", dest="consumer_key", |
25 help="Twitter consumer key", metavar="CONSUMER_KEY", required=True) |
25 help="Twitter consumer key", metavar="CONSUMER_KEY") |
26 parser.add_argument("-s", "--secret", dest="consumer_secret", |
26 parser.add_option("-s", "--secret", dest="consumer_secret", |
27 help="Twitter consumer secret", metavar="CONSUMER_SECRET", required=True) |
27 help="Twitter consumer secret", metavar="CONSUMER_SECRET") |
28 |
28 |
29 return parser.parse_args() |
29 return parser.parse_args() |
30 |
30 |
31 def get_auth(options, access_token): |
31 def get_auth(options, access_token): |
32 consumer_key = options.consumer_key |
32 consumer_key = options.consumer_key |
58 #conn.commit() |
58 #conn.commit() |
59 |
59 |
60 results = None |
60 results = None |
61 page = 1 |
61 page = 1 |
62 print options.query |
62 print options.query |
|
63 |
|
64 #get current_maxid |
|
65 results = t.search.tweets(q=options.query, result_type="recent") |
|
66 max_id = results.get('search_metadata',{}).get('max_id',0) |
|
67 if max_id==0: |
|
68 print("No results, exit") |
|
69 sys.exit(0) |
63 |
70 |
64 while page <= int(1500/int(options.rpp)) and ( results is None or len(results) > 0): |
71 while page <= int(1500/int(options.rpp)) and \ |
65 results = t.search.tweets(q=options.query, rpp=options.rpp, page=page, include_entities=True) |
72 ( results is None or len(results.get('statuses',0)) > 0) and \ |
|
73 max_id > 0: |
|
74 results = t.search.tweets(q=options.query, count=options.rpp, max_id=max_id, include_entities=True, result_type='recent') |
|
75 max_id = results.get('search_metadata',{}).get('since_id',1) - 1 |
66 |
76 |
67 for tweet in results["results"]: |
77 for tweet in results["statuses"]: |
68 print tweet |
78 print tweet |
69 tweet_str = anyjson.serialize(tweet) |
79 tweet_str = anyjson.serialize(tweet) |
70 #invalidate user id |
80 #invalidate user id |
71 processor = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=None, access_token=None, token_filename=options.token_filename, user_query_twitter=False, logger=None) |
81 p = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=(options.consumer_key, options.consumer_secret), access_token=access_token, token_filename=options.token_filename, user_query_twitter=False, logger=None) |
72 processor.process() |
82 p.process() |
73 session.flush() |
83 session.flush() |
74 session.commit() |
84 session.commit() |
75 page += 1 |
85 page += 1 |
76 #session.commit() |
86 #session.commit() |
77 finally: |
87 finally: |