11 help="verbose", metavar="VERBOSE", default=0) |
13 help="verbose", metavar="VERBOSE", default=0) |
12 parser.add_option("-q", dest="quiet", action="count", |
14 parser.add_option("-q", dest="quiet", action="count", |
13 help="quiet", metavar="QUIET", default=0) |
15 help="quiet", metavar="QUIET", default=0) |
14 parser.add_option("-r", "--request", dest="request", |
16 parser.add_option("-r", "--request", dest="request", |
15 help="twitter request", metavar="REQUEST", default=0) |
17 help="twitter request", metavar="REQUEST", default=0) |
|
18 parser.add_option("-Q", dest="query", |
|
19 help="query", metavar="QUERY") |
|
20 parser.add_option("-P", dest="rpp", metavar="RPP", default="50", |
|
21 help="Result per page", metavar="RPP") |
|
22 |
16 #add request token |
23 #add request token |
17 #add |
24 #add |
18 |
25 |
19 return parser.parse_args() |
26 return parser.parse_args() |
20 |
27 |
21 if __name__ == "__main__": |
28 if __name__ == "__main__": |
22 |
29 |
23 twitter = twython.Twython() |
30 (options, args) = get_option() |
24 conn = sqlite3.connect('enmi2010_twitter_rest.db') |
31 |
|
32 twitter = twitter.Twitter(domain="search.twitter.com") |
|
33 engine, metadata = models.setup_database('sqlite:///'+args[0], echo=((options.verbose-options.quiet)>0)) |
|
34 Session = sessionmaker(bind=engine) |
|
35 session = Session() |
25 try: |
36 try: |
26 conn.row_factory = sqlite3.Row |
37 #conn.row_factory = sqlite3.Row |
27 curs = conn.cursor() |
38 #curs = conn.cursor() |
28 curs.execute("create table if not exists tweet_tweet (json);") |
39 #curs.execute("create table if not exists tweet_tweet (json);") |
29 conn.commit() |
40 #conn.commit() |
30 |
41 |
31 results = twitter.searchTwitter(q="#enmi", rpp="50") |
42 results = None |
32 for tweet in results["results"]: |
43 page = 1 |
33 print tweet |
44 |
34 curs.execute("insert into tweet_tweet (json) values (:json);", {"json":unicode(tweet)}) |
45 while page <= int(1500/int(options.rpp)) and ( results is None or len(results) > 0): |
35 conn.commit() |
46 results = twitter. search(q=options.query, rpp=options.rpp, page=page) |
|
47 for tweet in results["results"]: |
|
48 print tweet |
|
49 processor = utils.TwitterProcessor(tweet, None, session) |
|
50 processor.process() |
|
51 session.flush() |
|
52 page += 1 |
|
53 session.commit() |
36 finally: |
54 finally: |
37 conn.close() |
55 session.close() |
38 |
56 |
39 |
57 |