|
1 from iri_tweet import models, utils |
|
2 from sqlalchemy.orm import sessionmaker |
|
3 import anyjson |
|
4 import sqlite3 |
|
5 import twitter |
|
6 from optparse import OptionParser |
|
7 |
|
8 |
|
9 def get_option(): |
|
10 |
|
11 parser = OptionParser() |
|
12 |
|
13 parser.add_option("-l", "--log", dest="logfile", |
|
14 help="log to file", metavar="LOG", default="stderr") |
|
15 parser.add_option("-v", dest="verbose", action="count", |
|
16 help="verbose", metavar="VERBOSE", default=0) |
|
17 parser.add_option("-q", dest="quiet", action="count", |
|
18 help="quiet", metavar="QUIET", default=0) |
|
19 parser.add_option("-r", "--request", dest="request", |
|
20 help="twitter request", metavar="REQUEST", default=0) |
|
21 parser.add_option("-Q", dest="query", |
|
22 help="query", metavar="QUERY") |
|
23 parser.add_option("-P", dest="rpp", metavar="RPP", default="50", |
|
24 help="Result per page") |
|
25 parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", |
|
26 help="Token file name") |
|
27 |
|
28 |
|
29 #add request token |
|
30 #add |
|
31 |
|
32 return parser.parse_args() |
|
33 |
|
34 if __name__ == "__main__": |
|
35 |
|
36 (options, args) = get_option() |
|
37 |
|
38 twitter = twitter.Twitter(domain="search.twitter.com") |
|
39 engine, metadata = models.setup_database('sqlite:///'+args[0], echo=((options.verbose-options.quiet)>0)) |
|
40 Session = sessionmaker(bind=engine) |
|
41 session = Session() |
|
42 try: |
|
43 #conn.row_factory = sqlite3.Row |
|
44 #curs = conn.cursor() |
|
45 #curs.execute("create table if not exists tweet_tweet (json);") |
|
46 #conn.commit() |
|
47 |
|
48 results = None |
|
49 page = 1 |
|
50 |
|
51 while page <= int(1500/int(options.rpp)) and ( results is None or len(results) > 0): |
|
52 results = twitter. search(q=options.query, rpp=options.rpp, page=page) |
|
53 for tweet in results["results"]: |
|
54 print tweet |
|
55 tweet_str = anyjson.serialize(tweet) |
|
56 #invalidate user id |
|
57 processor = utils.TwitterProcessor(tweet, tweet_str, session, options.token_filename) |
|
58 processor.process() |
|
59 session.flush() |
|
60 session.commit() |
|
61 page += 1 |
|
62 #session.commit() |
|
63 finally: |
|
64 session.close() |
|
65 |
|
66 |