1 from models import * |
1 from models import * |
|
2 import anyjson |
2 import datetime |
3 import datetime |
3 import email.utils |
4 import email.utils |
4 import anyjson |
|
5 import logging |
5 import logging |
6 import os.path |
6 import os.path |
7 import sys |
7 import sys |
8 import twitter |
8 import twitter |
9 import twitter.oauth |
9 import twitter.oauth |
10 import twitter.oauth_dance |
10 import twitter.oauth_dance |
11 import twitter_text |
11 import twitter_text |
12 |
12 |
13 |
13 |
|
14 CACHE_ACCESS_TOKEN = None |
|
15 |
14 def get_oauth_token(token_file_path=None): |
16 def get_oauth_token(token_file_path=None): |
|
17 |
|
18 global CACHE_ACCESS_TOKEN |
|
19 |
|
20 if CACHE_ACCESS_TOKEN is not None: |
|
21 return CACHE_ACCESS_TOKEN |
15 |
22 |
16 if token_file_path and os.path.exists(token_file_path): |
23 if token_file_path and os.path.exists(token_file_path): |
17 logging.debug("reading token from file %s" % token_file_path) |
24 logging.debug("reading token from file %s" % token_file_path) |
18 return twitter.oauth.read_token_file(token_file_path) |
25 CACHE_ACCESS_TOKEN = twitter.oauth.read_token_file(token_file_path) |
|
26 return CACHE_ACCESS_TOKEN |
19 #read access token info from path |
27 #read access token info from path |
20 |
28 |
21 if 'ACCESS_TOKEN_KEY' in dict() and 'ACCESS_TOKEN_SECRET' in dict() and ACCESS_TOKEN_KEY and ACCESS_TOKEN_SECRET: |
29 if 'ACCESS_TOKEN_KEY' in dict() and 'ACCESS_TOKEN_SECRET' in dict() and ACCESS_TOKEN_KEY and ACCESS_TOKEN_SECRET: |
22 return ACCESS_TOKEN_KEY,ACCESS_TOKEN_SECRET |
30 return ACCESS_TOKEN_KEY,ACCESS_TOKEN_SECRET |
23 |
31 |
24 return twitter.oauth_dance.oauth_dance(APPLICATION_NAME, CONSUMER_KEY, CONSUMER_SECRET, token_file_path) |
32 CACHE_ACCESS_TOKEN = twitter.oauth_dance.oauth_dance(APPLICATION_NAME, CONSUMER_KEY, CONSUMER_SECRET, token_file_path) |
|
33 return CACHE_ACCESS_TOKEN |
25 |
34 |
26 def parse_date(date_str): |
35 def parse_date(date_str): |
27 ts = email.utils.parsedate_tz(date_str) |
36 ts = email.utils.parsedate_tz(date_str) |
28 return datetime.datetime(*ts[0:7]) |
37 return datetime.datetime(*ts[0:7]) |
29 |
38 |
201 ts_copy = adapt_fields(self.json_dict, fields_adapter["stream"]["tweet"]) |
210 ts_copy = adapt_fields(self.json_dict, fields_adapter["stream"]["tweet"]) |
202 |
211 |
203 # get or create user |
212 # get or create user |
204 user = self.__get_user(self.json_dict["user"]) |
213 user = self.__get_user(self.json_dict["user"]) |
205 if user is None: |
214 if user is None: |
206 log.warning("USER not found " + repr(ts["user"])) |
215 logging.warning("USER not found " + repr(ts["user"])) |
207 ts_copy["user"] = None |
216 ts_copy["user"] = None |
208 ts_copy["user_id"] = None |
217 ts_copy["user_id"] = None |
209 else: |
218 else: |
210 ts_copy["user"] = user |
219 ts_copy["user"] = user |
211 ts_copy["user_id"] = ts_copy["user"].id |
220 ts_copy["user_id"] = ts_copy["user"].id |
249 'screen_name' : self.json_dict["from_user"], |
258 'screen_name' : self.json_dict["from_user"], |
250 } |
259 } |
251 |
260 |
252 user = self.__get_user(user_fields) |
261 user = self.__get_user(user_fields) |
253 if user is None: |
262 if user is None: |
254 log.warning("USER not found " + repr(user_fields)) |
263 logging.warning("USER not found " + repr(user_fields)) |
255 tweet_fields["user"] = None |
264 tweet_fields["user"] = None |
256 tweet_fields["user_id"] = None |
265 tweet_fields["user_id"] = None |
257 else: |
266 else: |
258 tweet_fields["user"] = user |
267 tweet_fields["user"] = user |
259 tweet_fields["user_id"] = user.id |
268 tweet_fields["user_id"] = user.id |
260 |
269 |
261 tweet_fields = adapt_fields(tweet_fields, fields_adapter["rest"]["tweet"]) |
270 tweet_fields = adapt_fields(tweet_fields, fields_adapter["rest"]["tweet"]) |
262 self.tweet = Tweet(**tweet_fields) |
271 self.tweet = Tweet(**tweet_fields) |
263 session.add(self.tweet) |
272 self.session.add(self.tweet) |
264 |
273 |
265 text = self.tweet.text |
274 text = self.tweet.text |
266 |
275 |
267 extractor = twitter_text.Extractor(text) |
276 extractor = twitter_text.Extractor(text) |
268 |
277 |