equal
deleted
inserted
replaced
8 import datetime |
8 import datetime |
9 import email.utils |
9 import email.utils |
10 import logging |
10 import logging |
11 import os.path |
11 import os.path |
12 import sys |
12 import sys |
|
13 import math |
13 import twitter.oauth #@UnresolvedImport |
14 import twitter.oauth #@UnresolvedImport |
14 import twitter.oauth_dance #@UnresolvedImport |
15 import twitter.oauth_dance #@UnresolvedImport |
15 import twitter_text #@UnresolvedImport |
16 import twitter_text #@UnresolvedImport |
16 |
17 |
17 |
18 |
169 class TwitterProcessorException(Exception): |
170 class TwitterProcessorException(Exception): |
170 pass |
171 pass |
171 |
172 |
172 class TwitterProcessor(object): |
173 class TwitterProcessor(object): |
173 |
174 |
174 def __init__(self, json_dict, json_txt, source_id, session, access_token=None, token_filename=None): |
175 def __init__(self, json_dict, json_txt, source_id, session, access_token=None, token_filename=None, user_query_twitter=False): |
175 |
176 |
176 if json_dict is None and json_txt is None: |
177 if json_dict is None and json_txt is None: |
177 raise TwitterProcessorException("No json") |
178 raise TwitterProcessorException("No json") |
178 |
179 |
179 if json_dict is None: |
180 if json_dict is None: |
192 self.source_id = source_id |
193 self.source_id = source_id |
193 self.session = session |
194 self.session = session |
194 self.token_filename = token_filename |
195 self.token_filename = token_filename |
195 self.access_token = access_token |
196 self.access_token = access_token |
196 self.obj_buffer = ObjectsBuffer() |
197 self.obj_buffer = ObjectsBuffer() |
197 |
198 self.user_query_twitter = user_query_twitter |
198 |
199 |
199 |
200 |
200 def __get_user(self, user_dict, do_merge, query_twitter = False): |
201 |
|
202 def __get_user(self, user_dict, do_merge): |
201 get_logger().debug("Get user : " + repr(user_dict)) #@UndefinedVariable |
203 get_logger().debug("Get user : " + repr(user_dict)) #@UndefinedVariable |
202 |
204 |
203 user_dict = adapt_fields(user_dict, fields_adapter["stream"]["user"]) |
205 user_dict = adapt_fields(user_dict, fields_adapter["stream"]["user"]) |
204 |
206 |
205 user_id = user_dict.get("id",None) |
207 user_id = user_dict.get("id",None) |
241 user = self.obj_buffer.add_object(User, None, user_dict, True, user_obj) |
243 user = self.obj_buffer.add_object(User, None, user_dict, True, user_obj) |
242 return user |
244 return user |
243 |
245 |
244 user_created_at = user_dict.get("created_at", None) |
246 user_created_at = user_dict.get("created_at", None) |
245 |
247 |
246 if user_created_at is None and query_twitter: |
248 if user_created_at is None and self.user_query_twitter: |
247 |
249 |
248 if self.access_token is not None: |
250 if self.access_token is not None: |
249 acess_token_key, access_token_secret = self.access_token |
251 acess_token_key, access_token_secret = self.access_token |
250 else: |
252 else: |
251 acess_token_key, access_token_secret = get_oauth_token(self.token_filename) |
253 acess_token_key, access_token_secret = get_oauth_token(self.token_filename) |
331 hashtag = self.__get_or_create_object(Hashtag, {'text':text}, Hashtag.text.ilike(text), ind, True, False) |
333 hashtag = self.__get_or_create_object(Hashtag, {'text':text}, Hashtag.text.ilike(text), ind, True, False) |
332 entity_dict['hashtag_id'] = hashtag.id |
334 entity_dict['hashtag_id'] = hashtag.id |
333 return EntityHashtag, entity_dict |
335 return EntityHashtag, entity_dict |
334 |
336 |
335 def process_user_mentions(): |
337 def process_user_mentions(): |
336 user_mention = self.__get_user(ind, False, False) |
338 user_mention = self.__get_user(ind, False) |
337 if user_mention is None: |
339 if user_mention is None: |
338 entity_dict['user_id'] = None |
340 entity_dict['user_id'] = None |
339 else: |
341 else: |
340 entity_dict['user_id'] = user_mention.id |
342 entity_dict['user_id'] = user_mention.id |
341 return EntityUser, entity_dict |
343 return EntityUser, entity_dict |
596 raise |
598 raise |
597 except (KeyboardInterrupt, SystemExit): |
599 except (KeyboardInterrupt, SystemExit): |
598 raise |
600 raise |
599 except: |
601 except: |
600 self.handleError(record) |
602 self.handleError(record) |
|
603 |
|
604 def show_progress(current_line, total_line, label, width): |
|
605 |
|
606 percent = (float(current_line) / float(total_line)) * 100.0 |
|
607 |
|
608 marks = math.floor(width * (percent / 100.0)) |
|
609 spaces = math.floor(width - marks) |
|
610 |
|
611 loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']' |
|
612 |
|
613 sys.stdout.write(u"%s %d%% %d/%d - %r\r" % (loader, percent, current_line - 1, total_line - 1, label[:50].rjust(50))) #takes the header into account |
|
614 if percent >= 100: |
|
615 sys.stdout.write("\n") |
|
616 sys.stdout.flush() |