rest tweets and stream tweets are now the same
authorYves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
Sat, 01 Jun 2013 01:24:36 +0200
changeset 915 c428119e8399
parent 914 43876221071f
child 916 5dce89631093
rest tweets and stream tweets are now the same
script/lib/iri_tweet/iri_tweet/processor.py
--- a/script/lib/iri_tweet/iri_tweet/processor.py	Sat Jun 01 00:20:41 2013 +0200
+++ b/script/lib/iri_tweet/iri_tweet/processor.py	Sat Jun 01 01:24:36 2013 +0200
@@ -232,7 +232,7 @@
             self.obj_buffer.add_object(entity_klass, None, entity_dict, False)
 
 
-    def __process_twitter_stream(self):
+    def __process_twitter(self):
         
         tweet_nb = self.session.query(Tweet).filter(Tweet.id == self.json_dict["id"]).count()
         if tweet_nb > 0:
@@ -274,50 +274,6 @@
             for ind in extractor.extract_mentioned_screen_names_with_indices():
                 self.__process_entity(ind, "user_mentions")
 
-    def __process_twitter_rest(self):
-        tweet_nb = self.session.query(Tweet).filter(Tweet.id == self.json_dict["id"]).count()
-        if tweet_nb > 0:
-            return
-        
-        
-        tweet_fields = {
-            'created_at': self.json_dict["created_at"], 
-            'favorited': False,
-            'id': self.json_dict["id"],
-            'id_str': self.json_dict["id_str"],
-            #'in_reply_to_screen_name': ts["to_user"], 
-            'in_reply_to_user_id': self.json_dict.get("in_reply_to_user_id",None),
-            'in_reply_to_user_id_str': self.json_dict.get("in_reply_to_user_id_str", None),
-            #'place': ts["place"],
-            'source': self.json_dict["source"],
-            'text': self.json_dict["text"],
-            'truncated': False,            
-            'tweet_source_id' : self.source_id,
-        }
-        
-        #user
-    
-        user_fields = {
-            'lang' : self.json_dict.get('iso_language_code',None),
-            'profile_image_url' : self.json_dict["profile_image_url"],
-            'screen_name' : self.json_dict["from_user"],
-            'id' : self.json_dict["from_user_id"],
-            'id_str' : self.json_dict["from_user_id_str"],
-            'name' : self.json_dict['from_user_name'],
-        }
-        
-        user = self.__get_user(user_fields, do_merge=False)
-        if user is None:
-            self.logger.warning("USER not found " + repr(user_fields)) #@UndefinedVariable
-            tweet_fields["user_id"] = None
-        else:
-            tweet_fields["user_id"] = user.id
-        
-        tweet_fields = adapt_fields(tweet_fields, fields_adapter["rest"]["tweet"])
-        self.tweet = self.obj_buffer.add_object(Tweet, None, tweet_fields, True)
-                
-        self.__process_entities()
-
 
 
     def process_source(self):
@@ -328,10 +284,7 @@
             self.obj_buffer.add_object(TweetLog, log, {'status': TweetLog.TWEET_STATUS['DELETE'], 'status_id': None})
             self.session.query(TweetSource).filter(TweetSource.id==self.source_id).delete()
         else:
-            if "metadata" in self.json_dict:
-                self.__process_twitter_rest()
-            else:
-                self.__process_twitter_stream()
+            self.__process_twitter()
 
         self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['OK']}, True)