--- a/script/lib/iri_tweet/utils.py	Tue Dec 20 16:26:34 2011 +0100
+++ b/script/lib/iri_tweet/utils.py	Sat Jan 07 16:12:44 2012 +0100
@@ -10,6 +10,7 @@
 import logging
 import os.path
 import sys
+import math
 import twitter.oauth #@UnresolvedImport
 import twitter.oauth_dance #@UnresolvedImport
 import twitter_text #@UnresolvedImport
@@ -171,7 +172,7 @@
 
 class TwitterProcessor(object):
     
-    def __init__(self, json_dict, json_txt, source_id, session, access_token=None, token_filename=None):
+    def __init__(self, json_dict, json_txt, source_id, session, access_token=None, token_filename=None, user_query_twitter=False):
 
         if json_dict is None and json_txt is None:
             raise TwitterProcessorException("No json")
@@ -194,10 +195,11 @@
         self.token_filename = token_filename
         self.access_token = access_token
         self.obj_buffer = ObjectsBuffer()
+        self.user_query_twitter = user_query_twitter  
         
 
 
-    def __get_user(self, user_dict, do_merge, query_twitter = False):
+    def __get_user(self, user_dict, do_merge):
         get_logger().debug("Get user : " + repr(user_dict)) #@UndefinedVariable
         
         user_dict = adapt_fields(user_dict, fields_adapter["stream"]["user"])
@@ -243,7 +245,7 @@
     
         user_created_at = user_dict.get("created_at", None)
         
-        if user_created_at is None and query_twitter:
+        if user_created_at is None and self.user_query_twitter:
             
             if self.access_token is not None:
                 acess_token_key, access_token_secret = self.access_token
@@ -333,7 +335,7 @@
             return EntityHashtag, entity_dict             
         
         def process_user_mentions():
-            user_mention = self.__get_user(ind, False, False)
+            user_mention = self.__get_user(ind, False)
             if user_mention is None:
                 entity_dict['user_id'] = None
             else:
@@ -598,3 +600,17 @@
             raise
         except:
             self.handleError(record)
+
+def show_progress(current_line, total_line, label, width):
+
+    percent = (float(current_line) / float(total_line)) * 100.0
+
+    marks = math.floor(width * (percent / 100.0))
+    spaces = math.floor(width - marks)
+
+    loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']'
+    
+    sys.stdout.write(u"%s %d%% %d/%d - %r\r" % (loader, percent, current_line - 1, total_line - 1, label[:50].rjust(50))) #takes the header into account
+    if percent >= 100:
+        sys.stdout.write("\n")
+    sys.stdout.flush()