--- a/script/stream/recorder_tweetstream.py	Wed Jan 12 13:25:01 2011 +0100
+++ b/script/stream/recorder_tweetstream.py	Tue Jan 18 10:08:03 2011 +0100
@@ -1,10 +1,15 @@
+from getpass import getpass
+from iri_tweet import models, utils
+from optparse import OptionParser
+from sqlalchemy.orm import sessionmaker
+from sqlite3 import *
+import logging
+import os
+import socket
+import sys
 import tweetstream
-from getpass import getpass
-import socket
 socket._fileobject.default_bufsize = 0
-from sqlite3 import *
-from optparse import OptionParser
-import os
+
 
 
 #columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user']
@@ -63,11 +68,13 @@
 
 
 
-def process_tweet(tweet, cursor, debug):
-    print tweet
-    cursor.execute("insert into tweet_tweet (json) values (:json);", {"json":unicode(tweet)});
+def process_tweet(tweet, session, debug):
+    
+    logging.debug("Process_tweet :" + repr(tweet))
+    processor = utils.TwitterProcessor(tweet, None, session)
+    processor.process()
 
-def main(username, password, track, curs, debug, reconnects):
+def main(username, password, track, session, debug, reconnects):
 
     username = username or raw_input('Twitter username: ')
     password = password or getpass('Twitter password: ')
@@ -78,12 +85,12 @@
     stream = ReconnectingTweetStream(username, password, track_list, reconnects=reconnects)
     try:
         for tweet in stream:
-            process_tweet(tweet, curs, debug)
+            process_tweet(tweet, session, debug)
+            session.commit()
     finally:
         stream.close()
-
-if __name__ == '__main__':
-    
+        
+def get_options():
     parser = OptionParser()
     parser.add_option("-f", "--file", dest="filename",  
                       help="write tweet to FILE", metavar="FILE", default="enmi2010_twitter.db")
@@ -95,14 +102,21 @@
                       help="Twitter track", metavar="TRACK")
     parser.add_option("-n", "--new", dest="new", action="store_true",
                       help="new database", default=False)
-    parser.add_option("-d", "--debug", dest="debug", action="store_true",
-                      help="debug", default=False)
     parser.add_option("-r", "--reconnects", dest="reconnects",
                       help="Reconnects", metavar="RECONNECTS", default=10, type='int')
+    
+    utils.set_logging_options(parser)
 
+    return parser.parse_args()
+    
 
-    (options, args) = parser.parse_args()
+if __name__ == '__main__':
     
+
+    (options, args) = get_options()
+    
+    utils.set_logging(options)
+        
     if options.debug:
         print "OPTIONS : "
         print repr(options)
@@ -110,16 +124,15 @@
     if options.new and os.path.exists(options.filename):
         os.remove(options.filename)
     
-    conn = connect(options.filename)
+    engine, metadata = models.setup_database('sqlite:///'+options.filename, echo=(options.debug))
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
     try:
-        conn.row_factory = Row
-        curs = conn.cursor()
-    
-        curs.execute("create table if not exists tweet_tweet (json);")
-    
         try:
-            main(options.username, options.password, options.track, curs, options.debug, options.reconnects)
+            main(options.username, options.password, options.track, session, options.debug, options.reconnects)
         except KeyboardInterrupt:
             print '\nGoodbye!'
+        session.commit()
     finally:
-        conn.close()
+        session.close()