script/lib/iri_tweet/export_twitter_alchemy.py
changeset 122 4c3a15877f80
parent 84 b1029aa40ec3
child 203 8124cde38141
--- a/script/lib/iri_tweet/export_twitter_alchemy.py	Tue Apr 26 10:53:35 2011 +0200
+++ b/script/lib/iri_tweet/export_twitter_alchemy.py	Tue Apr 26 13:57:29 2011 +0200
@@ -2,26 +2,20 @@
 # coding=utf-8
 
 from lxml import etree
-from models import *
-from optparse import OptionParser
-from sqlalchemy import Table, Column, Integer, BigInteger, String, MetaData, \
-    ForeignKey
-from sqlalchemy.orm import sessionmaker, mapper
-from sqlalchemy.sql import select, or_
-from utils import *
+from optparse import OptionParser #@UnresolvedImport
+from sqlalchemy import Table, Column, BigInteger, MetaData
+from sqlalchemy.orm import sessionmaker
+from utils import parse_date, set_logging_options, set_logging, get_filter_query
+from models import setup_database
 import datetime
-import email.utils
 import logging
-import os
 import os.path
 import re
-import re
 import sys
 import time
-import uuid
+import uuid #@UnresolvedImport
 import httplib2
 import anyjson
-import StringIO
 
 #class TweetExclude(object):
 #    def __init__(self, id):
@@ -30,9 +24,6 @@
 #    def __repr__(self):
 #        return "<TweetExclude(id=%d)>" % (self.id)
 
-def parse_date(date_str):
-    ts = email.utils.parsedate_tz(date_str)
-    return datetime.datetime(*ts[0:7])
 
 def parse_polemics(tw, extended_mode):
     """
@@ -108,7 +99,7 @@
     
     set_logging(options)
         
-    logging.debug("OPTIONS : " + repr(options))
+    logging.debug("OPTIONS : " + repr(options)) #@UndefinedVariable
     
     if len(sys.argv) == 1 or options.database is None:
         parser.print_help()
@@ -164,7 +155,7 @@
             
             for params in parameters:
                 
-                logging.debug("PARAMETERS " + repr(params))
+                logging.debug("PARAMETERS " + repr(params)) #@UndefinedVariable
                 
                 start_date_str = params.get("start_date",None)
                 end_date_str = params.get("end_date", None)
@@ -194,12 +185,12 @@
                 
                 if content_file and content_file.find("http") == 0:
                     
-                    logging.debug("url : " + content_file)
+                    logging.debug("url : " + content_file) #@UndefinedVariable
                     
                     h = httplib2.Http()
                     resp, content = h.request(content_file)
                     
-                    logging.debug("url response " + repr(resp) + " content " + repr(content))
+                    logging.debug("url response " + repr(resp) + " content " + repr(content)) #@UndefinedVariable
                     
                     project = anyjson.deserialize(content)
                     root = etree.fromstring(project["ldt"])
@@ -214,7 +205,7 @@
                 
                     root = etree.Element(u"iri")
                         
-                    project = etree.SubElement(root, u"project", {u"abstract":u"Twitter comments on ENMI",u"title":u"Twitter comments on ENMI 2009", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())})
+                    project = etree.SubElement(root, u"project", {u"abstract":u"Polemics Tweets",u"title":u"Polemic Tweets", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())})
                 
                     medias = etree.SubElement(root, u"medias")
                     media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(options.content), u"video":unicode(options.video), u"id":unicode(options.content_id), u"extra":u""})
@@ -256,7 +247,7 @@
                     
                     
                 if ensemble_parent is None:
-                    logging.error("Can not process file")
+                    logging.error("Can not process file") #@UndefinedVariable
                     sys.exit()
             
                 if options.replace:
@@ -311,18 +302,18 @@
                     
                     project["ldt"] = output_data
                     body = anyjson.serialize(project)
-                    logging.debug("write http " + content_file)
-                    logging.debug("write http " + repr(body))
+                    logging.debug("write http " + content_file) #@UndefinedVariable
+                    logging.debug("write http " + repr(body)) #@UndefinedVariable
                     h = httplib2.Http()
                     resp, content = h.request(content_file, "PUT", headers={'content-type':'application/json'}, body=body)
-                    logging.debug("write http " + repr(resp) + " content " + content)
+                    logging.debug("write http " + repr(resp) + " content " + content) #@UndefinedVariable
                 else:
                     if content_file and os.path.exists(content_file):
                         dest_file_name = content_file 
                     else:
                         dest_file_name = options.filename
             
-                    logging.debug("WRITE : " + dest_file_name)
+                    logging.debug("WRITE : " + dest_file_name) #@UndefinedVariable
                     output = open(dest_file_name, "w")
                     output.write(output_data)
                     output.flush()