diff -r 2b794b7901d6 -r 4c3a15877f80 script/lib/iri_tweet/export_twitter_alchemy.py --- a/script/lib/iri_tweet/export_twitter_alchemy.py Tue Apr 26 10:53:35 2011 +0200 +++ b/script/lib/iri_tweet/export_twitter_alchemy.py Tue Apr 26 13:57:29 2011 +0200 @@ -2,26 +2,20 @@ # coding=utf-8 from lxml import etree -from models import * -from optparse import OptionParser -from sqlalchemy import Table, Column, Integer, BigInteger, String, MetaData, \ - ForeignKey -from sqlalchemy.orm import sessionmaker, mapper -from sqlalchemy.sql import select, or_ -from utils import * +from optparse import OptionParser #@UnresolvedImport +from sqlalchemy import Table, Column, BigInteger, MetaData +from sqlalchemy.orm import sessionmaker +from utils import parse_date, set_logging_options, set_logging, get_filter_query +from models import setup_database import datetime -import email.utils import logging -import os import os.path import re -import re import sys import time -import uuid +import uuid #@UnresolvedImport import httplib2 import anyjson -import StringIO #class TweetExclude(object): # def __init__(self, id): @@ -30,9 +24,6 @@ # def __repr__(self): # return "" % (self.id) -def parse_date(date_str): - ts = email.utils.parsedate_tz(date_str) - return datetime.datetime(*ts[0:7]) def parse_polemics(tw, extended_mode): """ @@ -108,7 +99,7 @@ set_logging(options) - logging.debug("OPTIONS : " + repr(options)) + logging.debug("OPTIONS : " + repr(options)) #@UndefinedVariable if len(sys.argv) == 1 or options.database is None: parser.print_help() @@ -164,7 +155,7 @@ for params in parameters: - logging.debug("PARAMETERS " + repr(params)) + logging.debug("PARAMETERS " + repr(params)) #@UndefinedVariable start_date_str = params.get("start_date",None) end_date_str = params.get("end_date", None) @@ -194,12 +185,12 @@ if content_file and content_file.find("http") == 0: - logging.debug("url : " + content_file) + logging.debug("url : " + content_file) #@UndefinedVariable h = httplib2.Http() resp, content = h.request(content_file) - logging.debug("url response " + repr(resp) + " content " + repr(content)) + logging.debug("url response " + repr(resp) + " content " + repr(content)) #@UndefinedVariable project = anyjson.deserialize(content) root = etree.fromstring(project["ldt"]) @@ -214,7 +205,7 @@ root = etree.Element(u"iri") - project = etree.SubElement(root, u"project", {u"abstract":u"Twitter comments on ENMI",u"title":u"Twitter comments on ENMI 2009", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())}) + project = etree.SubElement(root, u"project", {u"abstract":u"Polemics Tweets",u"title":u"Polemic Tweets", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())}) medias = etree.SubElement(root, u"medias") media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(options.content), u"video":unicode(options.video), u"id":unicode(options.content_id), u"extra":u""}) @@ -256,7 +247,7 @@ if ensemble_parent is None: - logging.error("Can not process file") + logging.error("Can not process file") #@UndefinedVariable sys.exit() if options.replace: @@ -311,18 +302,18 @@ project["ldt"] = output_data body = anyjson.serialize(project) - logging.debug("write http " + content_file) - logging.debug("write http " + repr(body)) + logging.debug("write http " + content_file) #@UndefinedVariable + logging.debug("write http " + repr(body)) #@UndefinedVariable h = httplib2.Http() resp, content = h.request(content_file, "PUT", headers={'content-type':'application/json'}, body=body) - logging.debug("write http " + repr(resp) + " content " + content) + logging.debug("write http " + repr(resp) + " content " + content) #@UndefinedVariable else: if content_file and os.path.exists(content_file): dest_file_name = content_file else: dest_file_name = options.filename - logging.debug("WRITE : " + dest_file_name) + logging.debug("WRITE : " + dest_file_name) #@UndefinedVariable output = open(dest_file_name, "w") output.write(output_data) output.flush()