#!/usr/bin/env python
# coding=utf-8

from lxml import etree
from models import *
from optparse import OptionParser
from sqlalchemy import Table, Column, Integer, BigInteger, String, MetaData, \
    ForeignKey
from sqlalchemy.orm import sessionmaker, mapper
from sqlalchemy.sql import select
from utils import *
import datetime
import email.utils
import logging
import os
import os.path
import re
import re
import sys
import time
import uuid
import httplib2
import anyjson
import StringIO

#class TweetExclude(object):
#    def __init__(self, id):
#        self.id = id
#        
#    def __repr__(self):
#        return "<TweetExclude(id=%d)>" % (self.id)

def parse_date(date_str):
    ts = email.utils.parsedate_tz(date_str)
    return datetime.datetime(*ts[0:7])

def get_options():
    parser = OptionParser()
    parser.add_option("-f", "--file", dest="filename",
                      help="write export to file", metavar="FILE", default="project_enmi.ldt")
    parser.add_option("-d", "--database", dest="database",
                      help="Input database", metavar="DATABASE")
    parser.add_option("-s", "--start-date", dest="start_date",
                      help="start date", metavar="START_DATE")
    parser.add_option("-e", "--end-date", dest="end_date",
                      help="end date", metavar="END_DATE")
    parser.add_option("-I", "--content-file", dest="content_file",
                      help="Content file", metavar="CONTENT_FILE")
    parser.add_option("-c", "--content", dest="content",
                      help="Content url", metavar="CONTENT")
    parser.add_option("-V", "--video-url", dest="video",
                      help="video url", metavar="VIDEO")
    parser.add_option("-i", "--content-id", dest="content_id",
                      help="Content id", metavar="CONTENT_ID")
    parser.add_option("-x", "--exclude", dest="exclude",
                      help="file containing the id to exclude", metavar="EXCLUDE")
    parser.add_option("-C", "--color", dest="color",
                      help="Color code", metavar="COLOR", default="16763904")
    parser.add_option("-H", "--hashtag", dest="hashtag",
                      help="Hashtag", metavar="HASHTAG", default="enmi")                      
    parser.add_option("-D", "--duration", dest="duration", type="int",
                      help="Duration", metavar="DURATION", default=None)
    parser.add_option("-n", "--name", dest="name",
                      help="Cutting name", metavar="NAME", default=u"Tweets")
    parser.add_option("-R", "--replace", dest="replace", action="store_true",
                      help="Replace tweet ensemble", metavar="REPLACE", default=False)
    parser.add_option("-L", "--list-conf", dest="listconf",
                      help="list of file to process", metavar="LIST_CONF", default=None)
    
    set_logging_options(parser)

    
    return parser.parse_args()


if __name__ == "__main__" :

    (options, args) = get_options()
        
    set_logging(options)
        
    logging.debug("OPTIONS : " + repr(options))

    engine, metadata = setup_database('sqlite:///'+options.database, echo=((options.verbose-options.quiet)>0), create_all = False)        
    
    Session = sessionmaker()
    conn = engine.connect()
    try :
        session = Session(bind=conn)
        try : 
        
            metadata = MetaData(bind=conn)
            tweet_exclude_table = Table("tweet_exclude", metadata, Column('id', BigInteger, primary_key=True), prefixes=['TEMPORARY'])
            #mapper(TweetExclude, tweet_exclude_table)
            metadata.create_all()
            
            if options.exclude and os.path.exists(options.exclude):
                with open(options.exclude, 'r+') as f:
                    tei = tweet_exclude_table.insert()
                    for line in f:
                        conn.execute(tei.values(id=long(line.strip())))

            if options.listconf:
                
                parameters = []
                confdoc = etree.parse(options.listconf)
                for node in confdoc.xpath("/twitter_export/file"):
                    params = {}
                    for snode in node:
                        if snode.tag == "path":
                            params['content_file'] = snode.text
                        elif snode.tag == "start_date":
                            params['start_date'] = snode.text
                        elif snode.tag == "end_date":
                            params['end_date'] = snode.text
                        elif snode.tag == "duration":
                            params['duration'] = int(snode.text)
                    parameters.append(params)
            else:                        
                parameters = [{
                    'start_date': options.start_date,
                    'end_date' : options.end_date,
                    'duration' : options.duration,
                    'content_file' : options.content_file
                    
                }]
            
            for params in parameters:
                
                logging.debug("PARAMETERS " + repr(params))
                
                start_date_str = params.get("start_date",None)
                end_date_str = params.get("end_date", None)
                duration = params.get("duration", None)
                content_file = params.get("content_file", None)
                
                
                start_date = parse_date(start_date_str) 
                ts = time.mktime(start_date.timetuple())
            
                if end_date_str:
                    end_date = parse_date(end_date_str)
                    te = time.mktime(end_date.timetuple())
                else:
                    te = ts + duration
                    end_date = start_date + datetime.timedelta(seconds=duration)
        
            
                query_res = session.query(Tweet).join(EntityHashtag).join(Hashtag).filter(~Tweet.id.in_(select([tweet_exclude_table.c.id]))).filter(Hashtag.text.contains(options.hashtag)).filter(Tweet.created_at >=  start_date).filter(Tweet.created_at <=  end_date).all()
                                 
                root = None
                ensemble_parent = None
                
                #to do : analyse situation ldt or iri ? filename set or not ?
                
                if content_file and content_file.find("http") == 0:
                    
                    logging.debug("url : " + content_file)
                    
                    h = httplib2.Http()
                    resp, content = h.request(content_file)
                    
                    logging.debug("url response " + repr(resp) + " content " + repr(content))
                    
                    project = anyjson.deserialize(content)
                    root = etree.fromstring(project["ldt"])
                
                elif content_file and os.path.exists(content_file):

                    doc = etree.parse(content_file)
                    root = doc.getroot()
                    
                
                if root is None:
                
                    root = etree.Element(u"iri")
                        
                    project = etree.SubElement(root, u"project", {u"abstract":u"Twitter comments on ENMI",u"title":u"Twitter comments on ENMI 2009", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())})
                
                    medias = etree.SubElement(root, u"medias")
                    media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(options.content), u"video":unicode(options.video), u"id":unicode(options.content_id), u"extra":u""})
                    
                    annotations = etree.SubElement(root, u"annotations")    
                    content = etree.SubElement(annotations, u"content", {u"id":unicode(options.content_id)})
                    ensemble_parent = content
                    
                
                if ensemble_parent is None:
                    file_type = None
                    for node in root:
                        if node.tag == "project":
                            file_type = "ldt"
                            break
                        elif node.tag == "head":
                            file_type = "iri"
                            break
                    
                    if file_type == "ldt":
                        media_nodes = root.xpath("//media")
                        if len(media_nodes) > 0:
                            media = media_nodes[0]
                            annotations_node = root.find(u"annotations")
                            if annotations_node is None:
                                annotations_node = etree.SubElement(root, u"annotations")
                            content_node = annotations_node.find(u"content")
                            if content_node is None:
                                content_node = etree.SubElement(annotations_node,u"content", id=media["id"])
                            ensemble_parent = content_node
                    elif file_type == "iri":
                        body_node = root.find(u"body")
                        if body_node is None:
                            body_node = etree.SubElement(root, u"body")
                        ensembles_node = body_node.find(u"ensembles")
                        if ensembles_node is None:
                            ensembles_node = etree.SubElement(body_node, u"ensembles")
                        ensemble_parent = ensembles_node
                    
                    
                if ensemble_parent is None:
                    logging.error("Can not process file")
                    sys.exit()
            
                if options.replace:
                    for ens in ensemble_parent.iterchildren(tag=u"ensemble"):
                        if ens.get("id","").startswith("tweet_"):
                            ensemble_parent.remove(ens)
            
                ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"tweet_" + unicode(uuid.uuid4()), u"title":u"Ensemble Twitter", u"author":u"IRI Web", u"abstract":u"Ensemble Twitter"})
                decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"})
                
                etree.SubElement(decoupage, u"title").text = unicode(options.name)
                etree.SubElement(decoupage, u"abstract").text = unicode(options.name)
                
                elements = etree.SubElement(decoupage, u"elements")
                
                for tw in query_res:
                    tweet_ts_dt = tw.created_at
                    tweet_ts = int(time.mktime(tweet_ts_dt.timetuple()))
                    tweet_ts_rel = (tweet_ts-ts) * 1000
                    username = None
                    if tw.user is not None:
                        username = tw.user.name
                    if not username:
                        username = "anon."
                    element = etree.SubElement(elements, u"element" , {u"id":unicode(uuid.uuid4())+u"-"+unicode(tw.id), u"color":unicode(options.color), u"author":unicode(username), u"date":unicode(tweet_ts_dt.strftime("%Y/%m/%d")), u"begin": unicode(tweet_ts_rel), u"dur":u"0", u"src":u""})
                    etree.SubElement(element, u"title").text = unicode(username) + u": " + unicode(tw.text)
                    etree.SubElement(element, u"abstract").text = unicode(tw.text)
            
                    tags_node = etree.SubElement(element, u"tags")
                    
                    for entity in tw.entity_list:
                        if entity.type == u'entity_hashtag': 
                            etree.SubElement(tags_node,u"tag").text = entity.hashtag.text
                            
                    meta_element = etree.SubElement(element, u'meta')
                    
                    polemics_element = etree.Element(u'polemics')
                    polemic_added = False
                    for m in re.finditer("(\+\+|\-\-|\?\?|\=\=)",tw.text):
                        polemic_added = True
                        pol_link = {
                            '++' : u'OK',
                            '--' : u'KO',
                            '??' : u'Q',
                            '==' : u'REF'}[m.group(1)]
                        etree.SubElement(polemics_element, u'polemic').text = pol_link
                    if polemic_added:
                        meta_element.append(polemics_element)
                    
                    etree.SubElement(meta_element, u"source", attrib={"url":u"http://dev.twitter.com", "mimetype":u"application/json"}).text = etree.CDATA(unicode(tw.original_json))
                
                output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=True, xml_declaration=True)  
                
                if content_file and content_file.find("http") == 0:
                    
                    project["ldt"] = output_data
                    body = anyjson.serialize(project)
                    logging.debug("write http " + content_file)
                    logging.debug("write http " + repr(body))
                    h = httplib2.Http()
                    resp, content = h.request(content_file, "PUT", headers={'content-type':'application/json'}, body=body)
                    logging.debug("write http " + repr(resp) + " content " + content)
                else:
                    if content_file and os.path.exists(content_file):
                        dest_file_name = content_file 
                    else:
                        dest_file_name = options.filename
            
                    logging.debug("WRITE : " + dest_file_name)
                    output = open(content_file, "w")
                    output.write(output_data)
                    output.flush()
                    output.close()
                
        finally:
            session.close()
    finally:
        conn.close()
