#!/usr/bin/env python

# coding=utf-8

from lxml import etree

from iri_tweet.models import setup_database, Tweet, User

from sqlalchemy import Table, Column, BigInteger, event, bindparam

from sqlalchemy.sql import select, func

    get_logger)

import argparse

import anyjson

import datetime

import requests

import os.path

import re

import sys

import time

import uuid #@UnresolvedImport

from dateutil.parser import parse as parse_date

#class TweetExclude(object):

#    def __init__(self, id):

#        self.id = id

#    def __repr__(self):

#        return "<TweetExclude(id=%d)>" % (self.id)

LDT_CONTENT_REST_API_PATH = "api/ldt/1.0/contents/"

LDT_PROJECT_REST_API_PATH = "api/ldt/1.0/projects/"

    reg = re.compile(expr, re.I)

    res = reg.search(item)

    if res:

        get_logger().debug("re_fn : " + repr(expr) + "~" + repr(item)) #@UndefinedVariable

def parse_polemics(tw, extended_mode):

    """

    parse polemics in text and return a list of polemic code. None if not polemic found

    """

    for m in re.finditer("(\+\+|\-\-|\?\?|\=\=)",tw.text):

        pol_link = {

            '++' : u'OK',

            '--' : u'KO',

            '??' : u'Q',

            '==' : u'REF'}[m.group(1)]

        polemics[pol_link] = pol_link

    if extended_mode:

        if "?" in tw.text:

            polemics["Q"] = "Q"

        for entity in tw.entity_list:

            if entity.type == "entity_url":

    if len(polemics) > 0:

        return polemics.keys()

    else:

        return None

def get_options():

    usage = "usage: %(prog)s [options]"

    parser = argparse.ArgumentParser(usage)

    parser.add_argument("-f", "--file", dest="filename",

                      help="write export to file", metavar="FILE", default="project.ldt")

    parser.add_argument("-d", "--database", dest="database",

                      help="Input database", metavar="DATABASE")

    parser.add_argument("-s", "--start-date", dest="start_date",

                      help="start date", metavar="START_DATE", default=None)

    parser.add_argument("-e", "--end-date", dest="end_date",

                      help="end date", metavar="END_DATE", default=None)

    parser.add_argument("-I", "--content-file", dest="content_file",

                      help="Content file", metavar="CONTENT_FILE")

    parser.add_argument("-c", "--content", dest="content",

                      help="Content url", metavar="CONTENT")

    parser.add_argument("-V", "--video-url", dest="video",

                      help="video url", metavar="VIDEO")

    parser.add_argument("-i", "--content-id", dest="content_id",

                      help="Content id", metavar="CONTENT_ID")

    parser.add_argument("-x", "--exclude", dest="exclude",

                      help="file containing the id to exclude", metavar="EXCLUDE")

    parser.add_argument("-C", "--color", dest="color",

                      help="Color code", metavar="COLOR", default="16763904")

    parser.add_argument("-H", "--hashtag", dest="hashtag",

    parser.add_argument("-D", "--duration", dest="duration", type=int,

                      help="Duration", metavar="DURATION", default=None)

    parser.add_argument("-n", "--name", dest="name",

                      help="Cutting name", metavar="NAME", default=u"Tweets")

    parser.add_argument("-R", "--replace", dest="replace", action="store_true",

                      help="Replace tweet ensemble", default=False)

    parser.add_argument("-m", "--merge", dest="merge", action="store_true",

                      help="merge tweet ensemble, choose the first ensemble", default=False)

    parser.add_argument("-L", "--list-conf", dest="listconf",

                      help="list of file to process", metavar="LIST_CONF", default=None)

    parser.add_argument("-E", "--extended", dest="extended_mode", action="store_true",

                      help="Trigger polemic extended mode", default=False)

    parser.add_argument("-b", "--base-url", dest="base_url",

                      help="base URL of the platform", metavar="BASE_URL", default="http://ldt.iri.centrepompidou.fr/ldtplatform/")

                      help="Project id", metavar="PROJECT_ID", default=None)

    parser.add_argument("--user-whitelist", dest="user_whitelist", action="store",

                      help="A list of user screen name", metavar="USER_WHITELIST",default=None)

    set_logging_options(parser)

    return (parser.parse_args(), parser)

if __name__ == "__main__" :

    (options, parser) = get_options()

    set_logging(options)

    get_logger().debug("OPTIONS : " + repr(options)) #@UndefinedVariable

    if len(sys.argv) == 1 or options.database is None:

        parser.print_help()

        sys.exit(1)

    conn_str = options.database.strip()

    if not re.match("^\w+://.+", conn_str):

        conn_str = 'sqlite:///' + conn_str

    conn = None

    try :

        conn = engine.connect()

        @event.listens_for(conn, "begin")

        def do_begin(conn):

        session = None

        try :

            tweet_exclude_table = Table("tweet_exclude", metadata, Column('id', BigInteger, primary_key=True), prefixes=['TEMPORARY'])

            #mapper(TweetExclude, tweet_exclude_table)

            metadata.create_all(bind=conn, tables=[tweet_exclude_table])

            if options.exclude and os.path.exists(options.exclude):

                with open(options.exclude, 'r+') as f:

                    tei = tweet_exclude_table.insert()

                    ex_regexp = re.compile("(?P<field>\w+)(?P<op>[~=])(?P<value>.+)", re.I)

                    for line in f:

                        res = ex_regexp.match(line.strip())

                        if res:

                                conn.execute(tei.values(id=res.group('value')))

                            else:

                                exclude_query = session.query(Tweet)

                                filter_obj = Tweet

                                filter_field = res.group('field')

                                if filter_field.startswith("user__"):

                                    exclude_query = exclude_query.outerjoin(User, Tweet.user_id==User.id)

                                    filter_obj = User

                                if res.group('op') == "=":

                                    exclude_query = exclude_query.filter(getattr(filter_obj, filter_field) == res.group('value'))

                                else:

                                    exclude_query = exclude_query.filter(getattr(filter_obj, filter_field).op('regexp')(res.group('value')))

                                test_query = select([func.count()]).where(tweet_exclude_table.c.id==bindparam('t_id'))

                                for t in exclude_query.all():

                                    get_logger().debug("t : " + repr(t))

                                    if conn.execute(test_query, t_id=t.id).fetchone()[0] == 0:

                                        conn.execute(tei.values(id=t.id))

            user_whitelist_file = options.user_whitelist

            user_whitelist = None

            if options.listconf:

                parameters = []

                confdoc = etree.parse(options.listconf)

                for node in confdoc.xpath("/twitter_export/file"):

                    params = {}

                    for snode in node:

                        if snode.tag == "path":

                            params['content_file'] = snode.text

                            params['content_file_write'] = snode.text

                        elif snode.tag == "project_id":

                            params['content_file'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"

                            params['content_file_write'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"

                            params['project_id'] = snode.text

                        elif snode.tag == "start_date":

                            params['start_date'] = snode.text

                        elif snode.tag == "end_date":

                            params['end_date'] = snode.text

                        elif snode.tag == "duration":

                            params['duration'] = int(snode.text)

                        elif snode.tag == "hashtags":

                            params['hashtags'] = [snode.text]

                    if options.hashtag or 'hashtags' not in params :

                        params['hashtags'] = options.hashtag

                    parameters.append(params)

            else:

                if options.project_id:

                    content_file = options.base_url + LDT_PROJECT_REST_API_PATH + options.project_id + "/?format=json"

                else:

                parameters = [{

                    'start_date': options.start_date,

                    'end_date' : options.end_date,

                    'duration' : options.duration,

                    'content_file' : content_file,

                    'content_file_write' : content_file,

                    'hashtags' : options.hashtag,

                }]

            post_param = {}

            if options.post_param:

                post_param = anyjson.loads(options.post_param)

            for params in parameters:

                get_logger().debug("PARAMETERS " + repr(params)) #@UndefinedVariable

                start_date_str = params.get("start_date",None)

                end_date_str = params.get("end_date", None)

                duration = params.get("duration", None)

                content_file = params.get("content_file", None)

                content_file_write = params.get("content_file_write", None)

                hashtags = params.get('hashtags', [])

                if user_whitelist_file:

                    with open(user_whitelist_file, 'r+') as f:

                        user_whitelist = list(set([s.strip() for s in f]))

                start_date = None

                ts = None

                if start_date_str:

                    ts = time.mktime(start_date.timetuple())

                root = None

                ensemble_parent = None

                #to do : analyse situation ldt or iri ? filename set or not ?

                if content_file and content_file.find("http") == 0:

                    get_logger().debug("url : " + content_file) #@UndefinedVariable

                    project = r.json()

                    text_match = re.match(r"\<\?\s*xml.*?\?\>(.*)", project['ldt'], re.I|re.S)

                    root = etree.fromstring(text_match.group(1) if text_match else project['ldt'])

                elif content_file and os.path.exists(content_file):

                    doc = etree.parse(content_file)

                    root = doc.getroot()

                if root is None:

                    root = etree.Element(u"iri")

                    project = etree.SubElement(root, u"project", {u"abstract":u"Polemics Tweets",u"title":u"Polemic Tweets", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())})

                    medias = etree.SubElement(root, u"medias")

                    media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(options.content), u"video":unicode(options.video), u"id":unicode(options.content_id), u"extra":u""})

                    content = etree.SubElement(annotations, u"content", {u"id":unicode(options.content_id)})

                    ensemble_parent = content

                    content_id = options.content_id

                if ensemble_parent is None:

                    file_type = None

                    for node in root:

                        if node.tag == "project":

                            file_type = "ldt"

                            break

                        elif node.tag == "head":

                            file_type = "iri"

                            break

                    if file_type == "ldt":

                        media_nodes = root.xpath("//media")

                        if len(media_nodes) > 0:

                            media = media_nodes[0]

                        annotations_node = root.find(u"annotations")

                        if annotations_node is None:

                            annotations_node = etree.SubElement(root, u"annotations")

                        content_node = annotations_node.find(u"content")

                        if content_node is None:

                            content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id"))

                        ensemble_parent = content_node

                        content_id = content_node.get(u"id")

                        display_nodes = root.xpath("//displays/display/content[@id='%s']" % content_id)

                        if len(display_nodes) == 0: