#!/usr/bin/env python

# coding=utf-8

import argparse

import bisect

import datetime

import json

import os.path

import re

import sys

import uuid  # @UnresolvedImport

import requests

from lxml import etree

from sqlalchemy import BigInteger, Column, Table, bindparam, event

from sqlalchemy.sql import func, select

from dateutil.parser import parse as parse_date_raw

from dateutil.tz import tzutc

from iri_tweet.models import Tweet, User, setup_database

from iri_tweet.utils import (get_filter_query, get_logger, set_logging,

                             set_logging_options)

#class TweetExclude(object):

#    def __init__(self, id):

#        self.id = id

#

#    def __repr__(self):

#        return "<TweetExclude(id=%d)>" % (self.id)

LDT_CONTENT_REST_API_PATH = "api/ldt/1.0/contents/"

LDT_PROJECT_REST_API_PATH = "api/ldt/1.0/projects/"

def parse_date(datestr):

    res = parse_date_raw(datestr)

    if res.tzinfo is None:

        res = res.replace(tzinfo=tzutc())

    return res

def re_fn(expr, item):

    reg = re.compile(expr, re.I)

    res = reg.search(item)

    if res:

        get_logger().debug("re_fn : " + repr(expr) + "~" + repr(item)) #@UndefinedVariable

    return res is not None

def parse_polemics_1(tw, extended_mode):

    """

    parse polemics in text and return a list of polemic code. None if not polemic found

    """

    polemics = {}

    for m in re.finditer(r"(\+\+|\-\-|\?\?|\=\=)",tw.text):

        pol_link = {

            '++' : 'OK',

            '--' : 'KO',

            '??' : 'Q',

            '==' : 'REF'}[m.group(1)]

        polemics[pol_link] = pol_link

    if extended_mode:

        if "?" in tw.text:

            polemics["Q"] = "Q"

        for entity in tw.entity_list:

            if entity.type == "entity_url":

                polemics["REF"] = "REF"

    if len(polemics) > 0:

        return polemics.keys()

    else:

        return None

def parse_polemics_2(tw, extended_mode):

    """

    parse polemics in text and return a list of polemic code. None if not polemic found

    """

    polemics = {}

    for m in re.finditer(r"(\+\+|\!\!|\?\?|\=\=)",tw.text):

        pol_link = {

            '++' : 'OK',

            '!!' : 'KO',

            '??' : 'Q',

            '==' : 'REF'}[m.group(1)]

        polemics[pol_link] = pol_link

    if extended_mode:

        if "?" in tw.text:

            polemics["Q"] = "Q"

        for entity in tw.entity_list:

            if entity.type == "entity_url":

                polemics["REF"] = "REF"

    if len(polemics) > 0:

        return polemics.keys()

    else:

        return None

def parse_polemics_3(tw, extended_mode):

    """

    parse polemics in text and return a list of polemic code. None if not polemic found

    """

    polemics = {}

    for m in re.finditer(r"(\+\+|\?\?|\*\*|\=\=)",tw.text):

        pol_link = {

            '++' : 'OK',

            '??' : 'KO',

            '**' : 'REF',

            '==' : 'Q'}[m.group(1)]

        polemics[pol_link] = pol_link

    if extended_mode:

        for entity in tw.entity_list:

            if entity.type == "entity_url":

                polemics["REF"] = "REF"

    if len(polemics) > 0:

        return polemics.keys()

    else:

        return None

protocol_version_map = {

    "1" : parse_polemics_1,

    "2" : parse_polemics_2,

    "3" : parse_polemics_3

}

def get_options():

    parser = argparse.ArgumentParser(description="All date should be given using iso8601 format. If no timezone is used, the date is considered as UTC")

    parser.add_argument("-f", "--file", dest="filename",

                      help="write export to file", metavar="FILE", default="project.ldt")

    parser.add_argument("-d", "--database", dest="database",

                      help="Input database", metavar="DATABASE")

    parser.add_argument("-a", "--annotation-protocol", dest="protocol_version",

                      help="annotation protocol version", metavar="PROTOCOL_VERSION",

                      default="2")

    parser.add_argument("-s", "--start-date", dest="start_date",

                      help="start date", metavar="START_DATE", default=None)

    parser.add_argument("-e", "--end-date", dest="end_date",

                      help="end date", metavar="END_DATE", default=None)

    parser.add_argument("-I", "--content-file", dest="content_file",

                      help="Content file", metavar="CONTENT_FILE")

    parser.add_argument("-c", "--content", dest="content",

                      help="Content url", metavar="CONTENT")

    parser.add_argument("-V", "--video-url", dest="video",

                      help="video url", metavar="VIDEO")

    parser.add_argument("-i", "--content-id", dest="content_id",

                      help="Content id", metavar="CONTENT_ID")

    parser.add_argument("-x", "--exclude", dest="exclude",

                      help="file containing the id to exclude", metavar="EXCLUDE")

    parser.add_argument("-C", "--color", dest="color",

                      help="Color code", metavar="COLOR", default="16763904")

    parser.add_argument("-H", "--hashtag", dest="hashtag",

                      help="Hashtag", metavar="HASHTAG", default=[], action="append")

    parser.add_argument("-D", "--duration", dest="duration", type=int,

                      help="Duration", metavar="DURATION", default=None)

    parser.add_argument("-n", "--name", dest="name",

                      help="Cutting name", metavar="NAME", default="Tweets")

    parser.add_argument("-R", "--replace", dest="replace", action="store_true",

                      help="Replace tweet ensemble", default=False)

    parser.add_argument("-m", "--merge", dest="merge", action="store_true",

                      help="merge tweet ensemble, choose the first ensemble", default=False)

    parser.add_argument("-L", "--list-conf", dest="listconf",

                      help="list of file to process", metavar="LIST_CONF", default=None)

    parser.add_argument("-E", "--extended", dest="extended_mode", action="store_true",

                      help="Trigger polemic extended mode", default=False)

    parser.add_argument("-b", "--base-url", dest="base_url",

                      help="base URL of the platform", metavar="BASE_URL", default="http://ldt.iri.centrepompidou.fr/ldtplatform/")

    parser.add_argument("-p", "--project", dest="project_id",

                      help="Project id", metavar="PROJECT_ID", default=None)

    parser.add_argument("-P", "--post-param", dest="post_param",

                      help="Post param", metavar="POST_PARAM", default=None)

    parser.add_argument("--user-whitelist", dest="user_whitelist", action="store",

                      help="A list of user screen name", metavar="USER_WHITELIST",default=None)

    parser.add_argument("--cut", dest="cuts", action="append",

                      help="A cut with the forma <ts in ms>::<duration>", metavar="CUT", default=[])

    set_logging_options(parser)

    return (parser.parse_args(), parser)

def find_delta(deltas, ts):

    i = bisect.bisect_right(deltas, (ts+1,0))

    if i:

        return deltas[i-1]

    return (0,0)

def parse_duration(s):

    try:

        return int(s)

    except ValueError:

        parts = s.split(":")

        if len(parts) < 2:

            raise ValueError("Bad duration format")

        time_params = {

            'hours': int(parts[0]),

            'minutes': int(parts[1]),

            'seconds': int(parts[2]) if len(parts)>2 else 0

        }

        return int(round(datetime.timedelta(**time_params).total_seconds()*1000))

if __name__ == "__main__" :

    (options, parser) = get_options()

    set_logging(options)

    get_logger().debug("OPTIONS : " + repr(options)) #@UndefinedVariable

    deltas = [(0,0)]

    total_delta = 0

    if options.cuts:

        cuts_raw = sorted([tuple([parse_duration(s) for s in c.split("::")]) for c in options.cuts])

        for c, d in cuts_raw:

            deltas.append((c+total_delta, -1))

            total_delta += d

            deltas.append((c+total_delta, total_delta))

    if len(sys.argv) == 1 or options.database is None:

        parser.print_help()

        sys.exit(1)

    conn_str = options.database.strip()

    if not re.match(r"^\w+://.+", conn_str):

        conn_str = 'sqlite:///' + conn_str

    engine, metadata, Session = setup_database(conn_str, echo=((options.verbose-options.quiet)>0), create_all = False)

    conn = None

    try :

        conn = engine.connect()

        @event.listens_for(conn, "begin")

        def do_begin(conn):

            conn.connection.create_function('regexp', 2, re_fn)

        session = None

        try :

            session = Session(bind=conn)

            tweet_exclude_table = Table("tweet_exclude", metadata, Column('id', BigInteger, primary_key=True), prefixes=['TEMPORARY'])

            #mapper(TweetExclude, tweet_exclude_table)

            metadata.create_all(bind=conn, tables=[tweet_exclude_table])

            if options.exclude and os.path.exists(options.exclude):

                with open(options.exclude, 'r+') as f:

                    tei = tweet_exclude_table.insert() # pylint: disable=E1120

                    ex_regexp = re.compile(r"(?P<field>\w+)(?P<op>[~=])(?P<value>.+)", re.I)

                    for line in f:

                        res = ex_regexp.match(line.strip())

                        if res:

                            if res.group('field') == "id":

                                conn.execute(tei.values(id=res.group('value')))

                            else:

                                exclude_query = session.query(Tweet)

                                filter_obj = Tweet

                                filter_field = res.group('field')

                                if filter_field.startswith("user__"):

                                    exclude_query = exclude_query.outerjoin(User, Tweet.user_id==User.id)

                                    filter_obj = User

                                    filter_field = filter_field[len("user__"):]

                                if res.group('op') == "=":

                                    exclude_query = exclude_query.filter(getattr(filter_obj, filter_field) == res.group('value'))

                                else:

                                    exclude_query = exclude_query.filter(getattr(filter_obj, filter_field).op('regexp')(res.group('value')))

                                test_query = select([func.count()]).where(tweet_exclude_table.c.id==bindparam('t_id'))

                                for t in exclude_query.all():

                                    get_logger().debug("t : " + repr(t))

                                    if conn.execute(test_query, t_id=t.id).fetchone()[0] == 0:

                                        conn.execute(tei.values(id=t.id))

            user_whitelist_file = options.user_whitelist

            user_whitelist = None

            if options.listconf:

                parameters = []

                confdoc = etree.parse(options.listconf)

                for node in confdoc.xpath("/twitter_export/file"):

                    params = {}

                    for snode in node:

                        if snode.tag == "path":

                            params['content_file'] = snode.text

                            params['content_file_write'] = snode.text

                        elif snode.tag == "project_id":

                            params['content_file'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"

                            params['content_file_write'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"

                            params['project_id'] = snode.text

                        elif snode.tag == "start_date":

                            params['start_date'] = snode.text

                        elif snode.tag == "end_date":

                            params['end_date'] = snode.text

                        elif snode.tag == "duration":

                            params['duration'] = int(snode.text)

                        elif snode.tag == "hashtags":

                            params['hashtags'] = [snode.text]

                    if options.hashtag or 'hashtags' not in params :

                        params['hashtags'] = options.hashtag

                    parameters.append(params)

            else:

                if options.project_id:

                    content_file = options.base_url + LDT_PROJECT_REST_API_PATH + options.project_id + "/?format=json"

                else:

                    content_file = options.content_file

                parameters = [{

                    'start_date': options.start_date,

                    'end_date' : options.end_date,

                    'duration' : options.duration,

                    'content_file' : content_file,

                    'content_file_write' : content_file,

                    'hashtags' : options.hashtag,

                    'project_id' : options.project_id

                }]

            post_param = {}

            if options.post_param:

                post_param = json.loads(options.post_param)

            for params in parameters:

                get_logger().debug("PARAMETERS " + repr(params)) #@UndefinedVariable

                start_date_str = params.get("start_date",None)

                end_date_str = params.get("end_date", None)

                duration = params.get("duration", None)

                content_file = params.get("content_file", None)

                content_file_write = params.get("content_file_write", None)

                hashtags = list(set(params.get('hashtags', [])))

                if user_whitelist_file:

                    with open(user_whitelist_file, 'r+') as f:

                        user_whitelist = list(set([s.strip() for s in f]))

                start_date = None

                if start_date_str:

                    start_date = parse_date(start_date_str)

                root = None

                ensemble_parent = None

                #to do : analyse situation ldt or iri ? filename set or not ?

                if content_file and content_file.find("http") == 0:

                    get_logger().debug("url : " + content_file) #@UndefinedVariable