import argparse

import datetime

import inspect

import json

import logging

import os

import queue

import re

import shutil

import signal

import socket

import sys

import threading

import time

import traceback

import urllib

from http.server import BaseHTTPRequestHandler, HTTPServer

from io import StringIO

from multiprocessing import Event, Process

from multiprocessing import Queue as mQueue

import requests_oauthlib

import sqlalchemy.schema

import twitter

from sqlalchemy.exc import OperationalError

from sqlalchemy.orm import scoped_session

import _thread

import iri_tweet.stream

from iri_tweet import models, utils

from iri_tweet.models import ProcessEvent, TweetLog, TweetSource

from iri_tweet.processor import get_processor

# columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user']

columns_tweet = [u'user', u'favorited', u'contributors', u'truncated', u'text', u'created_at', u'retweeted', u'in_reply_to_status_id_str', u'coordinates', u'in_reply_to_user_id_str', u'entities', u'in_reply_to_status_id', u'place', u'in_reply_to_user_id', u'id', u'in_reply_to_screen_name', u'retweet_count', u'geo', u'id_str', u'source']

# columns_user = [u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'geo_enabled', u'profile_background_image_url', u'screen_name', u'profile_background_tile', u'favourites_count', u'name', u'url', u'created_at', u'time_zone', u'profile_sidebar_border_color', u'following']

columns_user = [u'follow_request_sent', u'profile_use_background_image', u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'id_str', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'show_all_inline_media', u'geo_enabled', u'profile_background_image_url', u'name', u'lang', u'following', u'profile_background_tile', u'favourites_count', u'screen_name', u'url', u'created_at', u'contributors_enabled', u'time_zone', u'profile_sidebar_border_color', u'is_translator', u'listed_count']

# just put it in a sqlite3 tqble

DEFAULT_TIMEOUT = 3

class Requesthandler(BaseHTTPRequestHandler):

    def do_GET(self):

        self.send_response(200)

        self.end_headers()

    def log_message(self, format, *args):        # @ReservedAssignment

        pass

def set_logging(options):

    loggers = []

    loggers.append(utils.set_logging(options, logging.getLogger('iri.tweet')))

    loggers.append(utils.set_logging(options, logging.getLogger('multiprocessing')))

    if options.debug >= 2:

        loggers.append(utils.set_logging(options, logging.getLogger('sqlalchemy.engine')))

    # utils.set_logging(options, logging.getLogger('sqlalchemy.dialects'))

    # utils.set_logging(options, logging.getLogger('sqlalchemy.pool'))

    # utils.set_logging(options, logging.getLogger('sqlalchemy.orm'))

    return loggers

def set_logging_process(options, queue):

    qlogger = utils.set_logging(options, logging.getLogger('iri.tweet.p'), queue)

    qlogger.propagate = 0

    return qlogger

def get_auth(consumer_key, consumer_secret, token_key, token_secret):

    return requests_oauthlib.OAuth1(client_key=consumer_key, client_secret=consumer_secret, resource_owner_key=token_key, resource_owner_secret=token_secret, signature_type='auth_header')

def add_process_event(event_type, args, session_maker):

    session = session_maker()

    try:

        evt = ProcessEvent(args=None if args is None else json.dumps(args), type=event_type)

        session.add(evt)

        session.commit()

    finally:

        session.close()

class BaseProcess(Process):

    def __init__(self, session_maker, queue, options, twitter_auth, stop_event, logger_queue, parent_pid):

        self.parent_pid = parent_pid

        self.session_maker = session_maker

        self.queue = queue

        self.options = options

        self.logger_queue = logger_queue

        self.stop_event = stop_event

        self.twitter_auth = twitter_auth

        super(BaseProcess, self).__init__()

    #

    # from http://stackoverflow.com/questions/2542610/python-daemon-doesnt-kill-its-kids

    #

    def parent_is_alive(self):

        try:

            # try to call Parent

            os.kill(self.parent_pid, 0)

        except OSError:

            # *beeep* oh no! The phone's disconnected!

            return False

        else:

            # *ring* Hi mom!

            return True

    def __get_process_event_args(self):

        return {'name':self.name, 'pid':self.pid, 'parent_pid':self.parent_pid, 'options':self.options.__dict__}

    def run(self):

        try:

            add_process_event("start_worker", self.__get_process_event_args(), self.session_maker)

            self.do_run()

        finally:

            add_process_event("stop_worker", self.__get_process_event_args(), self.session_maker)

    def do_run(self):

        raise NotImplementedError()

class SourceProcess(BaseProcess):

    def __init__(self, session_maker, queue, options, twitter_auth, stop_event, logger_queue, parent_pid):

        self.track = options.track

        self.timeout = options.timeout

        self.stream = None

        super(SourceProcess, self).__init__(session_maker, queue, options, twitter_auth, stop_event, logger_queue, parent_pid)

    def __source_stream_iter(self):

        self.logger.debug("SourceProcess : run ")

        self.logger.debug("SourceProcess : get_auth auth with option %s and token %s " %(self.options, self.twitter_auth))

        self.auth = get_auth(self.twitter_auth.consumer_key, self.twitter_auth.consumer_secret, self.twitter_auth.token, self.twitter_auth.token_secret)

        self.logger.debug("SourceProcess : auth set ")

        track_list = self.track  # or raw_input('Keywords to track (comma seperated): ').strip()

        self.logger.debug("SourceProcess : track list " + track_list)

        track_list = [k.strip() for k in track_list.split(',')]

        self.logger.debug("SourceProcess : before connecting to stream %s, url : %s, auth : %s" % (repr(track_list), self.options.url, repr(self.auth)))

        self.stream = iri_tweet.stream.FilterStream(self.auth, track=track_list, raw=True, url=self.options.url, timeout=self.timeout, logger=self.logger)

        self.logger.debug("SourceProcess : after connecting to stream")

        self.stream.muststop = lambda: self.stop_event.is_set()

        stream_wrapper = iri_tweet.stream.SafeStreamWrapper(self.stream, logger=self.logger)

        session = self.session_maker()

        #import pydevd

        #pydevd.settrace(suspend=False)

        try:

            for tweet in stream_wrapper:

                if not self.parent_is_alive():

                    self.stop_event.set()

                    sys.exit()

                self.logger.debug("SourceProcess : tweet " + repr(tweet))

                source = TweetSource(original_json=tweet)

                self.logger.debug("SourceProcess : source created")

                add_retries = 0

                while add_retries < 10:

                    try:

                        add_retries += 1

                        session.add(source)

                        session.flush()

                        break

                    except OperationalError as e:

                        session.rollback()

                        self.logger.debug("SourceProcess : Operational Error %s nb %d" % (repr(e), add_retries))

                        if add_retries == 10:

                            raise

                source_id = source.id

                self.logger.debug("SourceProcess : before queue + source id " + repr(source_id))

                self.logger.info("SourceProcess : Tweet count: %d - current rate : %.2f - running : %s" % (self.stream.count, self.stream.rate, int(time.time() - self.stream.starttime)))

                session.commit()

                self.queue.put((source_id, tweet), False)

        except Exception as e:

            self.logger.error("SourceProcess : Error when processing tweet " + repr(e))

            raise

        finally:

            session.rollback()

            session.close()

            self.stream.close()

            self.stream = None

            if not self.stop_event.is_set():

                self.stop_event.set()

    def do_run(self):

        self.logger = set_logging_process(self.options, self.logger_queue)

        source_stream_iter_thread = threading.Thread(target=self.__source_stream_iter , name="SourceStreamIterThread")

        source_stream_iter_thread.start()

        try:

            while not self.stop_event.is_set():

                self.logger.debug("SourceProcess : In while after start")

                self.stop_event.wait(DEFAULT_TIMEOUT)

        except KeyboardInterrupt:

            self.stop_event.set()

            pass

        if self.stop_event.is_set() and self.stream:

            self.stream.close()

        elif not self.stop_event.is_set() and not source_stream_iter_thread.is_alive:

            self.stop_event.set()

        self.queue.cancel_join_thread()

        self.logger_queue.cancel_join_thread()

        self.logger.info("SourceProcess : join")

        source_stream_iter_thread.join(30)

def process_tweet(tweet, source_id, session, twitter_auth, twitter_query_user, logger):

    try:

        if not tweet.strip():

            return

        tweet_obj = json.loads(tweet)

        processor_klass = get_processor(tweet_obj)

        if not processor_klass:

            tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['NOT_TWEET'])

            session.add(tweet_log)

            return

        processor = processor_klass(json_dict=tweet_obj,

                                    json_txt=tweet,

                                    source_id=source_id,

                                    session=session,

                                    twitter_auth=twitter_auth,

                                    user_query_twitter=twitter_query_user,

                                    logger=logger)

        logger.info(processor.log_info())

        logger.debug(u"Process_tweet :" + repr(tweet))

        processor.process()

    except ValueError as e:

        message = u"Value Error %s processing tweet %s" % (repr(e), tweet)

        output = StringIO()

        try:

            traceback.print_exc(file=output)

            error_stack = output.getvalue()

        finally:

            output.close()

        tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['NOT_TWEET'], error=message, error_stack=error_stack)

        session.add(tweet_log)

        session.commit()

    except Exception as e:

        message = u"Error %s processing tweet %s" % (repr(e), tweet)

        logger.exception(message)

        output = StringIO()

        try:

            traceback.print_exc(file=output)

            error_stack = output.getvalue()

        finally:

            output.close()

        session.rollback()

        tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['ERROR'], error=message, error_stack=error_stack)

        session.add(tweet_log)

        session.commit()

class TweetProcess(BaseProcess):

    def __init__(self, session_maker, queue, options, twitter_auth, stop_event, logger_queue, parent_pid):

        super(TweetProcess, self).__init__(session_maker, queue, options, twitter_auth, stop_event, logger_queue, parent_pid)

        self.twitter_query_user = options.twitter_query_user

    def do_run(self):

        self.logger = set_logging_process(self.options, self.logger_queue)

        session = self.session_maker()

        try:

            while not self.stop_event.is_set() and self.parent_is_alive():

                try:

                    source_id, tweet_txt = self.queue.get(True, 3)

                    self.logger.debug("Processing source id " + repr(source_id))

                except Exception as e:

                    self.logger.debug('Process tweet exception in loop : ' + repr(e))

                    continue

                process_tweet(tweet_txt, source_id, session, self.twitter_auth, self.twitter_query_user, self.logger)

                session.commit()

        except KeyboardInterrupt:

            self.stop_event.set()

        finally:

            session.rollback()

            session.close()

def get_sessionmaker(conn_str):

    engine, metadata, Session = models.setup_database(conn_str, echo=False, create_all=False, autocommit=False)

    Session = scoped_session(Session)

    return Session, engine, metadata

def process_leftovers(session, twitter_auth, twitter_query_user, ask_process_leftovers, logger):

    sources = session.query(TweetSource).outerjoin(TweetLog).filter(TweetLog.id == None)

    sources_count = sources.count()

    if sources_count > 10 and ask_process_leftovers:

        resp = input("Do you want to process leftovers (Y/n) ? (%d tweet to process)" % sources_count)

        if resp and resp.strip().lower() == "n":

            return

    logger.info("Process leftovers, %d tweets to process" % (sources_count))

    for src in sources:

        tweet_txt = src.original_json

        process_tweet(tweet_txt, src.id, session, twitter_auth, twitter_query_user, logger)

        session.commit()

def process_log(logger_queues, stop_event):

    while not stop_event.is_set():

        for lqueue in logger_queues:

            try:

                record = lqueue.get_nowait()

                logging.getLogger(record.name).handle(record)

            except queue.Empty:

                continue

            except IOError:

                continue

        time.sleep(0.1)

def get_options():

    usage = "usage: %(prog)s [options]"

    parser = argparse.ArgumentParser(usage=usage)

    parser.add_argument("-f", "--file", dest="conn_str",

                        help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR", default="enmi2010_twitter.db")

    parser.add_argument("-T", "--track", dest="track",

                        help="Twitter track", metavar="TRACK")

    parser.add_argument("-k", "--key", dest="consumer_key",

                        help="Twitter consumer key", metavar="CONSUMER_KEY", required=True)

    parser.add_argument("-s", "--secret", dest="consumer_secret",

                        help="Twitter consumer secret", metavar="CONSUMER_SECRET", required=True)

    parser.add_argument("-n", "--new", dest="new", action="store_true",