import twython
from sqlite3 import *
import datetime, time
import email.utils
from optparse import OptionParser
import os.path
import os
import sys
import simplejson


#options filename rpp page total_pages start_date end_date


 
def adapt_datetime(ts):
    return time.mktime(ts.timetuple())
    
def adapt_geo(geo):
	return simplejson.dumps(geo)
	
def convert_geo(s):
	return simplejson.loads(s)


register_adapter(datetime.datetime, adapt_datetime)
register_converter("geo", convert_geo)

columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user']
columns_user = [u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'geo_enabled', u'profile_background_image_url', u'screen_name', u'profile_background_tile', u'favourites_count', u'name', u'url', u'created_at', u'time_zone', u'profile_sidebar_border_color', u'following']

def processDate(entry):
    ts = email.utils.parsedate(entry["created_at"])
    entry["created_at_ts"] = datetime.datetime.fromtimestamp(time.mktime(ts))

def processPage(page, cursor, debug):
    for entry in page:
        if debug:
            print "ENTRY : " + repr(entry)
        curs.execute("select id from tweet_tweet where id = ?", (entry["id"],))
        res = curs.fetchone()
        if res:
            continue

        entry_user = entry["user"]
        processDate(entry_user)
        cursor.execute("insert into tweet_user ("+",".join(entry_user.keys())+") values (:"+",:".join(entry_user.keys())+");", entry_user);
        new_id = cursor.lastrowid
        processDate(entry)
        entry["user"] = new_id
        if entry["geo"]:
            entry["geo"] = adapt_geo(entry["geo"])
        new_id = cursor.execute("insert into tweet_tweet ("+",".join(entry.keys())+") values (:"+",:".join(entry.keys())+");", entry);


if __name__ == "__main__" :

    parser = OptionParser()
    parser.add_option("-f", "--file", dest="filename",  
                      help="write tweet to FILE", metavar="FILE", default="enmi2010_twitter_rest.db")
    parser.add_option("-r", "--rpp", dest="rpp",
                      help="Results per page", metavar="RESULT_PER_PAGE", default=200, type='int')
    parser.add_option("-p", "--page", dest="page",
                      help="page result", metavar="PAGE", default=1, type='int')
    parser.add_option("-t", "--total-page", dest="total_page",
                      help="Total page number", metavar="TOTAL_PAGE", default=16, type='int')
    parser.add_option("-s", "--screenname", dest="screen_name",
                      help="Twitter screen name", metavar="SCREEN_NAME")
    parser.add_option("-u", "--user", dest="username",
                      help="Twitter user", metavar="USER", default=None)
    parser.add_option("-w", "--password", dest="password",
                      help="Twitter password", metavar="PASSWORD", default=None)
    parser.add_option("-n", "--new", dest="new", action="store_true",
                      help="new database", default=False)
    parser.add_option("-d", "--debug", dest="debug", action="store_true",
                      help="debug", default=False)
    


    (options, args) = parser.parse_args()
    
    if options.debug:
    	print "OPTIONS : "
    	print repr(options)

    if options.screen_name is None:
        print "No Screen name. Exiting"
        sys.exit()
    
    if options.new and os.path.exists(options.filename):
        os.remove(options.filename)
    
    conn = connect(options.filename)
    conn.row_factory = Row
    curs = conn.cursor()

    curs.execute("create table if not exists tweet_user ("+ ",".join(columns_user) +", created_at_ts integer);")

    curs.execute("create table if not exists tweet_tweet ("+ ",".join(columns_tweet) +", created_at_ts integer);")
    curs.execute("create index if not exists id_index on tweet_tweet (id asc);");
    
    curs.execute("select count(*) from tweet_tweet;")
    res = curs.fetchone()
    
    old_total = res[0]

    twitter = twython.setup(username=options.username, password=options.password, headers="IRI enmi (python urllib)")
    twitter = twython.Twython(twitter_token = "54ThDZhpEjokcMgHJOMnQA", twitter_secret = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA")

    search_results = None
    page = options.page-1

    while (page < options.total_page and ( search_results is None  or len(search_results) > 0)):
        page += 1
        try:
            search_results = twitter.getUserTimeline(screen_name=options.screen_name, count=options.rpp, page=page)
        except twython.TwythonError, (e):
            print "NAME : "+ options.screen_name + " ERROR : " + repr(e.msg)
            break
        print "NAME : "+ options.screen_name +" PAGE : " + repr(page) + " tweet: " + repr(len(search_results)) + " (total page : " + unicode(options.total_page) + " : rpp : "+unicode(options.rpp)+")"
        processPage(search_results, curs, options.debug)

    conn.commit() 

    curs.execute("select count(*) from tweet_tweet;")
    res = curs.fetchone()

    total = res[0]

    print "Tweet for " + options.screen_name + " : " + unicode(total - old_total) +", Tweet total : " + repr(total)

    conn.close()