diff -r 41ce1c341abe -r 10a19dd4e1c9 script/rest/enmi_profile.py --- a/script/rest/enmi_profile.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,134 +0,0 @@ -import twython -from sqlite3 import * -import datetime, time -import email.utils -from optparse import OptionParser -import os.path -import os -import sys -import simplejson - - -#options filename rpp page total_pages start_date end_date - - - -def adapt_datetime(ts): - return time.mktime(ts.timetuple()) - -def adapt_geo(geo): - return simplejson.dumps(geo) - -def convert_geo(s): - return simplejson.loads(s) - - -register_adapter(datetime.datetime, adapt_datetime) -register_converter("geo", convert_geo) - -columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user'] -columns_user = [u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'geo_enabled', u'profile_background_image_url', u'screen_name', u'profile_background_tile', u'favourites_count', u'name', u'url', u'created_at', u'time_zone', u'profile_sidebar_border_color', u'following'] - -def processDate(entry): - ts = email.utils.parsedate(entry["created_at"]) - entry["created_at_ts"] = datetime.datetime.fromtimestamp(time.mktime(ts)) - -def processPage(page, cursor, debug): - for entry in page: - if debug: - print "ENTRY : " + repr(entry) - curs.execute("select id from tweet_tweet where id = ?", (entry["id"],)) - res = curs.fetchone() - if res: - continue - - entry_user = entry["user"] - processDate(entry_user) - cursor.execute("insert into tweet_user ("+",".join(entry_user.keys())+") values (:"+",:".join(entry_user.keys())+");", entry_user); - new_id = cursor.lastrowid - processDate(entry) - entry["user"] = new_id - if entry["geo"]: - entry["geo"] = adapt_geo(entry["geo"]) - new_id = cursor.execute("insert into tweet_tweet ("+",".join(entry.keys())+") values (:"+",:".join(entry.keys())+");", entry); - - -if __name__ == "__main__" : - - parser = OptionParser() - parser.add_option("-f", "--file", dest="filename", - help="write tweet to FILE", metavar="FILE", default="enmi2010_twitter_rest.db") - parser.add_option("-r", "--rpp", dest="rpp", - help="Results per page", metavar="RESULT_PER_PAGE", default=200, type='int') - parser.add_option("-p", "--page", dest="page", - help="page result", metavar="PAGE", default=1, type='int') - parser.add_option("-t", "--total-page", dest="total_page", - help="Total page number", metavar="TOTAL_PAGE", default=16, type='int') - parser.add_option("-s", "--screenname", dest="screen_name", - help="Twitter screen name", metavar="SCREEN_NAME") - parser.add_option("-u", "--user", dest="username", - help="Twitter user", metavar="USER", default=None) - parser.add_option("-w", "--password", dest="password", - help="Twitter password", metavar="PASSWORD", default=None) - parser.add_option("-n", "--new", dest="new", action="store_true", - help="new database", default=False) - parser.add_option("-d", "--debug", dest="debug", action="store_true", - help="debug", default=False) - - - - (options, args) = parser.parse_args() - - if options.debug: - print "OPTIONS : " - print repr(options) - - if options.screen_name is None: - print "No Screen name. Exiting" - sys.exit() - - if options.new and os.path.exists(options.filename): - os.remove(options.filename) - - conn = connect(options.filename) - conn.row_factory = Row - curs = conn.cursor() - - curs.execute("create table if not exists tweet_user ("+ ",".join(columns_user) +", created_at_ts integer);") - - curs.execute("create table if not exists tweet_tweet ("+ ",".join(columns_tweet) +", created_at_ts integer);") - curs.execute("create index if not exists id_index on tweet_tweet (id asc);"); - - curs.execute("select count(*) from tweet_tweet;") - res = curs.fetchone() - - old_total = res[0] - - twitter = twython.setup(username=options.username, password=options.password, headers="IRI enmi (python urllib)") - twitter = twython.Twython(twitter_token = "54ThDZhpEjokcMgHJOMnQA", twitter_secret = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA") - - search_results = None - page = options.page-1 - - while (page < options.total_page and ( search_results is None or len(search_results) > 0)): - page += 1 - try: - search_results = twitter.getUserTimeline(screen_name=options.screen_name, count=options.rpp, page=page) - except twython.TwythonError, (e): - print "NAME : "+ options.screen_name + " ERROR : " + repr(e.msg) - break - print "NAME : "+ options.screen_name +" PAGE : " + repr(page) + " tweet: " + repr(len(search_results)) + " (total page : " + unicode(options.total_page) + " : rpp : "+unicode(options.rpp)+")" - processPage(search_results, curs, options.debug) - - conn.commit() - - curs.execute("select count(*) from tweet_tweet;") - res = curs.fetchone() - - total = res[0] - - print "Tweet for " + options.screen_name + " : " + unicode(total - old_total) +", Tweet total : " + repr(total) - - conn.close() - -