# HG changeset patch # User Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com> # Date 1368185680 -7200 # Node ID 10a19dd4e1c9061ae0718f0a90a59f87f567a80a # Parent 41ce1c341abeb087482e78b81f92562032c76c5f# Parent 8628c590f6087f86fce560065d3fb03260c8bb81 Merge with 357990a5f8cc85fd4181b2d8d0556b6c05fcf287 diff -r 41ce1c341abe -r 10a19dd4e1c9 .pydevproject --- a/.pydevproject Tue May 07 18:28:26 2013 +0200 +++ b/.pydevproject Fri May 10 13:34:40 2013 +0200 @@ -1,9 +1,11 @@ -Default -python 2.6 +python_tl +python 2.7 -/tweet_live/script/lib -/tweet_live/script/lib/tweetstream +/tweet_live/script/lib/iri_tweet +/tweet_live/script/stream +/tweet_live/script/rest +/tweet_live/script/utils diff -r 41ce1c341abe -r 10a19dd4e1c9 sbin/sync/sync_live --- a/sbin/sync/sync_live Tue May 07 18:28:26 2013 +0200 +++ b/sbin/sync/sync_live Fri May 10 13:34:40 2013 +0200 @@ -9,7 +9,8 @@ #text2unix ~/tmp/tweet_live_V$1 if [ -d ~/tmp/tweet_live_V$1 ]; then - cat <' % self.reason diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/iri_tweet/iri_tweet/models.py --- a/script/lib/iri_tweet/iri_tweet/models.py Tue May 07 18:28:26 2013 +0200 +++ b/script/lib/iri_tweet/iri_tweet/models.py Fri May 10 13:34:40 2013 +0200 @@ -1,5 +1,6 @@ from sqlalchemy import (Boolean, Column, Enum, BigInteger, Integer, String, - ForeignKey, DateTime, create_engine) + ForeignKey, DateTime, create_engine, event) +from sqlalchemy.engine import Engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship, sessionmaker import anyjson @@ -11,12 +12,10 @@ Base = declarative_base() APPLICATION_NAME = "IRI_TWITTER" -CONSUMER_KEY = "54ThDZhpEjokcMgHJOMnQA" -CONSUMER_SECRET = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA" +#CONSUMER_KEY = "54ThDZhpEjokcMgHJOMnQA" +#CONSUMER_SECRET = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA" ACCESS_TOKEN_KEY = None ACCESS_TOKEN_SECRET = None -#ACCESS_TOKEN_KEY= "47312923-LiNTtz0I18YXMVIrFeTuhmH7bOvYsK6p3Ln2Dc" -#ACCESS_TOKEN_SECRET = "r3LoXVcjImNAElUpWqTu2SG2xCdWFHkva7xeQoncA" def adapt_date(date_str): ts = email.utils.parsedate_tz(date_str) #@UndefinedVariable @@ -32,7 +31,7 @@ def __init__(cls, name, bases, ns): #@NoSelf def init(self, **kwargs): - for key, value in kwargs.items(): + for key, value in kwargs.iteritems(): if hasattr(self, key): setattr(self, key, value) super(cls, self).__init__() @@ -82,6 +81,14 @@ 'OK' : 1, 'ERROR' : 2, 'NOT_TWEET': 3, + 'DELETE': 4, + 'SCRUB_GEO': 5, + 'LIMIT': 6, + 'STATUS_WITHHELD': 7, + 'USER_WITHHELD': 8, + 'DISCONNECT': 9, + 'STALL_WARNING': 10, + 'DELETE_PENDING': 4 } __metaclass__ = TweetMeta @@ -90,6 +97,7 @@ ts = Column(DateTime, default=datetime.datetime.utcnow, index=True) tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id')) tweet_source = relationship("TweetSource", backref="logs") + status_id = Column(BigInteger, index=True, nullable=True, default=None) status = Column(Integer) error = Column(String) error_stack = Column(String) @@ -121,7 +129,7 @@ user = relationship("User", backref="tweets") tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id')) tweet_source = relationship("TweetSource", backref="tweet") - entity_list = relationship(Entity, backref='tweet') + entity_list = relationship(Entity, backref='tweet', cascade="all, delete-orphan") received_at = Column(DateTime, default=datetime.datetime.utcnow, index=True) @@ -266,9 +274,17 @@ session_argname = [ 'autoflush','binds', "class_", "_enable_transaction_accounting","expire_on_commit", "extension", "query_cls", "twophase", "weak_identity_map", "autocommit"] - kwargs_ce = dict((k, v) for k,v in kwargs.items() if (k not in session_argname and k != "create_all")) + kwargs_ce = dict((k, v) for k,v in kwargs.iteritems() if (k not in session_argname and k != "create_all")) engine = create_engine(*args, **kwargs_ce) + + if engine.name == "sqlite": + @event.listens_for(Engine, "connect") + def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + metadata = Base.metadata kwargs_sm = {'bind': engine} diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/iri_tweet/iri_tweet/processor.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/lib/iri_tweet/iri_tweet/processor.py Fri May 10 13:34:40 2013 +0200 @@ -0,0 +1,539 @@ +# -*- coding: utf-8 -*- +''' +Created on Apr 29, 2013 + +@author: ymh +''' +from iri_tweet.models import (User, EntityType, adapt_json, MediaType, Media, + EntityMedia, Hashtag, EntityHashtag, EntityUser, EntityUrl, Url, Entity, Tweet, + TweetSource, TweetLog) +from iri_tweet.utils import (ObjectsBuffer, adapt_fields, fields_adapter, + ObjectBufferProxy, get_oauth_token, clean_keys) +from sqlalchemy.orm import joinedload +import anyjson +import logging +import twitter +import twitter_text + + +class TwitterProcessorException(Exception): + pass + +class TwitterProcessor(object): + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + + if json_dict is None and json_txt is None: + raise TwitterProcessorException("No json") + + if json_dict is None: + self.json_dict = anyjson.deserialize(json_txt) + else: + self.json_dict = json_dict + + if not json_txt: + self.json_txt = anyjson.serialize(json_dict) + else: + self.json_txt = json_txt + + if "id" not in self.json_dict: + raise TwitterProcessorException("No id in json") + + self.source_id = source_id + self.session = session + self.consumer_key = consumer_token[0] + self.consumer_secret = consumer_token[1] + self.token_filename = token_filename + self.access_token = access_token + self.obj_buffer = ObjectsBuffer() + self.user_query_twitter = user_query_twitter + if not logger: + self.logger = logging.getLogger(__name__) + else: + self.logger = logger + + def process(self): + if self.source_id is None: + tweet_source = self.obj_buffer.add_object(TweetSource, None, {'original_json':self.json_txt}, True) + self.source_id = tweet_source.id + self.process_source() + self.obj_buffer.persists(self.session) + + def process_source(self): + raise NotImplementedError() + + def log_info(self): + return "Process tweet %s" % repr(self.__class__) + + +class TwitterProcessorStatus(TwitterProcessor): + + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def __get_user(self, user_dict, do_merge): + self.logger.debug("Get user : " + repr(user_dict)) #@UndefinedVariable + + user_dict = adapt_fields(user_dict, fields_adapter["stream"]["user"]) + + user_id = user_dict.get("id",None) + user_name = user_dict.get("screen_name", user_dict.get("name", None)) + + if user_id is None and user_name is None: + return None + + user = None + if user_id: + user = self.obj_buffer.get(User, id=user_id) + else: + user = self.obj_buffer.get(User, screen_name=user_name) + + #to do update user id needed + if user is not None: + user_created_at = None + if user.args is not None: + user_created_at = user.args.get('created_at', None) + if user_created_at is None and user_dict.get('created_at', None) is not None and do_merge: + if user.args is None: + user.args = user_dict + else: + user.args.update(user_dict) + return user + + #todo : add methpds to objectbuffer to get buffer user + user_obj = None + if user_id: + user_obj = self.session.query(User).filter(User.id == user_id).first() + else: + user_obj = self.session.query(User).filter(User.screen_name.ilike(user_name)).first() + + #todo update user if needed + if user_obj is not None: + if user_obj.created_at is not None or user_dict.get('created_at', None) is None or not do_merge : + user = ObjectBufferProxy(User, None, None, False, user_obj) + else: + user = self.obj_buffer.add_object(User, None, user_dict, True, user_obj) + return user + + user_created_at = user_dict.get("created_at", None) + + if user_created_at is None and self.user_query_twitter: + + if self.access_token is not None: + acess_token_key, access_token_secret = self.access_token + else: + acess_token_key, access_token_secret = get_oauth_token(consumer_key=self.consumer_key, consumer_secret=self.consumer_secret, token_file_path=self.token_filename) + #TODO pass it as argument + t = twitter.Twitter(auth=twitter.OAuth(acess_token_key, access_token_secret, self.consumer_key, self.consumer_secret)) + try: + if user_id: + user_dict = t.users.show(user_id=user_id) + else: + user_dict = t.users.show(screen_name=user_name) + except Exception as e: + self.logger.info("get_user : TWITTER ERROR : " + repr(e)) #@UndefinedVariable + self.logger.info("get_user : TWITTER ERROR : " + str(e)) #@UndefinedVariable + return None + + if "id" not in user_dict: + return None + + #TODO filter get, wrap in proxy + user_obj = self.session.query(User).filter(User.id == user_dict["id"]).first() + + if user_obj is not None and not do_merge: + return ObjectBufferProxy(User, None, None, False, user_obj) + else: + return self.obj_buffer.add_object(User, None, user_dict, True) + + def __get_or_create_object(self, klass, filter_by_kwargs, filter_arg, creation_kwargs, must_flush, do_merge): + + obj_proxy = self.obj_buffer.get(klass, **filter_by_kwargs) + if obj_proxy is None: + query = self.session.query(klass) + if filter_arg is not None: + query = query.filter(filter_arg) + else: + query = query.filter_by(**filter_by_kwargs) + obj_instance = query.first() + if obj_instance is not None: + if not do_merge: + obj_proxy = ObjectBufferProxy(klass, None, None, False, obj_instance) + else: + obj_proxy = self.obj_buffer.add_object(klass, None, creation_kwargs, must_flush, obj_instance) + if obj_proxy is None: + obj_proxy = self.obj_buffer.add_object(klass, None, creation_kwargs, must_flush) + return obj_proxy + + + def __process_entity(self, ind, ind_type): + self.logger.debug("Process_entity : " + repr(ind) + " : " + repr(ind_type)) #@UndefinedVariable + + ind = clean_keys(ind) + + entity_type = self.__get_or_create_object(EntityType, {'label':ind_type}, None, {'label':ind_type}, True, False) + + entity_dict = { + "indice_start" : ind["indices"][0], + "indice_end" : ind["indices"][1], + "tweet_id" : self.tweet.id, + "entity_type_id" : entity_type.id, + "source" : adapt_json(ind) + } + + def process_medias(): + + media_id = ind.get('id', None) + if media_id is None: + return None, None + + type_str = ind.get("type", "photo") + media_type = self.__get_or_create_object(MediaType, {'label': type_str}, None, {'label':type_str}, True, False) + media_ind = adapt_fields(ind, fields_adapter["entities"]["medias"]) + if "type" in media_ind: + del(media_ind["type"]) + media_ind['type_id'] = media_type.id + media = self.__get_or_create_object(Media, {'id':media_id}, None, media_ind, True, False) + + entity_dict['media_id'] = media.id + return EntityMedia, entity_dict + + def process_hashtags(): + text = ind.get("text", ind.get("hashtag", None)) + if text is None: + return None, None + ind['text'] = text + hashtag = self.__get_or_create_object(Hashtag, {'text':text}, Hashtag.text.ilike(text), ind, True, False) + entity_dict['hashtag_id'] = hashtag.id + return EntityHashtag, entity_dict + + def process_user_mentions(): + user_mention = self.__get_user(ind, False) + if user_mention is None: + entity_dict['user_id'] = None + else: + entity_dict['user_id'] = user_mention.id + return EntityUser, entity_dict + + def process_urls(): + url = self.__get_or_create_object(Url, {'url':ind["url"]}, None, ind, True, False) + entity_dict['url_id'] = url.id + return EntityUrl, entity_dict + + #{'': lambda } + entity_klass, entity_dict = { + 'hashtags': process_hashtags, + 'user_mentions' : process_user_mentions, + 'urls' : process_urls, + 'media': process_medias, + }.get(ind_type, lambda: (Entity, entity_dict))() + + self.logger.debug("Process_entity entity_dict: " + repr(entity_dict)) #@UndefinedVariable + if entity_klass: + self.obj_buffer.add_object(entity_klass, None, entity_dict, False) + + + def __process_twitter_stream(self): + + tweet_nb = self.session.query(Tweet).filter(Tweet.id == self.json_dict["id"]).count() + if tweet_nb > 0: + return + + ts_copy = adapt_fields(self.json_dict, fields_adapter["stream"]["tweet"]) + + # get or create user + user = self.__get_user(self.json_dict["user"], True) + if user is None: + self.logger.warning("USER not found " + repr(self.json_dict["user"])) #@UndefinedVariable + ts_copy["user_id"] = None + else: + ts_copy["user_id"] = user.id + + del(ts_copy['user']) + ts_copy["tweet_source_id"] = self.source_id + + self.tweet = self.obj_buffer.add_object(Tweet, None, ts_copy, True) + + self.__process_entities() + + + def __process_entities(self): + if "entities" in self.json_dict: + for ind_type, entity_list in self.json_dict["entities"].iteritems(): + for ind in entity_list: + self.__process_entity(ind, ind_type) + else: + + text = self.tweet.text + extractor = twitter_text.Extractor(text) + for ind in extractor.extract_hashtags_with_indices(): + self.__process_entity(ind, "hashtags") + + for ind in extractor.extract_urls_with_indices(): + self.__process_entity(ind, "urls") + + for ind in extractor.extract_mentioned_screen_names_with_indices(): + self.__process_entity(ind, "user_mentions") + + def __process_twitter_rest(self): + tweet_nb = self.session.query(Tweet).filter(Tweet.id == self.json_dict["id"]).count() + if tweet_nb > 0: + return + + + tweet_fields = { + 'created_at': self.json_dict["created_at"], + 'favorited': False, + 'id': self.json_dict["id"], + 'id_str': self.json_dict["id_str"], + #'in_reply_to_screen_name': ts["to_user"], + 'in_reply_to_user_id': self.json_dict.get("in_reply_to_user_id",None), + 'in_reply_to_user_id_str': self.json_dict.get("in_reply_to_user_id_str", None), + #'place': ts["place"], + 'source': self.json_dict["source"], + 'text': self.json_dict["text"], + 'truncated': False, + 'tweet_source_id' : self.source_id, + } + + #user + + user_fields = { + 'lang' : self.json_dict.get('iso_language_code',None), + 'profile_image_url' : self.json_dict["profile_image_url"], + 'screen_name' : self.json_dict["from_user"], + 'id' : self.json_dict["from_user_id"], + 'id_str' : self.json_dict["from_user_id_str"], + 'name' : self.json_dict['from_user_name'], + } + + user = self.__get_user(user_fields, do_merge=False) + if user is None: + self.logger.warning("USER not found " + repr(user_fields)) #@UndefinedVariable + tweet_fields["user_id"] = None + else: + tweet_fields["user_id"] = user.id + + tweet_fields = adapt_fields(tweet_fields, fields_adapter["rest"]["tweet"]) + self.tweet = self.obj_buffer.add_object(Tweet, None, tweet_fields, True) + + self.__process_entities() + + + + def process_source(self): + + status_id = self.json_dict["id"] + log = self.session.query(TweetLog).filter(TweetLog.status_id==status_id).first() + if(log): + self.obj_buffer.add_object(TweetLog, log, {'status': TweetLog.TWEET_STATUS['DELETE'], 'status_id': None}) + self.session.query(TweetSource).filter(TweetSource.id==self.source_id).delete() + else: + if "metadata" in self.json_dict: + self.__process_twitter_rest() + else: + self.__process_twitter_stream() + + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['OK']}, True) + + def log_info(self): + screen_name = self.json_dict.get("user",{}).get("screen_name","") + return u"Process Tweet from %s : %s" % (screen_name, self.json_dict.get('text',u"")) + + + +class TwitterProcessorDelete(TwitterProcessor): + """ + { + "delete":{ + "status":{ + "id":1234, + "id_str":"1234", + "user_id":3, + "user_id_str":"3" + } + } + } + """ + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process(self): + + #find tweet + tweet_id = self.json_dict.get('delete',{}).get('status',{}).get('id',None) + if tweet_id: + t = self.session.query(Tweet).options(joinedload(Tweet.tweet_source)).filter(Tweet.id == tweet_id).first() + if t: + tsource = t.tweet_source + self.session.delete(t) + self.session.query(TweetLog).filter(TweetLog.tweet_source_id == tsource.id).delete() + self.session.delete(tsource) + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['DELETE']}, True) + else: + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status_id': tweet_id,'status':TweetLog.TWEET_STATUS['DELETE_PENDING']}, True) + + def log_info(self): + status_del = self.json_dict.get('delete', {}).get("status",{}) + return u"Process delete for %s : %s" % (status_del.get('user_id_str',u""), status_del.get('id_str',u"")) + +class TwitterProcessorScrubGeo(TwitterProcessor): + """ + { + "scrub_geo":{ + "user_id":14090452, + "user_id_str":"14090452", + "up_to_status_id":23260136625, + "up_to_status_id_str":"23260136625" + } + } + """ + + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process_source(self): + up_to_status_id = self.json_dict.get("scrub_geo", {}).get("up_to_status_id", None) + if not up_to_status_id: + return + tweets = self.session.query(Tweet).options(joinedload(Tweet.tweet_source)).filter(Tweet.id <= up_to_status_id) + for t in tweets: + self.obj_buffer.add_object(Tweet, t, {'geo': None}) + tsource = t.tweet_source + tsource_dict = anyjson.serialize(tsource.original_json) + if tsource_dict.get("geo", None): + tsource_dict["geo"] = None + self.obj_buffer.add_object(TweetSource, tsource, {'original_json': anyjson.serialize(tsource_dict)}) + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['SCRUB_GEO']}, True) + + def log_info(self): + return u"Process scrub geo for %s : %s" % (self.json_dict["scrub_geo"].get('user_id_str',u""), self.json_dict["scrub_geo"].get('id_str',u"")) + + +class TwitterProcessorLimit(TwitterProcessor): + """ + { + "limit":{ + "track":1234 + } + } + """ + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process_source(self): + """ + do nothing, just log the information + """ + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['LIMIT'], 'error':self.json_txt}, True) + + def log_info(self): + return u"Process limit %d " % self.json_dict.get("limit", {}).get('track', 0) + +class TwitterProcessorStatusWithheld(TwitterProcessor): + """ + { + "status_withheld":{ + "id":1234567890, + "user_id":123456, + "withheld_in_countries":["DE", "AR"] + } + } + """ + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process_source(self): + """ + do nothing, just log the information + """ + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['STATUS_WITHHELD'], 'error':self.json_txt}, True) + + def log_info(self): + status_withheld = self.json_dict.get("status_withheld",{}) + return u"Process status withheld status id %d from user %d in countries %s" %(status_withheld.get("id",0), status_withheld.get("user_id",0), u",".join(status_withheld.get("withheld_in_countries",[]))) + +class TwitterProcessorUserWithheld(TwitterProcessor): + """ + { + "user_withheld":{ + "id":123456, + "withheld_in_countries":["DE","AR"] + } + } + """ + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process_source(self): + """ + do nothing, just log the information + """ + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['USER_WITHHELD'], 'error':self.json_txt}, True) + + + def log_info(self): + user_withheld = self.json_dict.get("user_withheld", {}) + return u"Process user withheld %d in countries %s" % (user_withheld.get("id",0), u"".join(user_withheld.get("withheld_in_countries",[]))) + +class TwitterProcessorDisconnect(TwitterProcessor): + """ + { + "disconnect":{ + "code": 4, + "stream_name":"< A stream identifier >", + "reason":"< Human readable status message >" + } + } + """ + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process_source(self): + """ + do nothing, just log the information + """ + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['DISCONNECT'], 'error':self.json_txt}, True) + + def log_info(self): + disconnect = self.json_dict.get("disconnect",{}) + return u"Process disconnect stream %s code %d reason %s" % (disconnect.get("stream_name",""), disconnect.get("code",0), disconnect.get("reason","")) + +class TwitterProcessorStallWarning(TwitterProcessor): + """ + { + "warning":{ + "code":"FALLING_BEHIND", + "message":"Your connection is falling behind and messages are being queued for delivery to you. Your queue is now over 60% full. You will be disconnected when the queue is full.", + "percent_full": 60 + } + } + """ + def __init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=None, token_filename=None, user_query_twitter=False, logger=None): + TwitterProcessor.__init__(self, json_dict, json_txt, source_id, session, consumer_token, access_token=access_token, token_filename=token_filename, user_query_twitter=user_query_twitter, logger=logger) + + def process_source(self): + """ + do nothing, just log the information + """ + self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['STALL_WARNING'], 'error':self.json_txt}, True) + + def log_info(self): + warning = self.json_dict.get("warning",{}) + return u"Process stall warning %d%% code %s, message %s" % (warning.get("percent_full",0),warning.get("code",u""), warning.get("message", u"")) + +TWEET_PROCESSOR_MAP = { + 'text': TwitterProcessorStatus, + 'delete': TwitterProcessorDelete, + 'scrub_geo': TwitterProcessorScrubGeo, + 'limit': TwitterProcessorLimit, + 'status_withheld': TwitterProcessorStatusWithheld, + 'user_withheld': TwitterProcessorUserWithheld, + 'disconnect': TwitterProcessorDisconnect, + 'warning': TwitterProcessorStallWarning +} + +def get_processor(tweet_dict): + for processor_key,processor_klass in TWEET_PROCESSOR_MAP.iteritems(): + if processor_key in tweet_dict: + return processor_klass + return None diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/iri_tweet/iri_tweet/stream.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/lib/iri_tweet/iri_tweet/stream.py Fri May 10 13:34:40 2013 +0200 @@ -0,0 +1,399 @@ +# -*- coding: utf-8 -*- +''' +Created on Mar 22, 2012 + +@author: ymh + +Module directly inspired by tweetstream + +''' +import time +import requests +from requests.utils import stream_decode_response_unicode +import anyjson +import select + +from . import USER_AGENT, ConnectionError, AuthenticationError + + +def iter_content_non_blocking(req, max_chunk_size=4096, decode_unicode=False, timeout=-1): + + if req._content_consumed: + raise RuntimeError( + 'The content for this response was already consumed' + ) + + req.raw._fp.fp._sock.setblocking(False) + + def generate(): + chunk_size = 1 + while True: + if timeout < 0: + rlist,_,_ = select.select([req.raw._fp.fp._sock], [], []) + else: + rlist,_,_ = select.select([req.raw._fp.fp._sock], [], [], timeout) + + if not rlist: + continue + + try: + chunk = req.raw.read(chunk_size, decode_content=True) + if not chunk: + break + if len(chunk) >= chunk_size and chunk_size < max_chunk_size: + chunk_size = min(chunk_size*2, max_chunk_size) + elif len(chunk) < chunk_size/2 and chunk_size < max_chunk_size: + chunk_size = max(chunk_size/2,1) + yield chunk + except requests.exceptions.SSLError as e: + if e.errno == 2: + # Apparently this means there was nothing in the socket buf + pass + else: + raise + + req._content_consumed = True + + gen = generate() + + if decode_unicode: + gen = stream_decode_response_unicode(gen, req) + + return gen + + + + +class BaseStream(object): + + """A network connection to Twitters streaming API + + :param auth: requests auth object. + :keyword catchup: Number of tweets from the past to get before switching to + live stream. + :keyword raw: If True, return each tweet's raw data direct from the socket, + without UTF8 decoding or parsing, rather than a parsed object. The + default is False. + :keyword timeout: If non-None, set a timeout in seconds on the receiving + socket. Certain types of network problems (e.g., disconnecting a VPN) + can cause the connection to hang, leading to indefinite blocking that + requires kill -9 to resolve. Setting a timeout leads to an orderly + shutdown in these cases. The default is None (i.e., no timeout). + :keyword url: Endpoint URL for the object. Note: you should not + need to edit this. It's present to make testing easier. + + .. attribute:: connected + + True if the object is currently connected to the stream. + + .. attribute:: url + + The URL to which the object is connected + + .. attribute:: starttime + + The timestamp, in seconds since the epoch, the object connected to the + streaming api. + + .. attribute:: count + + The number of tweets that have been returned by the object. + + .. attribute:: rate + + The rate at which tweets have been returned from the object as a + float. see also :attr: `rate_period`. + + .. attribute:: rate_period + + The ammount of time to sample tweets to calculate tweet rate. By + default 10 seconds. Changes to this attribute will not be reflected + until the next time the rate is calculated. The rate of tweets vary + with time of day etc. so it's usefull to set this to something + sensible. + + .. attribute:: user_agent + + User agent string that will be included in the request. NOTE: This can + not be changed after the connection has been made. This property must + thus be set before accessing the iterator. The default is set in + :attr: `USER_AGENT`. + """ + + def __init__(self, auth, + raw=False, timeout=-1, url=None, compressed=False, chunk_size=4096, logger=None): + self._conn = None + self._rate_ts = None + self._rate_cnt = 0 + self._auth = auth + self.raw_mode = raw + self.timeout = timeout + self._compressed = compressed + + self.rate_period = 10 # in seconds + self.connected = False + self.starttime = None + self.count = 0 + self.rate = 0 + self.user_agent = USER_AGENT + self.chunk_size = chunk_size + if url: self.url = url + + self.muststop = False + self._logger = logger + + self._iter = self.__iter__() + + + def __enter__(self): + return self + + def __exit__(self, *params): + self.close() + return False + + def _init_conn(self): + """Open the connection to the twitter server""" + + if self._logger : self._logger.debug("BaseStream Open the connection to the twitter server") + + headers = {'User-Agent': self.user_agent} + + if self._compressed: + headers['Accept-Encoding'] = "deflate, gzip" + + postdata = self._get_post_data() or {} + postdata['stall_warnings'] = 'true' + + if self._logger : self._logger.debug("BaseStream init connection url " + repr(self.url)) + if self._logger : self._logger.debug("BaseStream init connection headers " + repr(headers)) + if self._logger : self._logger.debug("BaseStream init connection data " + repr(postdata)) + + self._resp = requests.post(self.url, auth=self._auth, headers=headers, data=postdata, stream=True) + if self._logger : self._logger.debug("BaseStream init connection " + repr(self._resp)) + + self._resp.raise_for_status() + self.connected = True + + if not self._rate_ts: + self._rate_ts = time.time() + if not self.starttime: + self.starttime = time.time() + + + def _get_post_data(self): + """Subclasses that need to add post data to the request can override + this method and return post data. The data should be in the format + returned by urllib.urlencode.""" + return None + + def testmuststop(self): + if callable(self.muststop): + return self.muststop() + else: + return self.muststop + + def _update_rate(self): + rate_time = time.time() - self._rate_ts + if not self._rate_ts or rate_time > self.rate_period: + self.rate = self._rate_cnt / rate_time + self._rate_cnt = 0 + self._rate_ts = time.time() + + def _iter_object(self): + +# for line in self._resp.iter_lines(): +# yield line +# pending = None +# +# for chunk in self._resp.iter_content(chunk_size=self.chunk_size, decode_unicode=None): +# +# if pending is not None: +# chunk = pending + chunk +# lines = chunk.splitlines() +# +# if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: +# pending = lines.pop() +# else: +# pending = None +# +# for line in lines: +# yield line +# +# if pending is not None: +# yield pending + + pending = None + has_stopped = False + + if self._logger : self._logger.debug("BaseStream _iter_object") + + for chunk in self._resp.iter_content( + chunk_size=self.chunk_size, + decode_unicode=None): + + if self._logger : self._logger.debug("BaseStream _iter_object loop") + if self.testmuststop(): + has_stopped = True + break + + if pending is not None: + chunk = pending + chunk + lines = chunk.split('\r') + + if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: + pending = lines.pop() + else: + pending = None + + for line in lines: + yield line.strip('\n') + + if self.testmuststop(): + has_stopped = True + break + + if pending is not None: + yield pending + if has_stopped: + raise StopIteration() + + def __iter__(self): + + if self._logger : self._logger.debug("BaseStream __iter__") + if not self.connected: + if self._logger : self._logger.debug("BaseStream __iter__ not connected, connecting") + self._init_conn() + + if self._logger : self._logger.debug("BaseStream __iter__ connected") + + for line in self._iter_object(): + + if self._logger : self._logger.debug("BaseStream __iter__ line %s " % repr(line)) + + if not line: + continue + + if (self.raw_mode): + tweet = line + else: + line = line.decode("utf8") + try: + tweet = anyjson.deserialize(line) + except ValueError: + self.close() + raise ConnectionError("Got invalid data from twitter", details=line) + if 'text' in tweet: + self.count += 1 + self._rate_cnt += 1 + self._update_rate() + yield tweet + + + def next(self): + """Return the next available tweet. This call is blocking!""" + return self._iter.next() + + + def close(self): + """ + Close the connection to the streaming server. + """ + self.connected = False + + +class FilterStream(BaseStream): + url = "https://stream.twitter.com/1.1/statuses/filter.json" + + def __init__(self, auth, follow=None, locations=None, + track=None, url=None, raw=False, timeout=None, compressed=False, chunk_size=requests.models.ITER_CHUNK_SIZE, logger=None): + self._follow = follow + self._locations = locations + self._track = track + # remove follow, locations, track + BaseStream.__init__(self, auth, url=url, raw=raw, timeout=timeout, compressed=compressed, chunk_size=chunk_size, logger=logger) + + def _get_post_data(self): + postdata = {} + if self._follow: postdata["follow"] = ",".join([str(e) for e in self._follow]) + if self._locations: postdata["locations"] = ",".join(self._locations) + if self._track: postdata["track"] = ",".join(self._track) + return postdata + + +class SafeStreamWrapper(object): + + def __init__(self, base_stream, logger=None, error_cb=None, max_reconnects=-1, initial_tcp_wait=250, initial_http_wait=5000, max_wait=240000): + self._stream = base_stream + self._logger = logger + self._error_cb = error_cb + self._max_reconnects = max_reconnects + self._initial_tcp_wait = initial_tcp_wait + self._initial_http_wait = initial_http_wait + self._max_wait = max_wait + self._retry_wait = 0 + self._retry_nb = 0 + self._reconnects = 0 + + def __post_process_error(self,e): + # Note: error_cb is not called on the last error since we + # raise a ConnectionError instead + if callable(self._error_cb): + self._error_cb(e) + if self._logger: self._logger.info("stream sleeping for %d ms " % self._retry_wait) + time.sleep(float(self._retry_wait)/1000.0) + + + def __process_tcp_error(self,e): + if self._logger: self._logger.debug("connection error type :" + repr(type(e))) + if self._logger: self._logger.debug("connection error :" + repr(e)) + + self._reconnects += 1 + if self._max_reconnects >= 0 and self._reconnects > self._max_reconnects: + raise ConnectionError("Too many retries") + if self._retry_wait < self._max_wait: + self._retry_wait += self._initial_tcp_wait + if self._retry_wait > self._max_wait: + self._retry_wait = self._max_wait + + self.__post_process_error(e) + + + def __process_http_error(self,e): + if self._logger: self._logger.debug("http error type %s" % (repr(type(e)))) + if self._logger: self._logger.debug("http error on %s : %s" % (e.response.url,e.message)) + + if self._retry_wait < self._max_wait: + self._retry_wait = 2*self._retry_wait if self._retry_wait > 0 else self._initial_http_wait + if self._retry_wait > self._max_wait: + self._retry_wait = self._max_wait + + self.__post_process_error(e) + + def __iter__(self): + while not self._stream.testmuststop(): + self._retry_nb += 1 + try: + if self._logger: self._logger.debug("inner loop") + for tweet in self._stream: + if self._logger: self._logger.debug("tweet : " + repr(tweet)) + self._reconnects = 0 + self._retry_wait = 0 + if not tweet.strip(): + if self._logger: self._logger.debug("Empty Tweet received : PING") + continue + yield tweet + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + if self._logger: self._logger.debug("SafeStreamWrapper Connection Error http error on %s : %s" % (e.response.url,e.message)) + raise AuthenticationError("Error connecting to %s : %s : %s - %s" % (e.response.url,e.message, repr(e.response.headers),repr(e.response.text))) + if e.response.status_code > 200: + self.__process_http_error(e) + else: + self.__process_tcp_error(e) + except (ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.RequestException) as e: + self.__process_tcp_error(e) + + + + \ No newline at end of file diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/iri_tweet/iri_tweet/tests.py --- a/script/lib/iri_tweet/iri_tweet/tests.py Tue May 07 18:28:26 2013 +0200 +++ b/script/lib/iri_tweet/iri_tweet/tests.py Fri May 10 13:34:40 2013 +0200 @@ -3,10 +3,14 @@ from sqlalchemy.orm import relationship, backref import unittest #@UnresolvedImport from sqlalchemy.orm import sessionmaker -from iri_tweet.utils import ObjectsBuffer, TwitterProcessor +from iri_tweet.utils import ObjectsBuffer +from iri_tweet.processor import TwitterProcessorStatus from iri_tweet import models import tempfile #@UnresolvedImport import os +import logging + +logger = logging.getLogger(__name__); Base = declarative_base() @@ -129,12 +133,12 @@ def setUp(self): self.engine, self.metadata, sessionMaker = models.setup_database('sqlite:///:memory:', echo=True) self.session = sessionMaker() - file, self.tmpfilepath = tempfile.mkstemp() - os.close(file) + tmpfile, self.tmpfilepath = tempfile.mkstemp() + os.close(tmpfile) def testTwitterProcessor(self): - tp = TwitterProcessor(None, original_json, None, self.session, self.tmpfilepath) + tp = TwitterProcessorStatus(None, original_json, None, self.session, self.tmpfilepath, logger) tp.process() self.session.commit() @@ -154,7 +158,7 @@ def testTwitterProcessorMedia(self): - tp = TwitterProcessor(None, original_json_media, None, self.session, self.tmpfilepath) + tp = TwitterProcessorStatus(None, original_json_media, None, self.session, self.tmpfilepath, logger) tp.process() self.session.commit() @@ -174,7 +178,7 @@ def testTwitterProcessorMediaOthers(self): - tp = TwitterProcessor(None, original_json_media_others, None, self.session, self.tmpfilepath) + tp = TwitterProcessorStatus(None, original_json_media_others, None, self.session, self.tmpfilepath, logger) tp.process() self.session.commit() diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/iri_tweet/iri_tweet/utils.py --- a/script/lib/iri_tweet/iri_tweet/utils.py Tue May 07 18:28:26 2013 +0200 +++ b/script/lib/iri_tweet/iri_tweet/utils.py Fri May 10 13:34:40 2013 +0200 @@ -1,25 +1,22 @@ -from models import (Tweet, User, Hashtag, EntityHashtag, EntityUser, Url, - EntityUrl, CONSUMER_KEY, CONSUMER_SECRET, APPLICATION_NAME, ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET, adapt_date, adapt_json, TweetSource, TweetLog, MediaType, - Media, EntityMedia, Entity, EntityType) -from sqlalchemy.sql import select, or_ #@UnresolvedImport -import Queue #@UnresolvedImport -import anyjson #@UnresolvedImport +from models import (Tweet, User, Hashtag, EntityHashtag, APPLICATION_NAME, ACCESS_TOKEN_SECRET, adapt_date, adapt_json, + ACCESS_TOKEN_KEY) +from sqlalchemy.sql import select, or_ +import Queue import codecs import datetime import email.utils import logging import math import os.path +import socket import sys -import twitter.oauth #@UnresolvedImport -import twitter.oauth_dance #@UnresolvedImport -import twitter_text #@UnresolvedImport +import twitter.oauth +import twitter.oauth_dance CACHE_ACCESS_TOKEN = {} -def get_oauth_token(token_file_path=None, check_access_token=True, application_name=APPLICATION_NAME, consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET): +def get_oauth_token(consumer_key, consumer_secret, token_file_path=None, check_access_token=True, application_name=APPLICATION_NAME): global CACHE_ACCESS_TOKEN @@ -34,24 +31,27 @@ if res is not None and check_access_token: get_logger().debug("get_oauth_token : Check oauth tokens") #@UndefinedVariable - t = twitter.Twitter(auth=twitter.OAuth(res[0], res[1], CONSUMER_KEY, CONSUMER_SECRET)) + t = twitter.Twitter(auth=twitter.OAuth(res[0], res[1], consumer_key, consumer_secret)) status = None try: - status = t.account.rate_limit_status() + status = t.application.rate_limit_status(resources="account") except Exception as e: - get_logger().debug("get_oauth_token : error getting rate limit status %s" % repr(e)) + get_logger().debug("get_oauth_token : error getting rate limit status %s " % repr(e)) + get_logger().debug("get_oauth_token : error getting rate limit status %s " % str(e)) status = None get_logger().debug("get_oauth_token : Check oauth tokens : status %s" % repr(status)) #@UndefinedVariable - if status is None or status['remaining_hits'] == 0: + if status is None or status.get("resources",{}).get("account",{}).get('/account/verify_credentials',{}).get('remaining',0) == 0: get_logger().debug("get_oauth_token : Problem with status %s" % repr(status)) res = None if res is None: get_logger().debug("get_oauth_token : doing the oauth dance") res = twitter.oauth_dance(application_name, consumer_key, consumer_secret, token_file_path) + CACHE_ACCESS_TOKEN[application_name] = res + get_logger().debug("get_oauth_token : done got %s" % repr(res)) return res def parse_date(date_str): @@ -59,7 +59,7 @@ return datetime.datetime(*ts[0:7]) def clean_keys(dict_val): - return dict([(str(key),value) for key,value in dict_val.items()]) + return dict([(str(key),value) for key,value in dict_val.iteritems()]) fields_adapter = { 'stream': { @@ -100,7 +100,7 @@ return adapter_mapping[field](value) else: return value - return dict([(str(k),adapt_one_field(k,v)) for k,v in fields_dict.items()]) + return dict([(str(k),adapt_one_field(k,v)) for k,v in fields_dict.iteritems()]) class ObjectBufferProxy(object): @@ -113,7 +113,7 @@ def persists(self, session): new_args = [arg() if callable(arg) else arg for arg in self.args] if self.args is not None else [] - new_kwargs = dict([(k,v()) if callable(v) else (k,v) for k,v in self.kwargs.items()]) if self.kwargs is not None else {} + new_kwargs = dict([(k,v()) if callable(v) else (k,v) for k,v in self.kwargs.iteritems()]) if self.kwargs is not None else {} if self.instance is None: self.instance = self.klass(*new_args, **new_kwargs) @@ -160,7 +160,7 @@ if proxy.kwargs is None or len(proxy.kwargs) == 0 or proxy.klass != klass: continue found = True - for k,v in kwargs.items(): + for k,v in kwargs.iteritems(): if (k not in proxy.kwargs) or v != proxy.kwargs[k]: found = False break @@ -168,301 +168,6 @@ return proxy return None -class TwitterProcessorException(Exception): - pass - -class TwitterProcessor(object): - - def __init__(self, json_dict, json_txt, source_id, session, access_token=None, token_filename=None, user_query_twitter=False): - - if json_dict is None and json_txt is None: - raise TwitterProcessorException("No json") - - if json_dict is None: - self.json_dict = anyjson.deserialize(json_txt) - else: - self.json_dict = json_dict - - if not json_txt: - self.json_txt = anyjson.serialize(json_dict) - else: - self.json_txt = json_txt - - if "id" not in self.json_dict: - raise TwitterProcessorException("No id in json") - - self.source_id = source_id - self.session = session - self.token_filename = token_filename - self.access_token = access_token - self.obj_buffer = ObjectsBuffer() - self.user_query_twitter = user_query_twitter - - - - def __get_user(self, user_dict, do_merge): - get_logger().debug("Get user : " + repr(user_dict)) #@UndefinedVariable - - user_dict = adapt_fields(user_dict, fields_adapter["stream"]["user"]) - - user_id = user_dict.get("id",None) - user_name = user_dict.get("screen_name", user_dict.get("name", None)) - - if user_id is None and user_name is None: - return None - - user = None - if user_id: - user = self.obj_buffer.get(User, id=user_id) - else: - user = self.obj_buffer.get(User, screen_name=user_name) - - #to do update user id needed - if user is not None: - user_created_at = None - if user.args is not None: - user_created_at = user.args.get('created_at', None) - if user_created_at is None and user_dict.get('created_at', None) is not None and do_merge: - if user.args is None: - user.args = user_dict - else: - user.args.update(user_dict) - return user - - #todo : add methpds to objectbuffer to get buffer user - user_obj = None - if user_id: - user_obj = self.session.query(User).filter(User.id == user_id).first() - else: - user_obj = self.session.query(User).filter(User.screen_name.ilike(user_name)).first() - - #todo update user if needed - if user_obj is not None: - if user_obj.created_at is not None or user_dict.get('created_at', None) is None or not do_merge : - user = ObjectBufferProxy(User, None, None, False, user_obj) - else: - user = self.obj_buffer.add_object(User, None, user_dict, True, user_obj) - return user - - user_created_at = user_dict.get("created_at", None) - - if user_created_at is None and self.user_query_twitter: - - if self.access_token is not None: - acess_token_key, access_token_secret = self.access_token - else: - acess_token_key, access_token_secret = get_oauth_token(self.token_filename) - t = twitter.Twitter(auth=twitter.OAuth(acess_token_key, access_token_secret, CONSUMER_KEY, CONSUMER_SECRET)) - try: - if user_id: - user_dict = t.users.show(user_id=user_id) - else: - user_dict = t.users.show(screen_name=user_name) - except Exception as e: - get_logger().info("get_user : TWITTER ERROR : " + repr(e)) #@UndefinedVariable - get_logger().info("get_user : TWITTER ERROR : " + str(e)) #@UndefinedVariable - return None - - if "id" not in user_dict: - return None - - #TODO filter get, wrap in proxy - user_obj = self.session.query(User).filter(User.id == user_dict["id"]).first() - - if user_obj is not None and not do_merge: - return ObjectBufferProxy(User, None, None, False, user_obj) - else: - return self.obj_buffer.add_object(User, None, user_dict, True) - - def __get_or_create_object(self, klass, filter_by_kwargs, filter_arg, creation_kwargs, must_flush, do_merge): - - obj_proxy = self.obj_buffer.get(klass, **filter_by_kwargs) - if obj_proxy is None: - query = self.session.query(klass) - if filter_arg is not None: - query = query.filter(filter_arg) - else: - query = query.filter_by(**filter_by_kwargs) - obj_instance = query.first() - if obj_instance is not None: - if not do_merge: - obj_proxy = ObjectBufferProxy(klass, None, None, False, obj_instance) - else: - obj_proxy = self.obj_buffer.add_object(klass, None, creation_kwargs, must_flush, obj_instance) - if obj_proxy is None: - obj_proxy = self.obj_buffer.add_object(klass, None, creation_kwargs, must_flush) - return obj_proxy - - - def __process_entity(self, ind, ind_type): - get_logger().debug("Process_entity : " + repr(ind) + " : " + repr(ind_type)) #@UndefinedVariable - - ind = clean_keys(ind) - - entity_type = self.__get_or_create_object(EntityType, {'label':ind_type}, None, {'label':ind_type}, True, False) - - entity_dict = { - "indice_start" : ind["indices"][0], - "indice_end" : ind["indices"][1], - "tweet_id" : self.tweet.id, - "entity_type_id" : entity_type.id, - "source" : adapt_json(ind) - } - - def process_medias(): - - media_id = ind.get('id', None) - if media_id is None: - return None, None - - type_str = ind.get("type", "photo") - media_type = self.__get_or_create_object(MediaType, {'label': type_str}, None, {'label':type_str}, True, False) - media_ind = adapt_fields(ind, fields_adapter["entities"]["medias"]) - if "type" in media_ind: - del(media_ind["type"]) - media_ind['type_id'] = media_type.id - media = self.__get_or_create_object(Media, {'id':media_id}, None, media_ind, True, False) - - entity_dict['media_id'] = media.id - return EntityMedia, entity_dict - - def process_hashtags(): - text = ind.get("text", ind.get("hashtag", None)) - if text is None: - return None, None - ind['text'] = text - hashtag = self.__get_or_create_object(Hashtag, {'text':text}, Hashtag.text.ilike(text), ind, True, False) - entity_dict['hashtag_id'] = hashtag.id - return EntityHashtag, entity_dict - - def process_user_mentions(): - user_mention = self.__get_user(ind, False) - if user_mention is None: - entity_dict['user_id'] = None - else: - entity_dict['user_id'] = user_mention.id - return EntityUser, entity_dict - - def process_urls(): - url = self.__get_or_create_object(Url, {'url':ind["url"]}, None, ind, True, False) - entity_dict['url_id'] = url.id - return EntityUrl, entity_dict - - #{'': lambda } - entity_klass, entity_dict = { - 'hashtags': process_hashtags, - 'user_mentions' : process_user_mentions, - 'urls' : process_urls, - 'media': process_medias, - }.get(ind_type, lambda: (Entity, entity_dict))() - - get_logger().debug("Process_entity entity_dict: " + repr(entity_dict)) #@UndefinedVariable - if entity_klass: - self.obj_buffer.add_object(entity_klass, None, entity_dict, False) - - - def __process_twitter_stream(self): - - tweet_nb = self.session.query(Tweet).filter(Tweet.id == self.json_dict["id"]).count() - if tweet_nb > 0: - return - - ts_copy = adapt_fields(self.json_dict, fields_adapter["stream"]["tweet"]) - - # get or create user - user = self.__get_user(self.json_dict["user"], True) - if user is None: - get_logger().warning("USER not found " + repr(self.json_dict["user"])) #@UndefinedVariable - ts_copy["user_id"] = None - else: - ts_copy["user_id"] = user.id - - del(ts_copy['user']) - ts_copy["tweet_source_id"] = self.source_id - - self.tweet = self.obj_buffer.add_object(Tweet, None, ts_copy, True) - - self.__process_entities() - - - def __process_entities(self): - if "entities" in self.json_dict: - for ind_type, entity_list in self.json_dict["entities"].items(): - for ind in entity_list: - self.__process_entity(ind, ind_type) - else: - - text = self.tweet.text - extractor = twitter_text.Extractor(text) - for ind in extractor.extract_hashtags_with_indices(): - self.__process_entity(ind, "hashtags") - - for ind in extractor.extract_urls_with_indices(): - self.__process_entity(ind, "urls") - - for ind in extractor.extract_mentioned_screen_names_with_indices(): - self.__process_entity(ind, "user_mentions") - - def __process_twitter_rest(self): - tweet_nb = self.session.query(Tweet).filter(Tweet.id == self.json_dict["id"]).count() - if tweet_nb > 0: - return - - - tweet_fields = { - 'created_at': self.json_dict["created_at"], - 'favorited': False, - 'id': self.json_dict["id"], - 'id_str': self.json_dict["id_str"], - #'in_reply_to_screen_name': ts["to_user"], - 'in_reply_to_user_id': self.json_dict.get("in_reply_to_user_id",None), - 'in_reply_to_user_id_str': self.json_dict.get("in_reply_to_user_id_str", None), - #'place': ts["place"], - 'source': self.json_dict["source"], - 'text': self.json_dict["text"], - 'truncated': False, - 'tweet_source_id' : self.source_id, - } - - #user - - user_fields = { - 'lang' : self.json_dict.get('iso_language_code',None), - 'profile_image_url' : self.json_dict["profile_image_url"], - 'screen_name' : self.json_dict["from_user"], - 'id' : self.json_dict["from_user_id"], - 'id_str' : self.json_dict["from_user_id_str"], - 'name' : self.json_dict['from_user_name'], - } - - user = self.__get_user(user_fields, do_merge=False) - if user is None: - get_logger().warning("USER not found " + repr(user_fields)) #@UndefinedVariable - tweet_fields["user_id"] = None - else: - tweet_fields["user_id"] = user.id - - tweet_fields = adapt_fields(tweet_fields, fields_adapter["rest"]["tweet"]) - self.tweet = self.obj_buffer.add_object(Tweet, None, tweet_fields, True) - - self.__process_entities() - - - - def process(self): - - if self.source_id is None: - tweet_source = self.obj_buffer.add_object(TweetSource, None, {'original_json':self.json_txt}, True) - self.source_id = tweet_source.id - - if "metadata" in self.json_dict: - self.__process_twitter_rest() - else: - self.__process_twitter_stream() - - self.obj_buffer.add_object(TweetLog, None, {'tweet_source_id':self.source_id, 'status':TweetLog.TWEET_STATUS['OK']}, True) - - self.obj_buffer.persists(self.session) def set_logging(options, plogger=None, queue=None): @@ -507,12 +212,12 @@ return logger def set_logging_options(parser): - parser.add_option("-l", "--log", dest="logfile", + parser.add_argument("-l", "--log", dest="logfile", help="log to file", metavar="LOG", default="stderr") - parser.add_option("-v", dest="verbose", action="count", - help="verbose", metavar="VERBOSE", default=0) - parser.add_option("-q", dest="quiet", action="count", - help="quiet", metavar="QUIET", default=0) + parser.add_argument("-v", dest="verbose", action="count", + help="verbose", default=0) + parser.add_argument("-q", dest="quiet", action="count", + help="quiet", default=0) def get_base_query(session, query, start_date, end_date, hashtags, tweet_exclude_table, user_whitelist): @@ -561,19 +266,17 @@ def get_logger(): global logger_name - return logging.getLogger(logger_name) #@UndefinedVariable + return logging.getLogger(logger_name) -# Next two import lines for this demo only - -class QueueHandler(logging.Handler): #@UndefinedVariable +class QueueHandler(logging.Handler): """ This is a logging handler which sends events to a multiprocessing queue. """ def __init__(self, queue, ignore_full): """ - Initialise an instance, using the passed queue. + Initialize an instance, using the passed queue. """ logging.Handler.__init__(self) #@UndefinedVariable self.queue = queue @@ -588,10 +291,12 @@ try: ei = record.exc_info if ei: - dummy = self.format(record) # just to get traceback text into record.exc_text + _ = self.format(record) # just to get traceback text into record.exc_text record.exc_info = None # not needed any more - if not self.ignore_full or not self.queue.full(): + if not self.ignore_full or (not self.queue.full()): self.queue.put_nowait(record) + except AssertionError: + pass except Queue.Full: if self.ignore_full: pass @@ -615,9 +320,9 @@ spaces = math.floor(width - marks) loader = u'[' + (u'=' * int(marks)) + (u' ' * int(spaces)) + u']' - + s = u"%s %3d%% %*d/%d - %*s\r" % (loader, percent, len(str(total_line)), current_line, total_line, width, label[:width]) - + writer.write(s) #takes the header into account if percent >= 100: writer.write("\n") @@ -625,3 +330,10 @@ return writer +def get_unused_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('localhost', 0)) + _, port = s.getsockname() + s.close() + return port + diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/iri_tweet/setup.py --- a/script/lib/iri_tweet/setup.py Tue May 07 18:28:26 2013 +0200 +++ b/script/lib/iri_tweet/setup.py Fri May 10 13:34:40 2013 +0200 @@ -45,7 +45,7 @@ if line.strip() == '# -eof meta-': break acc.append(line) - for pattern, handler in pats.items(): + for pattern, handler in pats.iteritems(): m = pattern.match(line.strip()) if m: meta.update(handler(m)) diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/CHANGELOG --- a/script/lib/tweetstream/CHANGELOG Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,68 +0,0 @@ -0.1 - - - Initial version - -0.2 - - - Improved error handling - - Added AuthenticationError and ConnectionError exceptions - - Added ReconnectingTweetStream class that supports automatically - reconnecting if the connection is dropped - -0.3 - - - Fixed bugs in authtentication - - Added TrackStream and FollowStream classes - - Added list of endpoint names, and made them legal values for the url arg - -0.3.1 - - - Added lots of tests - - Added proper handling of keepalive newlines - - Improved handling of closing streams - - Added missing anyjson dependency to setup - - Fixed bug where newlines and malformed content were counted as a tweet - -0.3.2 - - - This release was skipped over, due to maintainer brainfart. - -0.3.3 - - - Fixed setup.py so it wont attempt to load modules that aren't installed - yet. Fixes installation issue. - -0.3.4 - - - Updated to latest twitter streaming urls - - Fixed a bug where we tried to call a method on None - -0.3.5 - - - Removed a spurious print statement left over from debugging - - Introduced common base class for all tweetstream exceptions - - Make sure we raise a sensible error on 404. Include url in desc of that error - -0.3.6 - - - Added LocationStream class for filtering on location bounding boxes. - -1.0.0 - - - Changed API to match latest twitter endpoints. This adds SampleStream and - FilterStream and deprecates TweetStream, FollowStream, LocationStream, - TrackStream and ReconnectingTweetStream. - -1.1.0 - - - Fixed issues #2 and #12, related to low volume streams not yielding tweets - until a relatively large buffer was filled. This meant that tweets would - arrive in bunches, not immediatly. - - Switched to HTTPS urls for streams. Twitter will switch off HTTP streams - on 29. sept. 2011. - - Added support for Python 3 - -1.1.1 - - - Fixed issue #16. Odd case where python_version_tuple was returning ints - rather than the strings the docs promis. Make sure we always cast to int. diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/LICENSE --- a/script/lib/tweetstream/LICENSE Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -Copyright (c) 2009, Rune Halvorsen -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -Neither the name of Rune Halvorsen nor the names of its contributors may be -used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS -BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/README --- a/script/lib/tweetstream/README Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,110 +0,0 @@ -.. -*- restructuredtext -*- - -########################################## -tweetstream - Simple twitter streaming API -########################################## - -Introduction ------------- - -tweetstream provides two classes, SampleStream and FollowStream, that can be -used to get tweets from Twitter's streaming API. An instance of one of the -classes can be used as an iterator. In addition to fetching tweets, the -object keeps track of the number of tweets collected and the rate at which -tweets are received. - -SampleStream delivers a sample of all tweets. FilterStream delivers -tweets that match one or more criteria. Note that it's not possible -to get all tweets without access to the "firehose" stream, which -is not currently avaliable to the public. - -Twitter's documentation about the streaming API can be found here: -http://dev.twitter.com/pages/streaming_api_methods . - -**Note** that the API is blocking. If for some reason data is not immediatly -available, calls will block until enough data is available to yield a tweet. - -Examples --------- - -Printing incoming tweets: - ->>> stream = tweetstream.SampleStream("username", "password") ->>> for tweet in stream: -... print tweet - - -The stream object can also be used as a context, as in this example that -prints the author for each tweet as well as the tweet count and rate: - ->>> with tweetstream.SampleStream("username", "password") as stream -... for tweet in stream: -... print "Got tweet from %-16s\t( tweet %d, rate %.1f tweets/sec)" % ( -... tweet["user"]["screen_name"], stream.count, stream.rate ) - - -Stream objects can raise ConnectionError or AuthenticationError exceptions: - ->>> try: -... with tweetstream.TweetStream("username", "password") as stream -... for tweet in stream: -... print "Got tweet from %-16s\t( tweet %d, rate %.1f tweets/sec)" % ( -... tweet["user"]["screen_name"], stream.count, stream.rate ) -... except tweetstream.ConnectionError, e: -... print "Disconnected from twitter. Reason:", e.reason - -To get tweets that match specific criteria, use the FilterStream. FilterStreams -take three keyword arguments: "locations", "follow" and "track". - -Locations are a list of bounding boxes in which geotagged tweets should originate. -The argument should be an iterable of longitude/latitude pairs. - -Track specifies keywords to track. The argument should be an iterable of -strings. - -Follow returns statuses that reference given users. Argument should be an iterable -of twitter user IDs. The IDs are userid ints, not the screen names. - ->>> words = ["opera", "firefox", "safari"] ->>> people = [123,124,125] ->>> locations = ["-122.75,36.8", "-121.75,37.8"] ->>> with tweetstream.FilterStream("username", "password", track=words, -... follow=people, locations=locations) as stream -... for tweet in stream: -... print "Got interesting tweet:", tweet - - -Deprecated classes ------------------- - -tweetstream used to contain the classes TweetStream, FollowStream, TrackStream -LocationStream and ReconnectingTweetStream. These were deprecated when twitter -changed its API end points. The same functionality is now available in -SampleStream and FilterStream. The deprecated methods will emit a warning when -used, but will remain functional for a while longer. - - -Changelog ---------- - -See the CHANGELOG file - -Contact -------- - -The author is Rune Halvorsen . The project resides at -http://bitbucket.org/runeh/tweetstream . If you find bugs, or have feature -requests, please report them in the project site issue tracker. Patches are -also very welcome. - -Contributors ------------- - -- Rune Halvorsen -- Christopher Schierkolk - -License -------- - -This software is licensed under the ``New BSD License``. See the ``LICENCE`` -file in the top distribution directory for the full license text. diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/conftest.py --- a/script/lib/tweetstream/conftest.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,10 +0,0 @@ -# content of conftest.py - -import pytest -def pytest_addoption(parser): - parser.addoption("--runslow", action="store_true", - help="run slow tests") - -def pytest_runtest_setup(item): - if 'slow' in item.keywords and not item.config.getvalue("runslow"): - pytest.skip("need --runslow option to run") \ No newline at end of file diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/servercontext.py --- a/script/lib/tweetstream/servercontext.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,221 +0,0 @@ -import threading -import contextlib -import time -import os -import socket -import random -from functools import partial -from inspect import isclass -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from SimpleHTTPServer import SimpleHTTPRequestHandler -from SocketServer import BaseRequestHandler - - -class ServerError(Exception): - pass - - -class ServerContext(object): - """Context object with information about a running test server.""" - - def __init__(self, address, port): - self.address = address or "localhost" - self.port = port - - @property - def baseurl(self): - return "http://%s:%s" % (self.address, self.port) - - def __str__(self): - return "" % self.baseurl - - __repr__ = __str__ - - -class _SilentSimpleHTTPRequestHandler(SimpleHTTPRequestHandler): - - def __init__(self, *args, **kwargs): - self.logging = kwargs.get("logging", False) - SimpleHTTPRequestHandler.__init__(self, *args, **kwargs) - - def log_message(self, *args, **kwargs): - if self.logging: - SimpleHTTPRequestHandler.log_message(self, *args, **kwargs) - - -class _TestHandler(BaseHTTPRequestHandler): - """RequestHandler class that handles requests that use a custom handler - callable.""" - - def __init__(self, handler, methods, *args, **kwargs): - self._handler = handler - self._methods = methods - self._response_sent = False - self._headers_sent = False - self.logging = kwargs.get("logging", False) - BaseHTTPRequestHandler.__init__(self, *args, **kwargs) - - def log_message(self, *args, **kwargs): - if self.logging: - BaseHTTPRequestHandler.log_message(self, *args, **kwargs) - - def send_response(self, *args, **kwargs): - self._response_sent = True - BaseHTTPRequestHandler.send_response(self, *args, **kwargs) - - def end_headers(self, *args, **kwargs): - self._headers_sent = True - BaseHTTPRequestHandler.end_headers(self, *args, **kwargs) - - def _do_whatever(self): - """Called in place of do_METHOD""" - data = self._handler(self) - - if hasattr(data, "next"): - # assume it's something supporting generator protocol - self._handle_with_iterator(data) - else: - # Nothing more to do then. - pass - - - def __getattr__(self, name): - if name.startswith("do_") and name[3:].lower() in self._methods: - return self._do_whatever - else: - # fixme instance or class? - raise AttributeError(name) - - def _handle_with_iterator(self, iterator): - self.connection.settimeout(0.1) - for data in iterator: - if not self.server.server_thread.running: - return - - if not self._response_sent: - self.send_response(200) - if not self._headers_sent: - self.end_headers() - - self.wfile.write(data) - # flush immediatly. We may want to do trickling writes - # or something else tha trequires bypassing normal caching - self.wfile.flush() - -class _TestServerThread(threading.Thread): - """Thread class for a running test server""" - - def __init__(self, handler, methods, cwd, port, address): - threading.Thread.__init__(self) - self.startup_finished = threading.Event() - self._methods = methods - self._cwd = cwd - self._orig_cwd = None - self._handler = self._wrap_handler(handler, methods) - self._setup() - self.running = True - self.serverloc = (address, port) - self.error = None - - def _wrap_handler(self, handler, methods): - if isclass(handler) and issubclass(handler, BaseRequestHandler): - return handler # It's OK. user passed in a proper handler - elif callable(handler): - return partial(_TestHandler, handler, methods) - # it's a callable, so wrap in a req handler - else: - raise ServerError("handler must be callable or RequestHandler") - - def _setup(self): - if self._cwd != "./": - self._orig_cwd = os.getcwd() - os.chdir(self._cwd) - - def _init_server(self): - """Hooks up the server socket""" - try: - if self.serverloc[1] == "random": - retries = 10 # try getting an available port max this many times - while True: - try: - self.serverloc = (self.serverloc[0], - random.randint(1025, 49151)) - self._server = HTTPServer(self.serverloc, self._handler) - except socket.error: - retries -= 1 - if not retries: # not able to get a port. - raise - else: - break - else: # use specific port. this might throw, that's expected - self._server = HTTPServer(self.serverloc, self._handler) - except socket.error, e: - self.running = False - self.error = e - # set this here, since we'll never enter the serve loop where - # it is usually set: - self.startup_finished.set() - return - - self._server.allow_reuse_address = True # lots of tests, same port - self._server.timeout = 0.1 - self._server.server_thread = self - - - def run(self): - self._init_server() - - while self.running: - self._server.handle_request() # blocks for self.timeout secs - # First time this falls through, signal the parent thread that - # the server is ready for incomming connections - if not self.startup_finished.is_set(): - self.startup_finished.set() - - self._cleanup() - - def stop(self): - """Stop the server and attempt to make the thread terminate. - This happens async but the calling code can check periodically - the isRunning flag on the thread object. - """ - # actual stopping happens in the run method - self.running = False - - def _cleanup(self): - """Do some rudimentary cleanup.""" - if self._orig_cwd: - os.chdir(self._orig_cwd) - - -@contextlib.contextmanager -def test_server(handler=_SilentSimpleHTTPRequestHandler, port=8514, - address="", methods=("get", "head"), cwd="./"): - """Context that makes available a web server in a separate thread""" - thread = _TestServerThread(handler=handler, methods=methods, cwd=cwd, - port=port, address=address) - thread.start() - - # fixme: should this be daemonized? If it isn't it will block the entire - # app, but that should never happen anyway.. - thread.startup_finished.wait() - - if thread.error: # startup failed! Bail, throw whatever the server did - raise thread.error - - exc = None - try: - yield ServerContext(*thread.serverloc) - except Exception, exc: - pass - thread.stop() - thread.join(5) # giving it a lot of leeway. should never happen - - if exc: - raise exc - - # fixme: this takes second priorty after the internal exception but would - # still be nice to signal back to calling code. - - if thread.isAlive(): - raise Warning("Test server could not be stopped") diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/setup.py --- a/script/lib/tweetstream/setup.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -#@PydevCodeAnalysisIgnore -import sys -import os - -extra = {} -if sys.version_info >= (3, 0): - extra.update(use_2to3=True) - - -try: - from setuptools import setup, find_packages -except ImportError: - from distutils.core import setup, find_packages - - -# -*- Distribution Meta -*- -import re -re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') -re_vers = re.compile(r'VERSION\s*=\s*$(.*?)$') -re_doc = re.compile(r'^"""(.+?)"""', re.M|re.S) -rq = lambda s: s.strip("\"'") - - -def add_default(m): - attr_name, attr_value = m.groups() - return ((attr_name, rq(attr_value)), ) - - -def add_version(m): - v = list(map(rq, m.groups()[0].split(", "))) - return (("VERSION", ".".join(v[0:3]) + "".join(v[3:])), ) - - -def add_doc(m): - return (("doc", m.groups()[0].replace("\n", " ")), ) - -pats = {re_meta: add_default, - re_vers: add_version} -here = os.path.abspath(os.path.dirname(__file__)) -meta_fh = open(os.path.join(here, "tweetstream/__init__.py")) -try: - meta = {} - acc = [] - for line in meta_fh: - if line.strip() == '# -eof meta-': - break - acc.append(line) - for pattern, handler in pats.items(): - m = pattern.match(line.strip()) - if m: - meta.update(handler(m)) - m = re_doc.match("".join(acc).strip()) - if m: - meta.update(add_doc(m)) -finally: - meta_fh.close() - - -setup(name='tweetstream', - version=meta["VERSION"], - description=meta["doc"], - long_description=open("README").read(), - classifiers=[ - 'License :: OSI Approved :: BSD License', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.1', - ], - keywords='twitter', - author=meta["author"], - author_email=meta["contact"], - url=meta["homepage"], - license='BSD', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), - include_package_data=True, - zip_safe=False, - platforms=["any"], - install_requires=['anyjson'], - **extra -) diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/tests/test_tweetstream.py --- a/script/lib/tweetstream/tests/test_tweetstream.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,188 +0,0 @@ -import contextlib -import threading -import time - -from tweetstream import TweetStream, FollowStream, TrackStream, LocationStream -from tweetstream import ConnectionError, AuthenticationError, SampleStream, FilterStream -from tweepy.auth import BasicAuthHandler - -import pytest -from pytest import raises -slow = pytest.mark.slow - -from servercontext import test_server - -single_tweet = r"""{"in_reply_to_status_id":null,"in_reply_to_user_id":null,"favorited":false,"created_at":"Tue Jun 16 10:40:14 +0000 2009","in_reply_to_screen_name":null,"text":"record industry just keeps on amazing me: http:\/\/is.gd\/13lFo - $150k per song you've SHARED, not that somebody has actually DOWNLOADED.","user":{"notifications":null,"profile_background_tile":false,"followers_count":206,"time_zone":"Copenhagen","utc_offset":3600,"friends_count":191,"profile_background_color":"ffffff","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/250715794\/profile_normal.png","description":"Digital product developer, currently at Opera Software. My tweets are my opinions, not those of my employer.","verified_profile":false,"protected":false,"favourites_count":0,"profile_text_color":"3C3940","screen_name":"eiriksnilsen","name":"Eirik Stridsklev N.","following":null,"created_at":"Tue May 06 12:24:12 +0000 2008","profile_background_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_background_images\/10531192\/160x600opera15.gif","profile_link_color":"0099B9","profile_sidebar_fill_color":"95E8EC","url":"http:\/\/www.stridsklev-nilsen.no\/eirik","id":14672543,"statuses_count":506,"profile_sidebar_border_color":"5ED4DC","location":"Oslo, Norway"},"id":2190767504,"truncated":false,"source":"Twitter Opera widget<\/a>"}""" + "\r" - - -def parameterized(funcarglist): - def wrapper(function): - function.funcarglist = funcarglist - return function - return wrapper - -def pytest_generate_tests(metafunc): - for funcargs in getattr(metafunc.function, 'funcarglist', ()): - metafunc.addcall(funcargs=funcargs) - - -streamtypes = [ - dict(cls=TweetStream, args=[], kwargs=dict()), - dict(cls=SampleStream, args=[], kwargs=dict()), - dict(cls=FilterStream, args=[], kwargs=dict(track=("test",))), - dict(cls=FollowStream, args=[[1, 2, 3]], kwargs=dict()), - dict(cls=TrackStream, args=["opera"], kwargs=dict()), - dict(cls=LocationStream, args=["123,4321"], kwargs=dict()) -] - - -@parameterized(streamtypes) -def test_bad_auth(cls, args, kwargs): - """Test that the proper exception is raised when the user could not be - authenticated""" - def auth_denied(request): - request.send_error(401) - - with raises(AuthenticationError): - with test_server(handler=auth_denied, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("user", "passwd") - stream = cls(auth, *args, url=server.baseurl) - for e in stream: pass - - -@parameterized(streamtypes) -def test_404_url(cls, args, kwargs): - """Test that the proper exception is raised when the stream URL can't be - found""" - def not_found(request): - request.send_error(404) - - with raises(ConnectionError): - with test_server(handler=not_found, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("user", "passwd") - stream = cls(auth, *args, url=server.baseurl) - for e in stream: pass - - -@parameterized(streamtypes) -def test_bad_content(cls, args, kwargs): - """Test error handling if we are given invalid data""" - def bad_content(request): - for n in xrange(10): - # what json we pass doesn't matter. It's not verifying the - # strcuture, only checking that it's parsable - yield "[1,2,3]\r" - yield "[1,2, I need no stinking close brace\r" - yield "[1,2,3]\r" - - - with raises(ConnectionError): - with test_server(handler=bad_content, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("user", "passwd") - stream = cls(auth, *args, url=server.baseurl) - for tweet in stream: - pass - - -@parameterized(streamtypes) -def test_closed_connection(cls, args, kwargs): - """Test error handling if server unexpectedly closes connection""" - cnt = 1000 - def bad_content(request): - for n in xrange(cnt): - # what json we pass doesn't matter. It's not verifying the - # strcuture, only checking that it's parsable - yield "[1,2,3]\r" - - with raises(ConnectionError): - with test_server(handler=bad_content, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("foo", "bar") - stream = cls(auth, *args, url=server.baseurl) - for tweet in stream: - pass - - -@parameterized(streamtypes) -def test_bad_host(cls, args, kwargs): - """Test behaviour if we can't connect to the host""" - with raises(ConnectionError): - stream = cls("username", "passwd", *args, url="http://wedfwecfghhreewerewads.foo") - stream.next() - - -@parameterized(streamtypes) -def smoke_test_receive_tweets(cls, args, kwargs): - """Receive 100k tweets and disconnect (slow)""" - total = 100000 - - def tweetsource(request): - while True: - yield single_tweet + "\n" - - with test_server(handler=tweetsource, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("foo", "bar") - stream = cls(auth, *args, url=server.baseurl) - for tweet in stream: - if stream.count == total: - break - - -@parameterized(streamtypes) -def test_keepalive(cls, args, kwargs): - """Make sure we behave sanely when there are keepalive newlines in the - data recevived from twitter""" - def tweetsource(request): - yield single_tweet+"\n" - yield "\n" - yield "\n" - yield single_tweet+"\n" - yield "\n" - yield "\n" - yield "\n" - yield "\n" - yield "\n" - yield "\n" - yield "\n" - yield single_tweet+"\n" - yield "\n" - - - with test_server(handler=tweetsource, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("foo", "bar") - stream = cls(auth, *args, url=server.baseurl) - try: - for tweet in stream: - pass - except ConnectionError: - assert stream.count == 3, "Got %s, wanted 3" % stream.count - else: - assert False, "Didn't handle keepalive" - - -@slow -@parameterized(streamtypes) -def test_buffering(cls, args, kwargs): - """Test if buffering stops data from being returned immediately. - If there is some buffering in play that might mean data is only returned - from the generator when the buffer is full. If buffer is bigger than a - tweet, this will happen. Default buffer size in the part of socket lib - that enables readline is 8k. Max tweet length is around 3k.""" - - def tweetsource(request): - yield single_tweet+"\n" - time.sleep(2) - # need to yield a bunch here so we're sure we'll return from the - # blocking call in case the buffering bug is present. - for n in xrange(100): - yield single_tweet+"\n" - - - with test_server(handler=tweetsource, methods=("post", "get"), port="random") as server: - auth = BasicAuthHandler("foo", "bar") - stream = cls(auth, *args, url=server.baseurl) - start = time.time() - stream.next() - first = time.time() - diff = first - start - assert diff < 1, "Getting first tweet took more than a second!" - diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/tox.ini --- a/script/lib/tweetstream/tox.ini Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,16 +0,0 @@ -[tox] -envlist = py25,py26,py27,py30,py31,py32 - -[testenv] -deps=pytest -sitepackages=False -commands=py.test --runslow - -[testenv:py30] -changedir = .tox - -[testenv:py31] -changedir = .tox - -[testenv:py32] -changedir = .tox \ No newline at end of file diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/tweetstream/__init__.py --- a/script/lib/tweetstream/tweetstream/__init__.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,45 +0,0 @@ -"""Simple access to Twitter's streaming API""" - -VERSION = (1, 1, 1, 'iri') -__version__ = ".".join(map(str, VERSION[0:3])) + "".join(VERSION[3:]) -__author__ = "Rune Halvorsen" -__contact__ = "runefh@gmail.com" -__homepage__ = "http://bitbucket.org/runeh/tweetstream/" -__docformat__ = "restructuredtext" - -# -eof meta- - - -""" - .. data:: USER_AGENT - - The default user agent string for stream objects -""" - -USER_AGENT = "TweetStream %s" % __version__ - - -class TweetStreamError(Exception): - """Base class for all tweetstream errors""" - pass - - -class AuthenticationError(TweetStreamError): - """Exception raised if the username/password is not accepted""" - pass - - -class ConnectionError(TweetStreamError): - """Raised when there are network problems. This means when there are - dns errors, network errors, twitter issues""" - - def __init__(self, reason, details=None): - self.reason = reason - self.details = details - - def __str__(self): - return '' % self.reason - - -from .streamclasses import SampleStream, FilterStream -from .deprecated import FollowStream, TrackStream, LocationStream, TweetStream, ReconnectingTweetStream diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/tweetstream/deprecated.py --- a/script/lib/tweetstream/tweetstream/deprecated.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -from .streamclasses import FilterStream, SampleStream, ConnectionError -import time - -class DeprecatedStream(FilterStream): - def __init__(self, *args, **kwargs): - import warnings - warnings.warn("%s is deprecated. Use FilterStream instead" % self.__class__.__name__, DeprecationWarning) - super(DeprecatedStream, self).__init__(*args, **kwargs) - - -class FollowStream(DeprecatedStream): - def __init__(self, auth, follow, catchup=None, url=None): - super(FollowStream, self).__init__(auth, follow=follow, catchup=catchup, url=url) - - -class TrackStream(DeprecatedStream): - def __init__(self, auth, track, catchup=None, url=None, slow=False): - super(TrackStream, self).__init__(auth, track=track, catchup=catchup, url=url) - - -class LocationStream(DeprecatedStream): - def __init__(self, auth, locations, catchup=None, url=None, slow=False): - super(LocationStream, self).__init__(auth, locations=locations, catchup=catchup, url=url) - - -class TweetStream(SampleStream): - def __init__(self, *args, **kwargs): - import warnings - warnings.warn("%s is deprecated. Use SampleStream instead" % self.__class__.__name__, DeprecationWarning) - SampleStream.__init__(self, *args, **kwargs) - - -class ReconnectingTweetStream(TweetStream): - """TweetStream class that automatically tries to reconnect if the - connecting goes down. Reconnecting, and waiting for reconnecting, is - blocking. - - :param username: See :TweetStream: - - :param password: See :TweetStream: - - :keyword url: See :TweetStream: - - :keyword reconnects: Number of reconnects before a ConnectionError is - raised. Default is 3 - - :error_cb: Optional callable that will be called just before trying to - reconnect. The callback will be called with a single argument, the - exception that caused the reconnect attempt. Default is None - - :retry_wait: Time to wait before reconnecting in seconds. Default is 5 - - """ - - def __init__(self, auth, url="sample", - reconnects=3, error_cb=None, retry_wait=5): - self.max_reconnects = reconnects - self.retry_wait = retry_wait - self._reconnects = 0 - self._error_cb = error_cb - TweetStream.__init__(self, auth, url=url) - - def next(self): - while True: - try: - return TweetStream.next(self) - except ConnectionError, e: - self._reconnects += 1 - if self._reconnects > self.max_reconnects: - raise ConnectionError("Too many retries") - - # Note: error_cb is not called on the last error since we - # raise a ConnectionError instead - if callable(self._error_cb): - self._error_cb(e) - - time.sleep(self.retry_wait) - # Don't listen to auth error, since we can't reasonably reconnect - # when we get one. - - - diff -r 41ce1c341abe -r 10a19dd4e1c9 script/lib/tweetstream/tweetstream/streamclasses.py --- a/script/lib/tweetstream/tweetstream/streamclasses.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,256 +0,0 @@ -import time -import urllib -import urllib2 -import socket -from platform import python_version_tuple -import anyjson - -from . import AuthenticationError, ConnectionError, USER_AGENT - -class BaseStream(object): - """A network connection to Twitters streaming API - - :param auth: tweepy auth object. - :keyword catchup: Number of tweets from the past to get before switching to - live stream. - :keyword raw: If True, return each tweet's raw data direct from the socket, - without UTF8 decoding or parsing, rather than a parsed object. The - default is False. - :keyword timeout: If non-None, set a timeout in seconds on the receiving - socket. Certain types of network problems (e.g., disconnecting a VPN) - can cause the connection to hang, leading to indefinite blocking that - requires kill -9 to resolve. Setting a timeout leads to an orderly - shutdown in these cases. The default is None (i.e., no timeout). - :keyword url: Endpoint URL for the object. Note: you should not - need to edit this. It's present to make testing easier. - - .. attribute:: connected - - True if the object is currently connected to the stream. - - .. attribute:: url - - The URL to which the object is connected - - .. attribute:: starttime - - The timestamp, in seconds since the epoch, the object connected to the - streaming api. - - .. attribute:: count - - The number of tweets that have been returned by the object. - - .. attribute:: rate - - The rate at which tweets have been returned from the object as a - float. see also :attr: `rate_period`. - - .. attribute:: rate_period - - The ammount of time to sample tweets to calculate tweet rate. By - default 10 seconds. Changes to this attribute will not be reflected - until the next time the rate is calculated. The rate of tweets vary - with time of day etc. so it's usefull to set this to something - sensible. - - .. attribute:: user_agent - - User agent string that will be included in the request. NOTE: This can - not be changed after the connection has been made. This property must - thus be set before accessing the iterator. The default is set in - :attr: `USER_AGENT`. - """ - - def __init__(self, auth, - catchup=None, raw=False, timeout=None, url=None): - self._conn = None - self._rate_ts = None - self._rate_cnt = 0 - self._auth = auth - self._catchup_count = catchup - self._raw_mode = raw - self._timeout = timeout - self._iter = self.__iter__() - - self.rate_period = 10 # in seconds - self.connected = False - self.starttime = None - self.count = 0 - self.rate = 0 - self.user_agent = USER_AGENT - if url: self.url = url - - self.muststop = False - - def __enter__(self): - return self - - def __exit__(self, *params): - self.close() - return False - - def _init_conn(self): - """Open the connection to the twitter server""" - headers = {'User-Agent': self.user_agent} - - postdata = self._get_post_data() or {} - if self._catchup_count: - postdata["count"] = self._catchup_count - - poststring = urllib.urlencode(postdata) if postdata else None - - if self._auth: - self._auth.apply_auth(self.url, "POST", headers, postdata) - - req = urllib2.Request(self.url, poststring, headers) - - try: - self._conn = urllib2.urlopen(req, timeout=self._timeout) - - except urllib2.HTTPError, exception: - if exception.code == 401: - raise AuthenticationError("Access denied") - elif exception.code == 404: - raise ConnectionError("URL not found: %s" % self.url) - else: # re raise. No idea what would cause this, so want to know - raise - except urllib2.URLError, exception: - raise ConnectionError(exception.reason) - - # This is horrible. This line grabs the raw socket (actually an ssl - # wrapped socket) from the guts of urllib2/httplib. We want the raw - # socket so we can bypass the buffering that those libs provide. - # The buffering is reasonable when dealing with connections that - # try to finish as soon as possible. With twitters' never ending - # connections, it causes a bug where we would not deliver tweets - # until the buffer was full. That's problematic for very low volume - # filterstreams, since you might not see a tweet for minutes or hours - # after they occured while the buffer fills. - # - # Oh, and the inards of the http libs are different things on in - # py2 and 3, so need to deal with that. py3 libs do more of what I - # want by default, but I wont do more special casing for it than - # neccessary. - - major, _, _ = python_version_tuple() - # The cast is needed because apparently some versions return strings - # and some return ints. - # On my ubuntu with stock 2.6 I get strings, which match the docs. - # Someone reported the issue on 2.6.1 on macos, but that was - # manually built, not the bundled one. Anyway, cast for safety. - major = int(major) - if major == 2: - self._socket = self._conn.fp._sock.fp._sock - else: - self._socket = self._conn.fp.raw - # our code that reads from the socket expects a method called recv. - # py3 socket.SocketIO uses the name read, so alias it. - self._socket.recv = self._socket.read - - self.connected = True - if not self.starttime: - self.starttime = time.time() - if not self._rate_ts: - self._rate_ts = time.time() - - def _get_post_data(self): - """Subclasses that need to add post data to the request can override - this method and return post data. The data should be in the format - returned by urllib.urlencode.""" - return None - - def __muststop(self): - if callable(self.muststop): - return self.muststop() - else: - return self.muststop - - def _update_rate(self): - rate_time = time.time() - self._rate_ts - if not self._rate_ts or rate_time > self.rate_period: - self.rate = self._rate_cnt / rate_time - self._rate_cnt = 0 - self._rate_ts = time.time() - - def __iter__(self): - buf = b"" - while True: - try: - if self.__muststop(): - raise StopIteration() - - if not self.connected: - self._init_conn() - - buf += self._socket.recv(8192) - if buf == b"": # something is wrong - self.close() - raise ConnectionError("Got entry of length 0. Disconnected") - elif buf.isspace(): - buf = b"" - elif b"\r" not in buf: # not enough data yet. Loop around - continue - - lines = buf.split(b"\r") - buf = lines[-1] - lines = lines[:-1] - - for line in lines: - if (self._raw_mode): - tweet = line - else: - line = line.decode("utf8") - try: - tweet = anyjson.deserialize(line) - except ValueError, e: - self.close() - raise ConnectionError("Got invalid data from twitter", details=line) - - if 'text' in tweet: - self.count += 1 - self._rate_cnt += 1 - self._update_rate() - yield tweet - - - except socket.error, e: - self.close() - raise ConnectionError("Server disconnected") - - - def next(self): - """Return the next available tweet. This call is blocking!""" - return self._iter.next() - - - def close(self): - """ - Close the connection to the streaming server. - """ - self.connected = False - if self._conn: - self._conn.close() - - -class SampleStream(BaseStream): - url = "https://stream.twitter.com/1/statuses/sample.json" - - -class FilterStream(BaseStream): - url = "https://stream.twitter.com/1/statuses/filter.json" - - def __init__(self, auth, follow=None, locations=None, - track=None, catchup=None, url=None, raw=False, timeout=None): - self._follow = follow - self._locations = locations - self._track = track - # remove follow, locations, track - BaseStream.__init__(self, auth, url=url, raw=raw, catchup=catchup, timeout=timeout) - - def _get_post_data(self): - postdata = {} - if self._follow: postdata["follow"] = ",".join([str(e) for e in self._follow]) - if self._locations: postdata["locations"] = ",".join(self._locations) - if self._track: postdata["track"] = ",".join(self._track) - return postdata diff -r 41ce1c341abe -r 10a19dd4e1c9 script/rest/enmi_profile.py --- a/script/rest/enmi_profile.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,134 +0,0 @@ -import twython -from sqlite3 import * -import datetime, time -import email.utils -from optparse import OptionParser -import os.path -import os -import sys -import simplejson - - -#options filename rpp page total_pages start_date end_date - - - -def adapt_datetime(ts): - return time.mktime(ts.timetuple()) - -def adapt_geo(geo): - return simplejson.dumps(geo) - -def convert_geo(s): - return simplejson.loads(s) - - -register_adapter(datetime.datetime, adapt_datetime) -register_converter("geo", convert_geo) - -columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user'] -columns_user = [u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'geo_enabled', u'profile_background_image_url', u'screen_name', u'profile_background_tile', u'favourites_count', u'name', u'url', u'created_at', u'time_zone', u'profile_sidebar_border_color', u'following'] - -def processDate(entry): - ts = email.utils.parsedate(entry["created_at"]) - entry["created_at_ts"] = datetime.datetime.fromtimestamp(time.mktime(ts)) - -def processPage(page, cursor, debug): - for entry in page: - if debug: - print "ENTRY : " + repr(entry) - curs.execute("select id from tweet_tweet where id = ?", (entry["id"],)) - res = curs.fetchone() - if res: - continue - - entry_user = entry["user"] - processDate(entry_user) - cursor.execute("insert into tweet_user ("+",".join(entry_user.keys())+") values (:"+",:".join(entry_user.keys())+");", entry_user); - new_id = cursor.lastrowid - processDate(entry) - entry["user"] = new_id - if entry["geo"]: - entry["geo"] = adapt_geo(entry["geo"]) - new_id = cursor.execute("insert into tweet_tweet ("+",".join(entry.keys())+") values (:"+",:".join(entry.keys())+");", entry); - - -if __name__ == "__main__" : - - parser = OptionParser() - parser.add_option("-f", "--file", dest="filename", - help="write tweet to FILE", metavar="FILE", default="enmi2010_twitter_rest.db") - parser.add_option("-r", "--rpp", dest="rpp", - help="Results per page", metavar="RESULT_PER_PAGE", default=200, type='int') - parser.add_option("-p", "--page", dest="page", - help="page result", metavar="PAGE", default=1, type='int') - parser.add_option("-t", "--total-page", dest="total_page", - help="Total page number", metavar="TOTAL_PAGE", default=16, type='int') - parser.add_option("-s", "--screenname", dest="screen_name", - help="Twitter screen name", metavar="SCREEN_NAME") - parser.add_option("-u", "--user", dest="username", - help="Twitter user", metavar="USER", default=None) - parser.add_option("-w", "--password", dest="password", - help="Twitter password", metavar="PASSWORD", default=None) - parser.add_option("-n", "--new", dest="new", action="store_true", - help="new database", default=False) - parser.add_option("-d", "--debug", dest="debug", action="store_true", - help="debug", default=False) - - - - (options, args) = parser.parse_args() - - if options.debug: - print "OPTIONS : " - print repr(options) - - if options.screen_name is None: - print "No Screen name. Exiting" - sys.exit() - - if options.new and os.path.exists(options.filename): - os.remove(options.filename) - - conn = connect(options.filename) - conn.row_factory = Row - curs = conn.cursor() - - curs.execute("create table if not exists tweet_user ("+ ",".join(columns_user) +", created_at_ts integer);") - - curs.execute("create table if not exists tweet_tweet ("+ ",".join(columns_tweet) +", created_at_ts integer);") - curs.execute("create index if not exists id_index on tweet_tweet (id asc);"); - - curs.execute("select count(*) from tweet_tweet;") - res = curs.fetchone() - - old_total = res[0] - - twitter = twython.setup(username=options.username, password=options.password, headers="IRI enmi (python urllib)") - twitter = twython.Twython(twitter_token = "54ThDZhpEjokcMgHJOMnQA", twitter_secret = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA") - - search_results = None - page = options.page-1 - - while (page < options.total_page and ( search_results is None or len(search_results) > 0)): - page += 1 - try: - search_results = twitter.getUserTimeline(screen_name=options.screen_name, count=options.rpp, page=page) - except twython.TwythonError, (e): - print "NAME : "+ options.screen_name + " ERROR : " + repr(e.msg) - break - print "NAME : "+ options.screen_name +" PAGE : " + repr(page) + " tweet: " + repr(len(search_results)) + " (total page : " + unicode(options.total_page) + " : rpp : "+unicode(options.rpp)+")" - processPage(search_results, curs, options.debug) - - conn.commit() - - curs.execute("select count(*) from tweet_tweet;") - res = curs.fetchone() - - total = res[0] - - print "Tweet for " + options.screen_name + " : " + unicode(total - old_total) +", Tweet total : " + repr(total) - - conn.close() - - diff -r 41ce1c341abe -r 10a19dd4e1c9 script/rest/export_twitter.py --- a/script/rest/export_twitter.py Tue May 07 18:28:26 2013 +0200 +++ b/script/rest/export_twitter.py Fri May 10 13:34:40 2013 +0200 @@ -1,16 +1,15 @@ #!/usr/bin/env python # coding=utf-8 -from sqlite3 import * +from sqlite3 import register_adapter, register_converter, connect, Row import datetime, time import email.utils from optparse import OptionParser import os.path -import os -import sys from lxml import etree import uuid import re +import simplejson def parse_date(date_str): ts = email.utils.parsedate_tz(date_str) @@ -20,10 +19,10 @@ return time.mktime(ts.timetuple()) def adapt_geo(geo): - return simplejson.dumps(geo) - + return simplejson.dumps(geo) + def convert_geo(s): - return simplejson.loads(s) + return simplejson.loads(s) register_adapter(datetime.datetime, adapt_datetime) @@ -73,7 +72,7 @@ ts = int(parse_date(options.start_date)) if options.end_date: - te = int(parse_date(options.end_date)) + te = int(parse_date(options.end_date)) else: te = ts + options.duration diff -r 41ce1c341abe -r 10a19dd4e1c9 script/rest/getscreennames.py --- a/script/rest/getscreennames.py Tue May 07 18:28:26 2013 +0200 +++ b/script/rest/getscreennames.py Fri May 10 13:34:40 2013 +0200 @@ -1,11 +1,5 @@ -from sqlite3 import * -import datetime, time -import email.utils from optparse import OptionParser -import os.path -import os -import sys -import simplejson +from sqlite3 import connect, Row import re if __name__ == "__main__" : diff -r 41ce1c341abe -r 10a19dd4e1c9 script/rest/search_twitter.py --- a/script/rest/search_twitter.py Tue May 07 18:28:26 2013 +0200 +++ b/script/rest/search_twitter.py Fri May 10 13:34:40 2013 +0200 @@ -1,10 +1,8 @@ -from iri_tweet import models, utils -from sqlalchemy.orm import sessionmaker +from iri_tweet import models, processor +from optparse import OptionParser import anyjson -import sqlite3 +import re import twitter -import re -from optparse import OptionParser def get_option(): @@ -59,7 +57,7 @@ print tweet tweet_str = anyjson.serialize(tweet) #invalidate user id - processor = utils.TwitterProcessor(tweet, tweet_str, None, session, None, options.token_filename) + processor = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=None, access_token=None, token_filename=options.token_filename, user_query_twitter=False, logger=None) processor.process() session.flush() session.commit() diff -r 41ce1c341abe -r 10a19dd4e1c9 script/stream/recorder_stream.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/stream/recorder_stream.py Fri May 10 13:34:40 2013 +0200 @@ -0,0 +1,603 @@ +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +from iri_tweet import models, utils +from iri_tweet.models import TweetSource, TweetLog, ProcessEvent +from iri_tweet.processor import get_processor +from multiprocessing import Queue as mQueue, Process, Event +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import scoped_session +import Queue +import StringIO +import anyjson +import argparse +import datetime +import inspect +import iri_tweet.stream +import logging +import os +import re +import requests_oauthlib +import shutil +import signal +import socket +import sqlalchemy.schema +import sys +import thread +import threading +import time +import traceback +import urllib2 +socket._fileobject.default_bufsize = 0 + + + +# columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user'] +columns_tweet = [u'user', u'favorited', u'contributors', u'truncated', u'text', u'created_at', u'retweeted', u'in_reply_to_status_id_str', u'coordinates', u'in_reply_to_user_id_str', u'entities', u'in_reply_to_status_id', u'place', u'in_reply_to_user_id', u'id', u'in_reply_to_screen_name', u'retweet_count', u'geo', u'id_str', u'source'] +# columns_user = [u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'geo_enabled', u'profile_background_image_url', u'screen_name', u'profile_background_tile', u'favourites_count', u'name', u'url', u'created_at', u'time_zone', u'profile_sidebar_border_color', u'following'] +columns_user = [u'follow_request_sent', u'profile_use_background_image', u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'id_str', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'show_all_inline_media', u'geo_enabled', u'profile_background_image_url', u'name', u'lang', u'following', u'profile_background_tile', u'favourites_count', u'screen_name', u'url', u'created_at', u'contributors_enabled', u'time_zone', u'profile_sidebar_border_color', u'is_translator', u'listed_count'] +# just put it in a sqlite3 tqble + +DEFAULT_TIMEOUT = 3 + +class Requesthandler(BaseHTTPRequestHandler): + + def __init__(self, request, client_address, server): + BaseHTTPRequestHandler.__init__(self, request, client_address, server) + + def do_GET(self): + self.send_response(200) + self.end_headers() + + def log_message(self, format, *args): # @ReservedAssignment + pass + + +def set_logging(options): + loggers = [] + + loggers.append(utils.set_logging(options, logging.getLogger('iri.tweet'))) + loggers.append(utils.set_logging(options, logging.getLogger('multiprocessing'))) + if options.debug >= 2: + loggers.append(utils.set_logging(options, logging.getLogger('sqlalchemy.engine'))) + # utils.set_logging(options, logging.getLogger('sqlalchemy.dialects')) + # utils.set_logging(options, logging.getLogger('sqlalchemy.pool')) + # utils.set_logging(options, logging.getLogger('sqlalchemy.orm')) + return loggers + +def set_logging_process(options, queue): + qlogger = utils.set_logging(options, logging.getLogger('iri.tweet.p'), queue) + qlogger.propagate = 0 + return qlogger + +def get_auth(options, access_token): + consumer_key = options.consumer_key + consumer_secret = options.consumer_secret + auth = requests_oauthlib.OAuth1(client_key=consumer_key, client_secret=consumer_secret, resource_owner_key=access_token[0], resource_owner_secret=access_token[1], signature_type='auth_header') + return auth + + +def add_process_event(event_type, args, session_maker): + session = session_maker() + try: + evt = ProcessEvent(args=None if args is None else anyjson.serialize(args), type=event_type) + session.add(evt) + session.commit() + finally: + session.close() + + +class BaseProcess(Process): + + def __init__(self, session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid): + self.parent_pid = parent_pid + self.session_maker = session_maker + self.queue = queue + self.options = options + self.logger_queue = logger_queue + self.stop_event = stop_event + self.consumer_token = (options.consumer_key, options.consumer_secret) + self.access_token = access_token + + super(BaseProcess, self).__init__() + + # + # from http://stackoverflow.com/questions/2542610/python-daemon-doesnt-kill-its-kids + # + def parent_is_alive(self): + try: + # try to call Parent + os.kill(self.parent_pid, 0) + except OSError: + # *beeep* oh no! The phone's disconnected! + return False + else: + # *ring* Hi mom! + return True + + + def __get_process_event_args(self): + return {'name':self.name, 'pid':self.pid, 'parent_pid':self.parent_pid, 'options':self.options.__dict__, 'access_token':self.access_token} + + def run(self): + try: + add_process_event("start_worker", self.__get_process_event_args(), self.session_maker) + self.do_run() + finally: + add_process_event("stop_worker", self.__get_process_event_args(), self.session_maker) + + def do_run(self): + raise NotImplementedError() + + + +class SourceProcess(BaseProcess): + + def __init__(self, session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid): + self.track = options.track + self.token_filename = options.token_filename + self.timeout = options.timeout + self.stream = None + super(SourceProcess, self).__init__(session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid) + + def __source_stream_iter(self): + + self.logger.debug("SourceProcess : run ") + + self.logger.debug("SourceProcess : get_auth auth with option %s and token %s " %(self.options, self.access_token)) + self.auth = get_auth(self.options, self.access_token) + self.logger.debug("SourceProcess : auth set ") + + track_list = self.track # or raw_input('Keywords to track (comma seperated): ').strip() + self.logger.debug("SourceProcess : track list " + track_list) + + track_list = [k.strip() for k in track_list.split(',')] + + self.logger.debug("SourceProcess : before connecting to stream %s, url : %s, auth : %s" % (repr(track_list), self.options.url, repr(self.auth))) + self.stream = iri_tweet.stream.FilterStream(self.auth, track=track_list, raw=True, url=self.options.url, timeout=self.timeout, chunk_size=512, logger=self.logger) + self.logger.debug("SourceProcess : after connecting to stream") + self.stream.muststop = lambda: self.stop_event.is_set() + + stream_wrapper = iri_tweet.stream.SafeStreamWrapper(self.stream, logger=self.logger) + + session = self.session_maker() + + #import pydevd + #pydevd.settrace(suspend=False) + + + try: + for tweet in stream_wrapper: + if not self.parent_is_alive(): + self.stop_event.set() + sys.exit() + self.logger.debug("SourceProcess : tweet " + repr(tweet)) + source = TweetSource(original_json=tweet) + self.logger.debug("SourceProcess : source created") + add_retries = 0 + while add_retries < 10: + try: + add_retries += 1 + session.add(source) + session.flush() + break + except OperationalError as e: + session.rollback() + self.logger.debug("SourceProcess : Operational Error %s nb %d" % (repr(e), add_retries)) + if add_retries == 10: + raise + + source_id = source.id + self.logger.debug("SourceProcess : before queue + source id " + repr(source_id)) + self.logger.info("SourceProcess : Tweet count: %d - current rate : %.2f - running : %s" % (self.stream.count, self.stream.rate, int(time.time() - self.stream.starttime))) + session.commit() + self.queue.put((source_id, tweet), False) + + except Exception as e: + self.logger.error("SourceProcess : Error when processing tweet " + repr(e)) + raise + finally: + session.rollback() + session.close() + self.stream.close() + self.stream = None + if not self.stop_event.is_set(): + self.stop_event.set() + + + def do_run(self): + + self.logger = set_logging_process(self.options, self.logger_queue) + + source_stream_iter_thread = threading.Thread(target=self.__source_stream_iter , name="SourceStreamIterThread") + + source_stream_iter_thread.start() + + try: + while not self.stop_event.is_set(): + self.logger.debug("SourceProcess : In while after start") + self.stop_event.wait(DEFAULT_TIMEOUT) + except KeyboardInterrupt: + self.stop_event.set() + pass + + if self.stop_event.is_set() and self.stream: + self.stream.close() + elif not self.stop_event.is_set() and not source_stream_iter_thread.is_alive: + self.stop_event.set() + + self.queue.cancel_join_thread() + self.logger_queue.cancel_join_thread() + self.logger.info("SourceProcess : join") + source_stream_iter_thread.join(30) + + +def process_tweet(tweet, source_id, session, consumer_token, access_token, twitter_query_user, token_filename, logger): + try: + if not tweet.strip(): + return + tweet_obj = anyjson.deserialize(tweet) + processor_klass = get_processor(tweet_obj) + if not processor_klass: + tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['NOT_TWEET']) + session.add(tweet_log) + return + processor = processor_klass(json_dict=tweet_obj, + json_txt=tweet, + source_id=source_id, + session=session, + consumer_token=consumer_token, + access_token=access_token, + token_filename=token_filename, + user_query_twitter=twitter_query_user, + logger=logger) + logger.info(processor.log_info()) + logger.debug(u"Process_tweet :" + repr(tweet)) + processor.process() + + except ValueError as e: + message = u"Value Error %s processing tweet %s" % (repr(e), tweet) + output = StringIO.StringIO() + try: + traceback.print_exc(file=output) + error_stack = output.getvalue() + finally: + output.close() + tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['NOT_TWEET'], error=message, error_stack=error_stack) + session.add(tweet_log) + session.commit() + except Exception as e: + message = u"Error %s processing tweet %s" % (repr(e), tweet) + logger.exception(message) + output = StringIO.StringIO() + try: + traceback.print_exc(file=output) + error_stack = output.getvalue() + finally: + output.close() + session.rollback() + tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['ERROR'], error=message, error_stack=error_stack) + session.add(tweet_log) + session.commit() + + + +class TweetProcess(BaseProcess): + + def __init__(self, session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid): + super(TweetProcess, self).__init__(session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid) + self.twitter_query_user = options.twitter_query_user + + + def do_run(self): + + self.logger = set_logging_process(self.options, self.logger_queue) + session = self.session_maker() + try: + while not self.stop_event.is_set() and self.parent_is_alive(): + try: + source_id, tweet_txt = self.queue.get(True, 3) + self.logger.debug("Processing source id " + repr(source_id)) + except Exception as e: + self.logger.debug('Process tweet exception in loop : ' + repr(e)) + continue + process_tweet(tweet_txt, source_id, session, self.consumer_token, self.access_token, self.twitter_query_user, self.options.token_filename, self.logger) + session.commit() + except KeyboardInterrupt: + self.stop_event.set() + finally: + session.rollback() + session.close() + + +def get_sessionmaker(conn_str): + engine, metadata, Session = models.setup_database(conn_str, echo=False, create_all=False, autocommit=False) + Session = scoped_session(Session) + return Session, engine, metadata + + +def process_leftovers(session, consumer_token, access_token, twitter_query_user, token_filename, ask_process_leftovers, logger): + + sources = session.query(TweetSource).outerjoin(TweetLog).filter(TweetLog.id == None) + sources_count = sources.count() + + if sources_count > 10 and ask_process_leftovers: + resp = raw_input("Do you want to process leftovers (Y/n) ? (%d tweet to process)" % sources_count) + if resp and resp.strip().lower() == "n": + return + logger.info("Process leftovers, %d tweets to process" % (sources_count)) + for src in sources: + tweet_txt = src.original_json + process_tweet(tweet_txt, src.id, session, consumer_token, access_token, twitter_query_user, token_filename, logger) + session.commit() + + +def process_log(logger_queues, stop_event): + while not stop_event.is_set(): + for lqueue in logger_queues: + try: + record = lqueue.get_nowait() + logging.getLogger(record.name).handle(record) + except Queue.Empty: + continue + except IOError: + continue + time.sleep(0.1) + + +def get_options(): + + usage = "usage: %(prog)s [options]" + + parser = argparse.ArgumentParser(usage=usage) + + parser.add_argument("-f", "--file", dest="conn_str", + help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR", default="enmi2010_twitter.db") + parser.add_argument("-T", "--track", dest="track", + help="Twitter track", metavar="TRACK") + parser.add_argument("-k", "--key", dest="consumer_key", + help="Twitter consumer key", metavar="CONSUMER_KEY", required=True) + parser.add_argument("-s", "--secret", dest="consumer_secret", + help="Twitter consumer secret", metavar="CONSUMER_SECRET", required=True) + parser.add_argument("-n", "--new", dest="new", action="store_true", + help="new database", default=False) + parser.add_argument("-D", "--daemon", dest="daemon", action="store_true", + help="launch daemon", default=False) + parser.add_argument("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", + help="Token file name") + parser.add_argument("-d", "--duration", dest="duration", + help="Duration of recording in seconds", metavar="DURATION", default= -1, type=int) + parser.add_argument("-N", "--nb-process", dest="process_nb", + help="number of process.\nIf 0, only the lefovers of the database are processed.\nIf 1, no postprocessing is done on the tweets.", metavar="PROCESS_NB", default=2, type=int) + parser.add_argument("--url", dest="url", + help="The twitter url to connect to.", metavar="URL", default=iri_tweet.stream.FilterStream.url) + parser.add_argument("--query-user", dest="twitter_query_user", action="store_true", + help="Query twitter for users", default=False) + parser.add_argument("--timeout", dest="timeout", + help="timeout for connecting in seconds", default=60, metavar="TIMEOUT", type=int) + parser.add_argument("--ask-process-leftovers", dest="ask_process_leftovers", action="store_false", + help="ask process leftover", default=True) + + + utils.set_logging_options(parser) + + return parser.parse_args() + + +def do_run(options, session_maker): + + stop_args = {} + + consumer_token = (options.consumer_key, options.consumer_secret) + access_token = utils.get_oauth_token(consumer_key=consumer_token[0], consumer_secret=consumer_token[1], token_file_path=options.token_filename) + + + session = session_maker() + try: + process_leftovers(session, consumer_token, access_token, options.twitter_query_user, options.token_filename, options.ask_process_leftovers, utils.get_logger()) + session.commit() + finally: + session.rollback() + session.close() + + if options.process_nb <= 0: + utils.get_logger().debug("Leftovers processed. Exiting.") + return None + + queue = mQueue() + stop_event = Event() + + # workaround for bug on using urllib2 and multiprocessing + httpd = HTTPServer(('127.0.0.1',0), Requesthandler) + thread.start_new_thread(httpd.handle_request, ()) + + req = urllib2.Request('http://localhost:%d' % httpd.server_port) + conn = None + try: + conn = urllib2.urlopen(req) + except: + utils.get_logger().debug("could not open localhost") + # donothing + finally: + if conn is not None: + conn.close() + + process_engines = [] + logger_queues = [] + + SessionProcess, engine_process, _ = get_sessionmaker(conn_str) + process_engines.append(engine_process) + lqueue = mQueue(50) + logger_queues.append(lqueue) + pid = os.getpid() + sprocess = SourceProcess(SessionProcess, queue, options, access_token, stop_event, lqueue, pid) + + tweet_processes = [] + + for i in range(options.process_nb - 1): + SessionProcess, engine_process, _ = get_sessionmaker(conn_str) + process_engines.append(engine_process) + lqueue = mQueue(50) + logger_queues.append(lqueue) + cprocess = TweetProcess(SessionProcess, queue, options, access_token, stop_event, lqueue, pid) + tweet_processes.append(cprocess) + + log_thread = threading.Thread(target=process_log, name="loggingThread", args=(logger_queues, stop_event,)) + log_thread.daemon = True + + log_thread.start() + + sprocess.start() + for cprocess in tweet_processes: + cprocess.start() + + add_process_event("pid", {'main':os.getpid(), 'source':(sprocess.name, sprocess.pid), 'consumers':dict([(p.name, p.pid) for p in tweet_processes])}, session_maker) + + if options.duration >= 0: + end_ts = datetime.datetime.utcnow() + datetime.timedelta(seconds=options.duration) + + def interupt_handler(signum, frame): + utils.get_logger().debug("shutdown asked " + repr(signum) + " " + repr(inspect.getframeinfo(frame, 9))) + stop_args.update({'message': 'interupt', 'signum':signum, 'frameinfo':inspect.getframeinfo(frame, 9)}) + stop_event.set() + + signal.signal(signal.SIGINT , interupt_handler) + signal.signal(signal.SIGHUP , interupt_handler) + signal.signal(signal.SIGALRM, interupt_handler) + signal.signal(signal.SIGTERM, interupt_handler) + + + while not stop_event.is_set(): + if options.duration >= 0 and datetime.datetime.utcnow() >= end_ts: + stop_args.update({'message': 'duration', 'duration' : options.duration, 'end_ts' : end_ts}) + stop_event.set() + break + if sprocess.is_alive(): + utils.get_logger().debug("Source process alive") + time.sleep(1) + else: + stop_args.update({'message': 'Source process killed'}) + stop_event.set() + break + utils.get_logger().debug("Joining Source Process") + try: + sprocess.join(10) + except: + utils.get_logger().debug("Pb joining Source Process - terminating") + finally: + sprocess.terminate() + + for i, cprocess in enumerate(tweet_processes): + utils.get_logger().debug("Joining consumer process Nb %d" % (i + 1)) + try: + cprocess.join(3) + except: + utils.get_logger().debug("Pb joining consumer process Nb %d - terminating" % (i + 1)) + cprocess.terminate() + + + utils.get_logger().debug("Close queues") + try: + queue.close() + for lqueue in logger_queues: + lqueue.close() + except Exception as e: + utils.get_logger().error("error when closing queues %s", repr(e)) + # do nothing + + + if options.process_nb > 1: + utils.get_logger().debug("Processing leftovers") + session = session_maker() + try: + process_leftovers(session, consumer_token, access_token, options.twitter_query_user, options.token_filename, options.ask_process_leftovers, utils.get_logger()) + session.commit() + finally: + session.rollback() + session.close() + + for pengine in process_engines: + pengine.dispose() + + return stop_args + + +def main(options): + + global conn_str + + conn_str = options.conn_str.strip() + if not re.match("^\w+://.+", conn_str): + conn_str = 'sqlite:///' + options.conn_str + + if conn_str.startswith("sqlite") and options.new: + filepath = conn_str[conn_str.find(":///") + 4:] + if os.path.exists(filepath): + i = 1 + basename, extension = os.path.splitext(filepath) + new_path = '%s.%d%s' % (basename, i, extension) + while i < 1000000 and os.path.exists(new_path): + i += 1 + new_path = '%s.%d%s' % (basename, i, extension) + if i >= 1000000: + raise Exception("Unable to find new filename for " + filepath) + else: + shutil.move(filepath, new_path) + + Session, engine, metadata = get_sessionmaker(conn_str) + + if options.new: + check_metadata = sqlalchemy.schema.MetaData(bind=engine) + check_metadata.reflect() + if len(check_metadata.sorted_tables) > 0: + message = "Database %s not empty exiting" % conn_str + utils.get_logger().error(message) + sys.exit(message) + + metadata.create_all(engine) + session = Session() + try: + models.add_model_version(session) + finally: + session.close() + + stop_args = {} + try: + add_process_event(event_type="start", args={'options':options.__dict__, 'args': [], 'command_line': sys.argv}, session_maker=Session) + stop_args = do_run(options, Session) + except Exception as e: + utils.get_logger().exception("Error in main thread") + outfile = StringIO.StringIO() + try: + traceback.print_exc(file=outfile) + stop_args = {'error': repr(e), 'message': getattr(e, 'message', ''), 'stacktrace':outfile.getvalue()} + finally: + outfile.close() + raise + finally: + add_process_event(event_type="shutdown", args=stop_args, session_maker=Session) + + utils.get_logger().debug("Done. Exiting. " + repr(stop_args)) + + + +if __name__ == '__main__': + + options = get_options() + + loggers = set_logging(options) + + utils.get_logger().debug("OPTIONS : " + repr(options)) + + if options.daemon: + options.ask_process_leftovers = False + import daemon + + hdlr_preserve = [] + for logger in loggers: + hdlr_preserve.extend([h.stream for h in logger.handlers]) + + context = daemon.DaemonContext(working_directory=os.getcwd(), files_preserve=hdlr_preserve) + with context: + main(options) + else: + main(options) + diff -r 41ce1c341abe -r 10a19dd4e1c9 script/stream/recorder_tweetstream.py --- a/script/stream/recorder_tweetstream.py Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,528 +0,0 @@ -from getpass import getpass -from iri_tweet import models, utils -from iri_tweet.models import TweetSource, TweetLog, ProcessEvent -from multiprocessing import (Queue as mQueue, JoinableQueue, Process, Event, - get_logger) -from optparse import OptionParser -from sqlalchemy.exc import OperationalError -from sqlalchemy.orm import scoped_session -import Queue -import StringIO -import anyjson -import datetime -import inspect -import logging -import os -import re -import shutil -import signal -import socket -import sqlalchemy.schema -import sys -import threading -import time -import traceback -import tweepy.auth -import tweetstream -import urllib2 -socket._fileobject.default_bufsize = 0 - - - -#columns_tweet = [u'favorited', u'truncated', u'text', u'created_at', u'source', u'in_reply_to_status_id', u'in_reply_to_screen_name', u'in_reply_to_user_id', u'geo', u'id', u'user'] -columns_tweet = [u'user', u'favorited', u'contributors', u'truncated', u'text', u'created_at', u'retweeted', u'in_reply_to_status_id_str', u'coordinates', u'in_reply_to_user_id_str', u'entities', u'in_reply_to_status_id', u'place', u'in_reply_to_user_id', u'id', u'in_reply_to_screen_name', u'retweet_count', u'geo', u'id_str', u'source'] -#columns_user = [u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'geo_enabled', u'profile_background_image_url', u'screen_name', u'profile_background_tile', u'favourites_count', u'name', u'url', u'created_at', u'time_zone', u'profile_sidebar_border_color', u'following'] -columns_user = [u'follow_request_sent', u'profile_use_background_image', u'id', u'verified', u'profile_sidebar_fill_color', u'profile_text_color', u'followers_count', u'protected', u'location', u'profile_background_color', u'id_str', u'utc_offset', u'statuses_count', u'description', u'friends_count', u'profile_link_color', u'profile_image_url', u'notifications', u'show_all_inline_media', u'geo_enabled', u'profile_background_image_url', u'name', u'lang', u'following', u'profile_background_tile', u'favourites_count', u'screen_name', u'url', u'created_at', u'contributors_enabled', u'time_zone', u'profile_sidebar_border_color', u'is_translator', u'listed_count'] -#just put it in a sqlite3 tqble - - -def set_logging(options): - loggers = [] - - loggers.append(utils.set_logging(options, logging.getLogger('iri.tweet'))) - loggers.append(utils.set_logging(options, logging.getLogger('multiprocessing'))) - if options.debug >= 2: - loggers.append(utils.set_logging(options, logging.getLogger('sqlalchemy.engine'))) - #utils.set_logging(options, logging.getLogger('sqlalchemy.dialects')) - #utils.set_logging(options, logging.getLogger('sqlalchemy.pool')) - #utils.set_logging(options, logging.getLogger('sqlalchemy.orm')) - return loggers - -def set_logging_process(options, queue): - qlogger = utils.set_logging(options, logging.getLogger('iri.tweet.p'), queue) - qlogger.propagate = 0 - return qlogger - -def get_auth(options, access_token): - if options.username and options.password: - auth = tweepy.auth.BasicAuthHandler(options.username, options.password) - else: - consumer_key = models.CONSUMER_KEY - consumer_secret = models.CONSUMER_SECRET - auth = tweepy.auth.OAuthHandler(consumer_key, consumer_secret, secure=False) - auth.set_access_token(*access_token) - return auth - - -def add_process_event(type, args, session_maker): - session = session_maker() - try: - evt = ProcessEvent(args=None if args is None else anyjson.serialize(args), type=type) - session.add(evt) - session.commit() - finally: - session.close() - - -class BaseProcess(Process): - - def __init__(self, session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid): - self.parent_pid = parent_pid - self.session_maker = session_maker - self.queue = queue - self.options = options - self.logger_queue = logger_queue - self.stop_event = stop_event - self.access_token = access_token - - super(BaseProcess, self).__init__() - - # - # from http://stackoverflow.com/questions/2542610/python-daemon-doesnt-kill-its-kids - # - def parent_is_alive(self): - try: - # try to call Parent - os.kill(self.parent_pid, 0) - except OSError: - # *beeep* oh no! The phone's disconnected! - return False - else: - # *ring* Hi mom! - return True - - - def __get_process_event_args(self): - return {'name':self.name, 'pid':self.pid, 'parent_pid':self.parent_pid, 'options':self.options.__dict__, 'access_token':self.access_token} - - def run(self): - try: - add_process_event("start_worker", self.__get_process_event_args(), self.session_maker) - self.do_run() - finally: - add_process_event("stop_worker", self.__get_process_event_args(), self.session_maker) - - def do_run(self): - raise NotImplementedError() - - - -class SourceProcess(BaseProcess): - - def __init__(self, session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid): - self.track = options.track - self.token_filename = options.token_filename - self.catchup = options.catchup - self.timeout = options.timeout - super(SourceProcess, self).__init__(session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid) - - def do_run(self): - - #import pydevd - #pydevd.settrace(suspend=True) - - self.logger = set_logging_process(self.options, self.logger_queue) - self.auth = get_auth(self.options, self.access_token) - - self.logger.debug("SourceProcess : run ") - track_list = self.track # or raw_input('Keywords to track (comma seperated): ').strip() - self.logger.debug("SourceProcess : track list " + track_list) - - track_list = [k.strip() for k in track_list.split(',')] - - self.logger.debug("SourceProcess : before connecting to stream " + repr(track_list)) - stream = tweetstream.FilterStream(self.auth, track=track_list, raw=True, url=self.options.url, catchup=self.catchup, timeout=self.timeout) - self.logger.debug("SourceProcess : after connecting to stream") - stream.muststop = lambda: self.stop_event.is_set() - - session = self.session_maker() - - try: - for tweet in stream: - if not self.parent_is_alive(): - sys.exit() - self.logger.debug("SourceProcess : tweet " + repr(tweet)) - source = TweetSource(original_json=tweet) - self.logger.debug("SourceProcess : source created") - add_retries = 0 - while add_retries < 10: - try: - add_retries += 1 - session.add(source) - session.flush() - break - except OperationalError as e: - session.rollback() - self.logger.debug("SourceProcess : Operational Error %s nb %d" % (repr(e), add_retries)) - if add_retries == 10: - raise e - - source_id = source.id - self.logger.debug("SourceProcess : before queue + source id " + repr(source_id)) - self.logger.info("SourceProcess : Tweet count: %d - current rate : %.2f - running : %s" % (stream.count, stream.rate, int(time.time() - stream.starttime))) - session.commit() - self.queue.put((source_id, tweet), False) - - except Exception as e: - self.logger.error("SourceProcess : Error when processing tweet " + repr(e)) - finally: - session.rollback() - stream.close() - session.close() - self.queue.close() - self.stop_event.set() - - -def process_tweet(tweet, source_id, session, access_token, twitter_query_user, logger): - try: - tweet_obj = anyjson.deserialize(tweet) - if 'text' not in tweet_obj: - tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['NOT_TWEET']) - session.add(tweet_log) - return - screen_name = "" - if 'user' in tweet_obj and 'screen_name' in tweet_obj['user']: - screen_name = tweet_obj['user']['screen_name'] - logger.info(u"Process_tweet from %s : %s" % (screen_name, tweet_obj['text'])) - logger.debug(u"Process_tweet :" + repr(tweet)) - processor = utils.TwitterProcessor(tweet_obj, tweet, source_id, session, access_token, None, twitter_query_user) - processor.process() - except Exception as e: - message = u"Error %s processing tweet %s" % (repr(e), tweet) - logger.exception(message) - output = StringIO.StringIO() - try: - traceback.print_exc(file=output) - error_stack = output.getvalue() - finally: - output.close() - session.rollback() - tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['ERROR'], error=message, error_stack=error_stack) - session.add(tweet_log) - session.commit() - - - -class TweetProcess(BaseProcess): - - def __init__(self, session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid): - super(TweetProcess, self).__init__(session_maker, queue, options, access_token, stop_event, logger_queue, parent_pid) - self.twitter_query_user = options.twitter_query_user - - - def do_run(self): - - self.logger = set_logging_process(self.options, self.logger_queue) - session = self.session_maker() - try: - while not self.stop_event.is_set() and self.parent_is_alive(): - try: - source_id, tweet_txt = self.queue.get(True, 3) - self.logger.debug("Processing source id " + repr(source_id)) - except Exception as e: - self.logger.debug('Process tweet exception in loop : ' + repr(e)) - continue - process_tweet(tweet_txt, source_id, session, self.access_token, self.twitter_query_user, self.logger) - session.commit() - finally: - session.rollback() - self.stop_event.set() - session.close() - - -def get_sessionmaker(conn_str): - engine, metadata, Session = models.setup_database(conn_str, echo=False, create_all=False, autocommit=False) - Session = scoped_session(Session) - return Session, engine, metadata - - -def process_leftovers(session, access_token, twitter_query_user, logger): - - sources = session.query(TweetSource).outerjoin(TweetLog).filter(TweetLog.id == None) - - for src in sources: - tweet_txt = src.original_json - process_tweet(tweet_txt, src.id, session, access_token, twitter_query_user, logger) - session.commit() - - - - #get tweet source that do not match any message - #select * from tweet_tweet_source ts left join tweet_tweet_log tl on ts.id = tl.tweet_source_id where tl.id isnull; -def process_log(logger_queues, stop_event): - while not stop_event.is_set(): - for lqueue in logger_queues: - try: - record = lqueue.get_nowait() - logging.getLogger(record.name).handle(record) - except Queue.Empty: - continue - except IOError: - continue - time.sleep(0.1) - - -def get_options(): - - usage = "usage: %prog [options]" - - parser = OptionParser(usage=usage) - - parser.add_option("-f", "--file", dest="conn_str", - help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR", default="enmi2010_twitter.db") - parser.add_option("-u", "--user", dest="username", - help="Twitter user", metavar="USER", default=None) - parser.add_option("-w", "--password", dest="password", - help="Twitter password", metavar="PASSWORD", default=None) - parser.add_option("-T", "--track", dest="track", - help="Twitter track", metavar="TRACK") - parser.add_option("-n", "--new", dest="new", action="store_true", - help="new database", default=False) - parser.add_option("-D", "--daemon", dest="daemon", action="store_true", - help="launch daemon", default=False) - parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", - help="Token file name") - parser.add_option("-d", "--duration", dest="duration", - help="Duration of recording in seconds", metavar="DURATION", default= -1, type='int') - parser.add_option("-N", "--nb-process", dest="process_nb", - help="number of process.\nIf 0, only the lefovers of the database are processed.\nIf 1, no postprocessing is done on the tweets.", metavar="PROCESS_NB", default=2, type='int') - parser.add_option("--url", dest="url", - help="The twitter url to connect to.", metavar="URL", default=tweetstream.FilterStream.url) - parser.add_option("--query-user", dest="twitter_query_user", action="store_true", - help="Query twitter for users", default=False, metavar="QUERY_USER") - parser.add_option("--catchup", dest="catchup", - help="catchup count for tweets", default=None, metavar="CATCHUP", type='int') - parser.add_option("--timeout", dest="timeout", - help="timeout for connecting in seconds", default=60, metavar="TIMEOUT", type='int') - - - - - utils.set_logging_options(parser) - - return parser.parse_args() - - -def do_run(options, session_maker): - - stop_args = {} - - access_token = None - if not options.username or not options.password: - access_token = utils.get_oauth_token(options.token_filename) - - session = session_maker() - try: - process_leftovers(session, access_token, options.twitter_query_user, utils.get_logger()) - session.commit() - finally: - session.rollback() - session.close() - - if options.process_nb <= 0: - utils.get_logger().debug("Leftovers processed. Exiting.") - return None - - queue = mQueue() - stop_event = Event() - - #workaround for bug on using urllib2 and multiprocessing - req = urllib2.Request('http://localhost') - conn = None - try: - conn = urllib2.urlopen(req) - except: - utils.get_logger().debug("could not open localhost") - #donothing - finally: - if conn is not None: - conn.close() - - process_engines = [] - logger_queues = [] - - SessionProcess, engine_process, metadata_process = get_sessionmaker(conn_str) - process_engines.append(engine_process) - lqueue = mQueue(1) - logger_queues.append(lqueue) - pid = os.getpid() - sprocess = SourceProcess(SessionProcess, queue, options, access_token, stop_event, lqueue, pid) - - tweet_processes = [] - - for i in range(options.process_nb - 1): - SessionProcess, engine_process, metadata_process = get_sessionmaker(conn_str) - process_engines.append(engine_process) - lqueue = mQueue(1) - logger_queues.append(lqueue) - cprocess = TweetProcess(SessionProcess, queue, options, access_token, stop_event, lqueue, pid) - tweet_processes.append(cprocess) - - def interupt_handler(signum, frame): - utils.get_logger().debug("shutdown asked " + repr(signum) + " " + repr(inspect.getframeinfo(frame, 9))) - stop_args.update({'message': 'interupt', 'signum':signum, 'frameinfo':inspect.getframeinfo(frame, 9)}) - stop_event.set() - - signal.signal(signal.SIGINT , interupt_handler) - signal.signal(signal.SIGHUP , interupt_handler) - signal.signal(signal.SIGALRM, interupt_handler) - signal.signal(signal.SIGTERM, interupt_handler) - - log_thread = threading.Thread(target=process_log, name="loggingThread", args=(logger_queues, stop_event,)) - log_thread.daemon = True - - log_thread.start() - - sprocess.start() - for cprocess in tweet_processes: - cprocess.start() - - add_process_event("pid", {'main':os.getpid(), 'source':(sprocess.name, sprocess.pid), 'consumers':dict([(p.name, p.pid) for p in tweet_processes])}, session_maker) - - if options.duration >= 0: - end_ts = datetime.datetime.utcnow() + datetime.timedelta(seconds=options.duration) - - - while not stop_event.is_set(): - if options.duration >= 0 and datetime.datetime.utcnow() >= end_ts: - stop_args.update({'message': 'duration', 'duration' : options.duration, 'end_ts' : end_ts}) - stop_event.set() - break - if sprocess.is_alive(): - time.sleep(1) - else: - stop_args.update({'message': 'Source process killed'}) - stop_event.set() - break - utils.get_logger().debug("Joining Source Process") - try: - sprocess.join(10) - except: - utils.get_logger().debug("Pb joining Source Process - terminating") - sprocess.terminate() - - for i, cprocess in enumerate(tweet_processes): - utils.get_logger().debug("Joining consumer process Nb %d" % (i + 1)) - try: - cprocess.join(3) - except: - utils.get_logger().debug("Pb joining consumer process Nb %d - terminating" % (i + 1)) - cprocess.terminate() - - - utils.get_logger().debug("Close queues") - try: - queue.close() - for lqueue in logger_queues: - lqueue.close() - except exception as e: - utils.get_logger().error("error when closing queues %s", repr(e)) - #do nothing - - - if options.process_nb > 1: - utils.get_logger().debug("Processing leftovers") - session = session_maker() - try: - process_leftovers(session, access_token, options.twitter_query_user, utils.get_logger()) - session.commit() - finally: - session.rollback() - session.close() - - for pengine in process_engines: - pengine.dispose() - - return stop_args - - -def main(options, args): - - global conn_str - - conn_str = options.conn_str.strip() - if not re.match("^\w+://.+", conn_str): - conn_str = 'sqlite:///' + options.conn_str - - if conn_str.startswith("sqlite") and options.new: - filepath = conn_str[conn_str.find(":///") + 4:] - if os.path.exists(filepath): - i = 1 - basename, extension = os.path.splitext(filepath) - new_path = '%s.%d%s' % (basename, i, extension) - while i < 1000000 and os.path.exists(new_path): - i += 1 - new_path = '%s.%d%s' % (basename, i, extension) - if i >= 1000000: - raise Exception("Unable to find new filename for " + filepath) - else: - shutil.move(filepath, new_path) - - Session, engine, metadata = get_sessionmaker(conn_str) - - if options.new: - check_metadata = sqlalchemy.schema.MetaData(bind=engine, reflect=True) - if len(check_metadata.sorted_tables) > 0: - message = "Database %s not empty exiting" % conn_str - utils.get_logger().error(message) - sys.exit(message) - - metadata.create_all(engine) - session = Session() - try: - models.add_model_version(session) - finally: - session.close() - - stop_args = {} - try: - add_process_event(type="start", args={'options':options.__dict__, 'args': args, 'command_line': sys.argv}, session_maker=Session) - stop_args = do_run(options, Session) - except Exception as e: - utils.get_logger().exception("Error in main thread") - outfile = StringIO.StringIO() - try: - traceback.print_exc(file=outfile) - stop_args = {'error': repr(e), 'message': getattr(e, 'message', ''), 'stacktrace':outfile.getvalue()} - finally: - outfile.close() - raise - finally: - add_process_event(type="shutdown", args=stop_args, session_maker=Session) - - utils.get_logger().debug("Done. Exiting. " + repr(stop_args)) - - - -if __name__ == '__main__': - - (options, args) = get_options() - - loggers = set_logging(options) - - utils.get_logger().debug("OPTIONS : " + repr(options)) - - if options.daemon: - import daemon - import lockfile - - hdlr_preserve = [] - for logger in loggers: - hdlr_preserve.extend([h.stream for h in logger.handlers]) - - context = daemon.DaemonContext(working_directory=os.getcwd(), files_preserve=hdlr_preserve) - with context: - main(options, args) - else: - main(options, args) - diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/export_pad.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/utils/export_pad.py Fri May 10 13:34:40 2013 +0200 @@ -0,0 +1,320 @@ +#!/usr/bin/env python +# coding=utf-8 + +from dateutil.parser import parse as parse_date +from iri_tweet.utils import set_logging_options, set_logging, get_logger +from lxml import etree +from optparse import OptionParser +import anyjson +import datetime +import functools +import httplib2 +import os.path +import requests +import sys +import time +import uuid + + +class EtherpadRequestException(Exception): + def __init__(self, original_resp): + super(EtherpadRequestException, self).__init__(original_resp["message"]) + self.status = original_resp["status"] + self.original_resp = original_resp + + +class EtherpadRequest(): + + def __init__(self, base_url, api_key): + self.base_url = base_url + self.api_key = api_key + self.__request = None + + def __getattr__(self, name): + return functools.partial(self.__action, name) + + def __action(self, action, **kwargs): + url = "%s/%s" % (self.base_url, action) + params = dict(kwargs) + params['apikey'] = self.api_key + + r = requests.get(url, params) + + resp = anyjson.deserialize(r.text) + + if resp["code"] == 0: + return resp["data"] + else: + raise EtherpadRequestException(resp) + + return resp + + def getRevisionsCount(self, padID): + f = self.__getattr__("getRevisionsCount") + res = f(padID=padID) + + return res["revisions"] + + def getPadUrl(self, padID): + + return "%s/%s" % (self.base_url,padID) + + + +def abort(message, parser): + if message is not None: + sys.stderr.write(message + "\n") + parser.print_help() + sys.exit(1) + +def get_options(): + + parser = OptionParser() + parser.add_option("-u", "--api-url", dest="api_url", + help="Base etherpad-lite api url", metavar="API_URL", default=None) + parser.add_option("-k", "--api-key", dest="api_key", + help="Base etherpad-lite api url", metavar="API_KEY", default=None) + parser.add_option("-p", "--pad-id", dest="pad_id", + help="pad id", metavar="PADID") + parser.add_option("-s", "--start-date", dest="start_date", + help="start date", metavar="START_DATE", default=None) + parser.add_option("-e", "--end-date", dest="end_date", + help="end date", metavar="END_DATE", default=None) + parser.add_option("-f", "--format", dest="format", type="choice", + help="format", metavar="FORMAT", choice=['html', 'text'], default='html') + parser.add_option("-I", "--content-file", dest="content_file", + help="Content file", metavar="CONTENT_FILE") + parser.add_option("-C", "--color", dest="color", + help="Color code", metavar="COLOR", default="16763904") + parser.add_option("-D", "--duration", dest="duration", type="int", + help="Duration", metavar="DURATION", default=None) + parser.add_option("-n", "--name", dest="name", + help="Cutting name", metavar="NAME", default=u"pads") + parser.add_option("-R", "--replace", dest="replace", action="store_true", + help="Replace tweet ensemble", metavar="REPLACE", default=False) + parser.add_option("-m", "--merge", dest="merge", action="store_true", + help="merge tweet ensemble, choose the first ensemble", metavar="MERGE", default=False) + parser.add_option("-E", "--extended", dest="extended_mode", action="store_true", + help="Trigger polemic extended mode", metavar="EXTENDED", default=False) + parser.add_option("-S", "--step", dest="step", type=1, + help="step for version", metavar="STEP", default=False) + + + + set_logging_options(parser) + + + return parser.parse_args() + (parser,) + + +if __name__ == "__main__" : + + (options, args, parser) = get_options() + + set_logging(options) + get_logger().debug("OPTIONS : " + repr(options)) #@UndefinedVariable + + if len(sys.argv) == 1: + abort(None) + + base_url = options.get("api_url", None) + if not base_url: + abort("No base url") + + api_key = options.get("api_key", None) + if not api_key: + abort("api key missing") + + pad_id = options.get("pad_id", None) + if not pad_id: + abort("No pad id") + + start_date_str = options.get("start_date",None) + end_date_str = options.get("end_date", None) + duration = options.get("duration", None) + + start_date = None + start_ts = None + if start_date_str: + start_date = parse_date(start_date_str) + start_ts = time.mktime(start_date.timetuple())*1000 + + end_date = None + if end_date_str: + end_date = parse_date(end_date_str) + elif start_date and duration: + end_date = start_date + datetime.timedelta(seconds=duration) + + if start_date is None or end_date is None: + abort("No start date found") + + end_ts = None + if end_date is not None: + end_ts = time.mktime(end_date.timetuple())*1000 + + content_file = options.get("content_file", None) + + if not content_file: + abort("No content file") + + root = None + + if content_file.find("http") == 0: + + get_logger().debug("url : " + content_file) #@UndefinedVariable + + h = httplib2.Http() + resp, content = h.request(content_file) + + get_logger().debug("url response " + repr(resp) + " content " + repr(content)) #@UndefinedVariable + + project = anyjson.deserialize(content) + root = etree.fromstring(project["ldt"]) + + elif os.path.exists(content_file): + + doc = etree.parse(content_file) + root = doc.getroot() + + if root is None: + abort("No content file, file not found") + + cutting_name = options.get("name", None) + if cutting_name is None: + cutting_name = "pad_%s" % pad_id + + output_format = options.get('format','html') + ensemble_parent = None + + file_type = None + for node in root: + if node.tag == "project": + file_type = "ldt" + break + elif node.tag == "head": + file_type = "iri" + break + if file_type is None: + abort("Unknown file type") + + if file_type == "ldt": + media_nodes = root.xpath("//media") + if len(media_nodes) > 0: + media = media_nodes[0] + annotations_node = root.find(u"annotations") + if annotations_node is None: + annotations_node = etree.SubElement(root, u"annotations") + content_node = annotations_node.find(u"content") + if content_node is None: + content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id")) + ensemble_parent = content_node + elif file_type == "iri": + body_node = root.find(u"body") + if body_node is None: + body_node = etree.SubElement(root, u"body") + ensembles_node = body_node.find(u"ensembles") + if ensembles_node is None: + ensembles_node = etree.SubElement(body_node, u"ensembles") + ensemble_parent = ensembles_node + + if ensemble_parent is None: + abort("Can not add cutting") + + if options.replace: + for ens in ensemble_parent.iterchildren(tag=u"ensemble"): + if ens.get("id","").startswith(cutting_name): + ensemble_parent.remove(ens) + + ensemble = None + elements = None + + if options.merge: + ensemble = ensemble_parent.find(u"ensemble") + if ensemble is not None: + elements = ensemble.find(u".//elements") + + if ensemble is None or elements is None: + ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"tweet_" + unicode(uuid.uuid4()), u"title":u"Ensemble pad", u"author":u"IRI Web", u"abstract":u"Ensemble Pad"}) + decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"}) + + etree.SubElement(decoupage, u"title").text = unicode(cutting_name) + etree.SubElement(decoupage, u"abstract").text = unicode(cutting_name) + + elements = etree.SubElement(decoupage, u"elements") + + + etp_req = EtherpadRequest(base_url, api_key) + rev_count = etp_req.getRevisionCount(pad_id) + + + version_range = range(1,rev_count+1, 1) + #make sure that teh last version is exported + if rev_count not in version_range: + version_range.append(rev_count) + for rev in version_range: + + data = None + text = "" + + if output_format == "html": + data = etp_req.getHtml(padID=pad_id, rev=rev) + text = data.get("html", "") + else: + data = etp_req.getText(padID=pad_id, rev=rev) + text = data.get("text","") + + pad_ts = data['timestamp'] + + if pad_ts < start_ts: + continue + + if end_ts is not None and pad_ts > end_ts: + break + + pad_dt = datetime.datetime.fromtimestamp(float(pad_ts)/1000.0) + pad_ts_rel = pad_ts - start_ts + + username = None + color = "" + if 'author' in data: + username = data['author']['name'] if ('name' in data['author'] and data['author']['name']) else data['author']['id'] + color = data['author']['color'] if ('color' in data['author'] and data['author']['color']) else "" + + if not username: + username = "anon." + + + element = etree.SubElement(elements, u"element" , {u"id":"%s-%s-%d" %(unicode(uuid.uuid4()),unicode(pad_id),rev), u"color":unicode(color), u"author":unicode(username), u"date":unicode(pad_dt.strftime("%Y/%m/%d")), u"begin": unicode(pad_ts_rel), u"dur":u"0", u"src":""}) + etree.SubElement(element, u"title").text = "%s: %s - rev %d" % (unicode(username), unicode(pad_id), rev) + etree.SubElement(element, u"abstract").text = unicode(text) + + meta_element = etree.SubElement(element, u'meta') + etree.SubElement(meta_element, "pad_url").text = etree.CDATA(unicode(etp_req.getPadUrl(pad_id))) + etree.SubElement(meta_element, "revision").text = etree.CDATA(unicode(rev)) + + # sort by tc in + if options.merge : + elements[:] = sorted(elements,key=lambda n: int(n.get('begin'))) + + output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=False, xml_declaration=True) + + if content_file and content_file.find("http") == 0: + + project["ldt"] = output_data + body = anyjson.serialize(project) + h = httplib2.Http() + resp, content = h.request(content_file, "PUT", headers={'content-type':'application/json'}, body=body) + if resp.status != 200: + raise Exception("Error writing content : %d : %s"%(resp.status, resp.reason)) + else: + if content_file and os.path.exists(content_file): + dest_file_name = content_file + else: + dest_file_name = options.filename + + output = open(dest_file_name, "w") + output.write(output_data) + output.flush() + output.close() + + diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/export_tweet_db.py --- a/script/utils/export_tweet_db.py Tue May 07 18:28:26 2013 +0200 +++ b/script/utils/export_tweet_db.py Fri May 10 13:34:40 2013 +0200 @@ -1,8 +1,11 @@ -from models import setup_database -from optparse import OptionParser #@UnresolvedImport -from sqlalchemy.orm import sessionmaker -from utils import set_logging_options, set_logging, TwitterProcessor, logger -import sqlite3 #@UnresolvedImport +from iri_tweet.models import setup_database +from iri_tweet.processor import TwitterProcessorStatus +from iri_tweet.utils import set_logging_options, set_logging +from optparse import OptionParser +import logging +import sqlite3 + +logger = logging.getLogger(__name__) # 'entities': "tweet_entity", @@ -33,7 +36,7 @@ fields_mapping = {} for i,res in enumerate(curs_in.execute("select json from tweet_tweet;")): logger.debug("main loop %d : %s" % (i, res[0])) #@UndefinedVariable - processor = TwitterProcessor(eval(res[0]), res[0], None, session, options.token_filename) + processor = TwitterProcessorStatus(json_dict=eval(res[0]), json_txt=res[0], source_id=None, session=session, consumer_token=None, access_token=None, token_filename=options.token_filename, user_query_twitter=False, logger=logger) processor.process() session.commit() logger.debug("main : %d tweet processed" % (i+1)) #@UndefinedVariable diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/export_twitter_alchemy.py --- a/script/utils/export_twitter_alchemy.py Tue May 07 18:28:26 2013 +0200 +++ b/script/utils/export_twitter_alchemy.py Fri May 10 13:34:40 2013 +0200 @@ -366,7 +366,7 @@ username = None profile_url = "" if tw.user is not None: - username = tw.user.name + username = tw.user.screen_name profile_url = tw.user.profile_image_url if tw.user.profile_image_url is not None else "" if not username: username = "anon." @@ -416,7 +416,7 @@ get_logger().debug("write http " + repr(project)) #@UndefinedVariable r = requests.put(content_file_write, data=anyjson.dumps(project), headers={'content-type':'application/json'}, params=post_param); get_logger().debug("write http " + repr(r) + " content " + r.text) #@UndefinedVariable - if r.status_code != requests.codes.ok: + if r.status_code != requests.codes.ok: # @UndefinedVariable r.raise_for_status() else: if content_file_write and os.path.exists(content_file_write): diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/get_stats.py --- a/script/utils/get_stats.py Tue May 07 18:28:26 2013 +0200 +++ b/script/utils/get_stats.py Fri May 10 13:34:40 2013 +0200 @@ -1,14 +1,13 @@ +from lxml import etree import httplib2 -import anyjson -from lxml import etree +import pprint import sys -import pprint def get_stats(url): h = httplib2.Http() - resp, content = h.request(url) + _, content = h.request(url) #project = anyjson.deserialize(content) root = etree.fromstring(content) diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/merge_tweets.py --- a/script/utils/merge_tweets.py Tue May 07 18:28:26 2013 +0200 +++ b/script/utils/merge_tweets.py Fri May 10 13:34:40 2013 +0200 @@ -1,12 +1,15 @@ #from models import setup_database from iri_tweet.models import setup_database, TweetSource, Tweet, TweetLog -from iri_tweet.utils import TwitterProcessor, get_oauth_token, show_progress +from iri_tweet.processor import TwitterProcessorStatus +from iri_tweet.utils import get_oauth_token, show_progress +import anyjson import argparse -import sys +import codecs +import logging import re -import anyjson -import math -import codecs +import sys + +logger = logging.getLogger(__name__) def get_option(): @@ -16,6 +19,10 @@ help="log to file", metavar="LOG", default="stderr") parser.add_argument("-v", dest="verbose", action="count", help="verbose", default=0) + parser.add_option("-k", "--key", dest="consumer_key", + help="Twitter consumer key", metavar="CONSUMER_KEY") + parser.add_option("-s", "--secret", dest="consumer_secret", + help="Twitter consumer secret", metavar="CONSUMER_SECRET") parser.add_argument("-q", dest="quiet", action="count", help="quiet", default=0) parser.add_argument("--query-user", dest="query_user", action="store_true", @@ -38,7 +45,7 @@ access_token = None if options.query_user: - access_token = get_oauth_token(options.token_filename) + access_token = get_oauth_token(options.consumer_key, options.consumer_secret, options.token_filename) #open source src_conn_str = options.source[0].strip() @@ -60,7 +67,7 @@ session_src = Session_src() session_tgt = Session_tgt() - count_tw_query = Tweet.__table__.count() + count_tw_query = Tweet.__table__.count() # @UndefinedVariable count_tw = engine_src.scalar(count_tw_query) @@ -83,23 +90,28 @@ tweet_obj = anyjson.deserialize(tweet_source) if 'text' not in tweet_obj: - tweet_log = TweetLog(tweet_source_id=source_id, status=TweetLog.TWEET_STATUS['NOT_TWEET']) + tweet_log = TweetLog(tweet_source_id=tweet.tweet_source.id, status=TweetLog.TWEET_STATUS['NOT_TWEET']) session_tgt.add(tweet_log) else: - tp = TwitterProcessor(None, tweet_source, None, session_tgt, access_token, options.token_filename, user_query_twitter=options.query_user) + tp = TwitterProcessorStatus(None, tweet_source, None, session_tgt, access_token, options.token_filename, user_query_twitter=options.query_user, logger=logger) tp.process() session_tgt.flush() - show_progress(i+1, count_tw, repr(progress_text+tweet.text), 70) + ptext = progress_text + tweet.text + show_progress(i+1, count_tw, ptext.replace("\n",""), 70) session_tgt.commit() print u"%d new tweet added" % (added) finally: - session_tgt.close() if session_tgt is not None else None - session_src.close() if session_src is not None else None - conn_tgt.close() if conn_tgt is not None else None - conn_src.close() if conn_src is not None else None + if session_tgt is not None: + session_tgt.close() + if session_src is not None: + session_src.close() + if conn_tgt is not None: + conn_tgt.close() + if conn_src is not None: + conn_src.close() \ No newline at end of file diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/search_topsy.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/script/utils/search_topsy.py Fri May 10 13:34:40 2013 +0200 @@ -0,0 +1,168 @@ +from blessings import Terminal +from iri_tweet import models, utils +from iri_tweet.processor import TwitterProcessorStatus +from optparse import OptionParser +import logging +import math +import re +import requests +import time +import twitter + +logger = logging.getLogger(__name__) + +APPLICATION_NAME = "Tweet recorder user" +CONSUMER_KEY = "Vdr5ZcsjI1G3esTPI8yDg" +CONSUMER_SECRET = "LMhNrY99R6a7E0YbZZkRFpUZpX5EfB1qATbDk1sIVLs" + + +class TopsyResource(object): + + def __init__(self, query, **kwargs): + + self.options = kwargs + self.options['q'] = query + self.url = kwargs.get("url", "http://otter.topsy.com/search.json") + self.page = 0 + self.req = None + self.res = {} + + def __initialize(self): + + params = {} + params.update(self.options) + self.req = requests.get(self.url, params=params) + self.res = self.req.json + + def __next_page(self): + page = self.res.get("response").get("page") + 1 + params = {} + params.update(self.options) + params['page'] = page + self.req = requests.get(self.url, params=params) + self.res = self.req.json + + def __iter__(self): + if not self.req: + self.__initialize() + while "response" in self.res and "list" in self.res.get("response") and self.res.get("response").get("list"): + for item in self.res.get("response").get("list"): + yield item + self.__next_page() + + def total(self): + if not self.res: + return 0 + else: + return self.res.get("response",{}).get("total",0) + + + +def get_option(): + + parser = OptionParser() + + parser.add_option("-d", "--database", dest="database", + help="Input database", metavar="DATABASE") + parser.add_option("-Q", dest="query", + help="query", metavar="QUERY") + parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", + help="Token file name") + parser.add_option("-T", dest="topsy_apikey", metavar="TOPSY_APIKEY", default=None, + help="Topsy apikey") + + utils.set_logging_options(parser) + + return parser.parse_args() + + + +if __name__ == "__main__": + + (options, args) = get_option() + + utils.set_logging(options); + + + acess_token_key, access_token_secret = utils.get_oauth_token(consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET, options.token_filename, application_name=APPLICATION_NAME) + + t = twitter.Twitter(domain="api.twitter.com", auth=twitter.OAuth(acess_token_key, access_token_secret, CONSUMER_KEY, CONSUMER_SECRET), secure=True) + t.secure = True + + conn_str = options.database.strip() + if not re.match("^\w+://.+", conn_str): + conn_str = 'sqlite:///' + conn_str + + engine, metadata, Session = models.setup_database(conn_str, echo=((options.verbose-options.quiet)>0), create_all=True) + session = None + + + topsy_parameters = { + 'apikey': options.topsy_apikey, + 'perpage': 100, + 'window': 'a', + 'type': 'tweet', + 'hidden': True, + } + + term = Terminal() + + try: + session = Session() + + results = None + page = 1 + print options.query + + tr = TopsyResource(options.query, **topsy_parameters) + + move_up = 0 + + for i,item in enumerate(tr): + # get id + url = item.get("url") + tweet_id = url.split("/")[-1] + + if move_up > 0: + print((move_up+1)*term.move_up()) + move_up = 0 + + print ("%d/%d:%03d%% - %s - %r" % (i+1, tr.total(), int(float(i+1)/float(tr.total())*100.0), tweet_id, item.get("content") ) + term.clear_eol()) + move_up += 1 + + count_tweet = session.query(models.Tweet).filter_by(id_str=tweet_id).count() + + if count_tweet: + continue + try: + tweet = t.statuses.show(id=tweet_id, include_entities=True) + except twitter.api.TwitterHTTPError as e: + if e.e.code == 404 or e.e.code == 403: + continue + else: + raise + + processor = TwitterProcessorStatus(tweet, None, None, session, None, options.token_filename, logger) + processor.process() + session.flush() + session.commit() + + time_to_sleep = int(math.ceil((tweet.rate_limit_reset - time.mktime(time.gmtime())) / tweet.rate_limit_remaining)) + + print "rate limit remaining %s of %s" % (str(tweet.rate_limit_remaining), str(tweet.headers.getheader('x-ratelimit-limit'))) + term.clear_eol() + move_up += 1 + for i in xrange(time_to_sleep): + if i: + print(2*term.move_up()) + else: + move_up += 1 + print(("Sleeping for %d seconds, %d remaining" % (time_to_sleep, time_to_sleep-i)) + term.clear_eol()) + time.sleep(1) + + except twitter.api.TwitterHTTPError as e: + fmt = ("." + e.format) if e.format else "" + print "Twitter sent status %s for URL: %s%s using parameters: (%s)\ndetails: %s" % (repr(e.e.code), repr(e.uri), repr(fmt), repr(e.uriparts), repr(e.response_data)) + + finally: + if session: + session.close() diff -r 41ce1c341abe -r 10a19dd4e1c9 script/utils/tweet_twitter_user.py --- a/script/utils/tweet_twitter_user.py Tue May 07 18:28:26 2013 +0200 +++ b/script/utils/tweet_twitter_user.py Fri May 10 13:34:40 2013 +0200 @@ -38,6 +38,7 @@ parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token", help="Token file name") parser.add_option("-S", dest="simulate", metavar="SIMULATE", default=False, action="store_true", help="Simulate call to twitter. Do not change the database") + parser.add_option("--direct-message", dest="direct_message", metavar="DIRECT_MESSAGE", default=False, action="store_true", help="send direc t message to the user, else create a status update mentioning the user (@username)") parser.add_option("-f", dest="force", metavar="FORCE", default=False, action="store_true", help="force sending message to all user even if it has already been sent") @@ -103,16 +104,25 @@ query_res = query.all() - acess_token_key, access_token_secret = get_oauth_token(options.token_filename, application_name=APPLICATION_NAME, consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET) + acess_token_key, access_token_secret = get_oauth_token(consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET, token_file_path=options.token_filename, application_name=APPLICATION_NAME) t = twitter.Twitter(auth=twitter.OAuth(acess_token_key, access_token_secret, CONSUMER_KEY, CONSUMER_SECRET)) for user in query_res: screen_name = user.screen_name - message = u"@%s: %s" % (screen_name, base_message) - get_logger().debug("new status : " + message) #@UndefinedVariable + if options.direct_message: + message = base_message + else: + message = u"@%s: %s" % (screen_name, base_message) + + print("new message : " + message) + get_logger().debug("new message : " + message) #@UndefinedVariable + if not options.simulate: - t.statuses.update(status=message) + if options.direct_message: + t.direct_messages.new(user_id=user.id, screen_name=screen_name, text=message) + else: + t.statuses.update(status=message) user_message = UserMessage(user_id=user.id, message_id=message_obj.id) session.add(user_message) session.flush() diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/credential.txt --- a/script/virtualenv/res/credential.txt Tue May 07 18:28:26 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,20 +0,0 @@ -Consumer key -54ThDZhpEjokcMgHJOMnQA - -Consumer secret -wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA - -access_tokens: -47312923-LiNTtz0I18YXMVIrFeTuhmH7bOvYsK6p3Ln2Dc - -access_secret: -r3LoXVcjImNAElUpWqTu2SG2xCdWFHkva7xeQoncA - -Request token URL -http://twitter.com/oauth/request_token - -Access token URL -http://twitter.com/oauth/access_token - -Authorize URL -http://twitter.com/oauth/authorize \ No newline at end of file diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/lib/lib_create_env.py --- a/script/virtualenv/res/lib/lib_create_env.py Tue May 07 18:28:26 2013 +0200 +++ b/script/virtualenv/res/lib/lib_create_env.py Fri May 10 13:34:40 2013 +0200 @@ -16,26 +16,28 @@ URLS = { #'': {'setup': '', 'url':'', 'local':''}, - 'DISTRIBUTE': {'setup': 'distribute', 'url':'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.32.tar.gz', 'local':"distribute-0.6.32.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'DISTRIBUTE': {'setup': 'distribute', 'url':'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.34.tar.gz', 'local':"distribute-0.6.34.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'ANYJSON': {'setup': 'anyjson', 'url':'http://pypi.python.org/packages/source/a/anyjson/anyjson-0.3.3.tar.gz', 'local':"anyjson-0.3.3.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'OAUTH2': { 'setup': 'python-oauth2', 'url':"https://github.com/simplegeo/python-oauth2/tarball/hudson-python-oauth2-211", 'local':"python-oauth2-1.5-211.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'HTTPLIB2': { 'setup': 'httplib2', 'url':'http://pypi.python.org/packages/source/h/httplib2/httplib2-0.7.7.tar.gz', 'local':"httplib2-0.7.7.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'HTTPLIB2': { 'setup': 'httplib2', 'url':'http://pypi.python.org/packages/source/h/httplib2/httplib2-0.8.tar.gz', 'local':"httplib2-0.8.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'LOCKFILE': {'setup': 'lockfile', 'url':'http://code.google.com/p/pylockfile/downloads/detail?name=lockfile-0.9.1.tar.gz', 'local':"lockfile-0.9.1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'DAEMON': {'setup': 'python-daemon', 'url':'http://pypi.python.org/packages/source/p/python-daemon/python-daemon-1.5.5.tar.gz', 'local':"python-daemon-1.5.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'DATEUTIL': {'setup': 'python-dateutil', 'url':'http://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz', 'local':"python-dateutil-2.1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'PYTZ': {'setup': 'pytz', 'url':'http://pypi.python.org/packages/source/p/pytz/pytz-2012h.tar.bz2', 'local':"pytz-2012h.tar.bz2", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'SIMPLEJSON': {'setup': 'simplejson', 'url':'http://pypi.python.org/packages/source/s/simplejson/simplejson-2.6.2.tar.gz', 'local':"simplejson-2.6.2.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'SQLALCHEMY': {'setup': 'sqlalchemy', 'url':'http://downloads.sourceforge.net/project/sqlalchemy/sqlalchemy/0.8.0b1/SQLAlchemy-0.8.0b1.tar.gz?r=http%3A%2F%2Fwww.sqlalchemy.org%2Fdownload.html&ts=1355091775&use_mirror=ignum', 'local':"SQLAlchemy-0.8.0b1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'TWEEPY': {'setup': 'tweepy', 'url':'https://github.com/tweepy/tweepy/archive/1.12.tar.gz', 'local':"tweepy-1.12.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'TWITTER': {'setup': 'twitter', 'url':'http://pypi.python.org/packages/source/t/twitter/twitter-1.9.0.tar.gz', 'local':"twitter-1.9.0.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'PYTZ': {'setup': 'pytz', 'url':'http://pypi.python.org/packages/source/p/pytz/pytz-2013b.tar.bz2', 'local':"pytz-2013b.tar.bz2", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'SIMPLEJSON': {'setup': 'simplejson', 'url':'http://pypi.python.org/packages/source/s/simplejson/simplejson-3.1.3.tar.gz', 'local':"simplejson-3.1.3.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'SQLALCHEMY': {'setup': 'sqlalchemy', 'url':'http://www.python.org/pypi/SQLAlchemy/0.8.1', 'local':"SQLAlchemy-0.8.1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'TWITTER': {'setup': 'twitter', 'url':'http://pypi.python.org/packages/source/t/twitter/twitter-1.9.2.tar.gz', 'local':"twitter-1.9.2.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'TWITTER-TEXT': {'setup': 'twitter-text', 'url':'https://github.com/dryan/twitter-text-py/archive/master.tar.gz', 'local':"twitter-text-1.0.4.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'REQUESTS': {'setup': 'requests', 'url':'https://github.com/kennethreitz/requests/archive/v1.1.0.tar.gz', 'local':'requests-v1.1.0.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}}, + 'REQUESTS': {'setup': 'requests', 'url':'https://github.com/kennethreitz/requests/archive/v1.2.0.tar.gz', 'local':'requests-v1.2.0.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}}, + 'OAUTHLIB': {'setup': 'oauthlib', 'url':'https://github.com/idan/oauthlib/archive/0.4.0.tar.gz', 'local':'oauthlib-0.4.0.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}}, + 'REQUESTS-OAUTHLIB': {'setup': 'requests-oauthlib', 'url':'https://github.com/requests/requests-oauthlib/archive/master.tar.gz', 'local':'requests-oauthlib-0.3.0.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}}, + 'BLESSINGS': {'setup': 'blessings', 'url':'https://github.com/erikrose/blessings/archive/1.5.tar.gz', 'local':'blessings-1.5.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}} } if system_str == 'Windows': URLS.update({ - 'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.4.5.win32-py2.7-pg9.1.3-release.zip', 'local':"psycopg2-2.4.5.win32-py2.7-pg9.1.3-release.zip", 'install': {'method': 'install_psycopg2', 'option_str': None, 'dict_extra_env': None}}, + 'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.5.win32-py2.7-pg9.2.4-release.zip', 'local':"psycopg2-2.5.win32-py2.7-pg9.2.4-release.zip", 'install': {'method': 'install_psycopg2', 'option_str': None, 'dict_extra_env': None}}, 'LXML': {'setup': 'lxml', 'url': 'http://pypi.python.org/packages/2.7/l/lxml/lxml-2.3-py2.7-win32.egg', 'local':"lxml-2.3-py2.7-win32.egg", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}}, }) else: @@ -47,8 +49,8 @@ lxml_method = 'pip' URLS.update({ - 'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.5.tar.gz', 'local':"psycopg2-2.4.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'LXML': {'setup': 'lxml', 'url':"lxml-3.0.1.tar.gz", 'local':"lxml-3.0.1.tar.gz", 'install': {'method': lxml_method, 'option_str': None, 'dict_extra_env': lxml_options}}, + 'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-5/psycopg2-2.5.tar.gz', 'local':"psycopg2-2.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'LXML': {'setup': 'lxml', 'url':"lxml-3.1.2.tar.gz", 'local':"lxml-3.1.2.tar.gz", 'install': {'method': lxml_method, 'option_str': None, 'dict_extra_env': lxml_options}}, }) diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/SQLAlchemy-0.8.0b1.tar.gz Binary file script/virtualenv/res/src/SQLAlchemy-0.8.0b1.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/SQLAlchemy-0.8.1.tar.gz Binary file script/virtualenv/res/src/SQLAlchemy-0.8.1.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/blessings-1.5.tar.gz Binary file script/virtualenv/res/src/blessings-1.5.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/distribute-0.6.32.tar.gz Binary file script/virtualenv/res/src/distribute-0.6.32.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/distribute-0.6.34.tar.gz Binary file script/virtualenv/res/src/distribute-0.6.34.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/httplib2-0.7.7.tar.gz Binary file script/virtualenv/res/src/httplib2-0.7.7.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/httplib2-0.8.tar.gz Binary file script/virtualenv/res/src/httplib2-0.8.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/lxml-3.0.1.tar.gz Binary file script/virtualenv/res/src/lxml-3.0.1.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/lxml-3.1.2.tar.gz Binary file script/virtualenv/res/src/lxml-3.1.2.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/oauthlib-0.4.0.tar.gz Binary file script/virtualenv/res/src/oauthlib-0.4.0.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/psycopg2-2.4.5.tar.gz Binary file script/virtualenv/res/src/psycopg2-2.4.5.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/psycopg2-2.5.tar.gz Binary file script/virtualenv/res/src/psycopg2-2.5.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/psycopg2-2.5.win32-py2.7-pg9.2.4-release.zip Binary file script/virtualenv/res/src/psycopg2-2.5.win32-py2.7-pg9.2.4-release.zip has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/pytz-2012h.tar.bz2 Binary file script/virtualenv/res/src/pytz-2012h.tar.bz2 has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/pytz-2013b.tar.bz2 Binary file script/virtualenv/res/src/pytz-2013b.tar.bz2 has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/requests-1.1.0.tar.gz Binary file script/virtualenv/res/src/requests-1.1.0.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/requests-oauthlib-0.3.0.tar.gz Binary file script/virtualenv/res/src/requests-oauthlib-0.3.0.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/requests-v1.2.0.tar.gz Binary file script/virtualenv/res/src/requests-v1.2.0.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/simplejson-2.6.2.tar.gz Binary file script/virtualenv/res/src/simplejson-2.6.2.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/simplejson-3.1.3.tar.gz Binary file script/virtualenv/res/src/simplejson-3.1.3.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/tweepy-1.12.tar.gz Binary file script/virtualenv/res/src/tweepy-1.12.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/twitter-1.9.0.tar.gz Binary file script/virtualenv/res/src/twitter-1.9.0.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/res/src/twitter-1.9.2.tar.gz Binary file script/virtualenv/res/src/twitter-1.9.2.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/script/res/res_create_env.py --- a/script/virtualenv/script/res/res_create_env.py Tue May 07 18:28:26 2013 +0200 +++ b/script/virtualenv/script/res/res_create_env.py Fri May 10 13:34:40 2013 +0200 @@ -17,10 +17,12 @@ 'DATEUTIL', 'SIMPLEJSON', 'SQLALCHEMY', - 'TWEEPY', 'TWITTER', 'TWITTER-TEXT', - 'REQUESTS', + 'REQUESTS', + 'OAUTHLIB', + 'REQUESTS-OAUTHLIB', + 'BLESSINGS' ] if system_str == "Linux": diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/script/virtualenv.py --- a/script/virtualenv/script/virtualenv.py Tue May 07 18:28:26 2013 +0200 +++ b/script/virtualenv/script/virtualenv.py Fri May 10 13:34:40 2013 +0200 @@ -4,7 +4,7 @@ # If you change the version here, change it in setup.py # and docs/conf.py as well. -__version__ = "1.8.4" # following best practices +__version__ = "1.9.1" # following best practices virtualenv_version = __version__ # legacy, again import base64 @@ -862,6 +862,19 @@ 'VIRTUALENV_DISTRIBUTE to make it the default ') parser.add_option( + '--no-setuptools', + dest='no_setuptools', + action='store_true', + help='Do not install distribute/setuptools (or pip) ' + 'in the new virtualenv.') + + parser.add_option( + '--no-pip', + dest='no_pip', + action='store_true', + help='Do not install pip in the new virtualenv.') + + parser.add_option( '--setuptools', dest='use_distribute', action='store_false', @@ -961,7 +974,9 @@ use_distribute=options.use_distribute, prompt=options.prompt, search_dirs=options.search_dirs, - never_download=options.never_download) + never_download=options.never_download, + no_setuptools=options.no_setuptools, + no_pip=options.no_pip) if 'after_install' in globals(): after_install(options, home_dir) @@ -1048,7 +1063,8 @@ def create_environment(home_dir, site_packages=False, clear=False, unzip_setuptools=False, use_distribute=False, - prompt=None, search_dirs=None, never_download=False): + prompt=None, search_dirs=None, never_download=False, + no_setuptools=False, no_pip=False): """ Creates a new environment in ``home_dir``. @@ -1066,14 +1082,16 @@ install_distutils(home_dir) - if use_distribute: - install_distribute(py_executable, unzip=unzip_setuptools, - search_dirs=search_dirs, never_download=never_download) - else: - install_setuptools(py_executable, unzip=unzip_setuptools, - search_dirs=search_dirs, never_download=never_download) + if not no_setuptools: + if use_distribute: + install_distribute(py_executable, unzip=unzip_setuptools, + search_dirs=search_dirs, never_download=never_download) + else: + install_setuptools(py_executable, unzip=unzip_setuptools, + search_dirs=search_dirs, never_download=never_download) - install_pip(py_executable, search_dirs=search_dirs, never_download=never_download) + if not no_pip: + install_pip(py_executable, search_dirs=search_dirs, never_download=never_download) install_activate(home_dir, bin_dir, prompt) @@ -1189,8 +1207,9 @@ else: if f is not None: f.close() - # special-case custom readline.so on OS X: - if modname == 'readline' and sys.platform == 'darwin' and not filename.endswith(join('lib-dynload', 'readline.so')): + # special-case custom readline.so on OS X, but not for pypy: + if modname == 'readline' and sys.platform == 'darwin' and not ( + is_pypy or filename.endswith(join('lib-dynload', 'readline.so'))): dst_filename = join(dst_prefix, 'lib', 'python%s' % sys.version[:3], 'readline.so') else: dst_filename = change_prefix(filename, dst_prefix) @@ -1351,11 +1370,6 @@ if sys.executable != py_executable: ## FIXME: could I just hard link? executable = sys.executable - if is_cygwin and os.path.exists(executable + '.exe'): - # Cygwin misreports sys.executable sometimes - executable += '.exe' - py_executable += '.exe' - logger.info('Executable actually exists in %s' % executable) shutil.copyfile(executable, py_executable) make_exe(py_executable) if is_win or is_cygwin: @@ -1901,143 +1915,146 @@ ##file site.py SITE_PY = convert(""" -eJzFPf1z2zaWv/OvwMqToeTKdOJ0OztO3RsncVrvuYm3SWdz63q0lARZrCmSJUjL6s3d337vAwAB -kpLtTXdO04klEnh4eHhfeHgPHQwGp0Uhs7lY5fM6lULJuJwtRRFXSyUWeSmqZVLOD4q4rDbwdHYb -30glqlyojYqwVRQE+1/4CfbFp2WiDArwLa6rfBVXySxO041IVkVeVnIu5nWZZDciyZIqidPkd2iR -Z5HY/3IMgvNMwMzTRJbiTpYK4CqRL8TlplrmmRjWBc75RfTn+OVoLNSsTIoKGpQaZ6DIMq6CTMo5 -oAktawWkTCp5oAo5SxbJzDZc53U6F0Uaz6T45z95atQ0DAOVr+R6KUspMkAGYEqAVSAe8DUpxSyf -y0iI13IW4wD8vCFWwNDGuGYKyZjlIs2zG5hTJmdSqbjciOG0rggQoSzmOeCUAAZVkqbBOi9v1QiW -lNZjDY9EzOzhT4bZA+aJ43c5B3D8kAU/Z8n9mGED9yC4aslsU8pFci9iBAs/5b2cTfSzYbIQ82Sx -ABpk1QibBIyAEmkyPSxoOb7VK/TdIWFluTKGMSSizI35JfWIgvNKxKkCtq0LpJEizN/KaRJnQI3s -DoYDiEDSoG+ceaIqOw7NTuQAoMR1rEBKVkoMV3GSAbP+GM8I7b8n2TxfqxFRAFZLiV9rVbnzH/YQ -AFo7BBgHuFhmNessTW5luhkBAp8A+1KqOq1QIOZJKWdVXiZSEQBAbSPkPSA9FnEpNQmZM43cjon+ -RJMkw4VFAUOBx5dIkkVyU5ckYWKRAOcCV7z78JN4e/b6/PS95jEDjGX2ZgU4AxRaaAcnGEAc1qo8 -THMQ6Ci4wD8ins9RyG5wfMCraXD44EoHQ5h7EbX7OAsOZNeLq4eBOVagTGisgPr9N3QZqyXQ538e -WO8gON1GFZo4f1svc5DJLF5JsYyZv5Azgm81nO+iolq+Am5QCKcCUilcHEQwQXhAEpdmwzyTogAW -S5NMjgKg0JTa+qsIrPA+zw5orVucABDKIIOXzrMRjZhJmGgX1ivUF6bxhmammwR2nVd5SYoD+D+b -kS5K4+yWcFTEUPxtKm+SLEOEkBeCcC+kgdVtApw4j8QFtSK9YBqJkLUXt0SRqIGXkOmAJ+V9vCpS -OWbxRd26W43QYLISZq1T5jhoWZF6pVVrptrLe0fR5xbXEZrVspQAvJ56QrfI87GYgs4mbIp4xeJV -rXPinKBHnqgT8gS1hL74HSh6qlS9kvYl8gpoFmKoYJGnab4Gkh0HgRB72MgYZZ854S28g38BLv6b -ymq2DAJnJAtYg0Lkt4FCIGASZKa5WiPhcZtm5baSSTLWFHk5lyUN9ThiHzLij2yMcw3e55U2ajxd -XOV8lVSokqbaZCZs8bKwYv34iucN0wDLrYhmpmlDpxVOLy2W8VQal2QqFygJepFe2WWHMYOeMckW -V2LFVgbeAVlkwhakX7Gg0llUkpwAgMHCF2dJUafUSCGDiRgGWhUEfxWjSc+1swTszWY5QIXE5nsG -9gdw+x3EaL1MgD4zgAAaBrUULN80qUp0EBp9FPhG3/Tn8YFTzxfaNvGQizhJtZWPs+CcHp6VJYnv -TBbYa6yJoWCGWYWu3U0GdEQxHwwGQWDcoY0yX3MVVOXmGFhBmHEmk2mdoOGbTNDU6x8q4FGEM7DX -zbaz8EBDmE7vgUpOl0WZr/C1ndtHUCYwFvYI9sQlaRnJDrLHia+QfK5KL0xTtN0OOwvUQ8HlT2fv -zj+ffRQn4qpRaeO2PruGMc+yGNiaLAIwVWvYRpdBS1R8Ceo+8Q7MOzEF2DPqTeIr46oG3gXUP5U1 -vYZpzLyXwdn709cXZ5OfP579NPl4/ukMEAQ7I4M9mjKaxxocRhWBcABXzlWk7WvQ6UEPXp9+tA+C -SaImxabYwAMwlMDC5RDmOxYhPpxoGzxJskUejqjxr+yEn7Ba0R7X1fHX1+LkRIS/xndxGIDX0zTl -RfyRBODTppDQtYI/w1yNgmAuFyAstxJFarhPnuyIOwARoWWuLeuveZKZ98xH7hAk8UPqAThMJrM0 -VgobTyYhkJY69HygQ8TuMMrJEDoWG7frSKOCn1LCUmTYZYz/9KAYT6kfosEoul1MIxCw1SxWklvR -9KHfZIJaZjIZ6gFB/IjHwUVixREK0wS1TJmAJ0q8glpnqvIUfyJ8lFsSGdwMoV7DRdKbneguTmup -hs6kgIjDYYuMqBoTRRwETsUQbGezdKNRm5qGZ6AZkC/NQe+VLcrhZw88FFAwZtuFWzPeLTHNENO/ -8t6AcAAnMUQFrVQLCuszcXl2KV4+PzpABwR2iXNLHa852tQkq6V9uIDVupGVgzD3CsckDCOXLgvU -jPj0eDfMVWRXpssKC73EpVzld3IO2CIDO6ssfqI3sJeGecxiWEXQxGTBWekZTy/GnSPPHqQFrT1Q -b0VQzPqbpd/j7bvMFKgO3goTqfU+nY1XUeZ3CboH041+CdYN1BvaOOOKBM7CeUyGRgw0BPitGVJq -LUNQYGXNLibhjSBRw88bVRgRuAvUrdf09TbL19mE964nqCaHI8u6KFiaebFBswR74h3YDUAyh61Y -QzSGAk66QNk6AORh+jBdoCztBgAQmGZFGywHltmc0RR5n4fDIozRK0HCW0q08HdmCNocGWI4kOht -ZB8YLYGQYHJWwVnVoJkMZc00g4EdkvhcdxHxptEH0KJiBIZuqKFxI0O/q2NQzuLCVUpOP7Shnz9/ -ZrZRS4qIIGJTnDQa/QWZt6jYgClMQCcYH4rjK8QGa3BHAUytNGuKg48iL9h/gvW81LINlhv2Y1VV -HB8ertfrSMcD8vLmUC0O//yXb775y3PWifM58Q9Mx5EWHRyLDukd+qDRt8YCfWdWrsWPSeZzI8Ea -SvKjyHlE/L6vk3kujg9GVn8iFzeGFf81zgcokIkZlKkMtB00GD1TB8+il2ognomh23Y4Yk9Cm1Rr -xXyrCz2qHGw3eBqzvM6q0FGkSnwF1g321HM5rW9CO7hnI80PmCrK6dDywMGLa8TA5wzDV8YUT1BL -EFugxXdI/xOzTUz+jNYQSF40UZ397qZfixnizh8v79Y7dITGzDBRyB0oEX6TRwugbdyVHPxoZxTt -nuOMmo9nCIylDwzzaldwiIJDuOBajF2pc7gafVSQpjWrZlAwrmoEBQ1u3ZSprcGRjQwRJHo3ZnvO -C6tbAJ1asT6zozerAC3ccTrWrs0KjieEPHAiXtATCU7tcefdc17aOk0pBNPiUY8qDNhbaLTTOfDl -0AAYi0H584Bbmo3Fh9ai8Br0AMs5aoMMtugwE75xfcDB3qCHnTpWf1tvpnEfCFykIUePHgWdUD7h -EUoF0lQM/Z7bWNwStzvYTotDTGWWiURabRGutvLoFaqdhmmRZKh7nUWKZmkOXrHVisRIzXvfWaCd -Cz7uM2ZaAjUZGnI4jU7I2/MEMNTtMOB1U2NowI2cIEarRJF1QzIt4R9wKygiQeEjoCVBs2AeK2X+ -xP4AmbPz1V+2sIclNDKE23SbG9KxGBqOeb8nkIw6fgJSkAEJu8JIriOrgxQ4zFkgT7jhtdwq3QQj -UiBnjgUhNQO400tvg4NPIjyzIAlFyPeVkoX4Sgxg+dqi+jjd/YdyqQkbDJ0G5CroeMOJG4tw4hAn -rbiEz9B+RIJON4ocOHgKLo8bmnfZ3DCtDZOAs+4rbosUaGSKnAxGLqrXhjBu+PdPJ06LhlhmEMNQ -3kDeIYwZaRTY5dagYcENGG/N22Ppx27EAvsOw1wdydU97P/CMlGzXIW4we3ELtyP5ooubSy2F8l0 -AH+8BRiMrj1IMtXxC4yy/AuDhB70sA+6N1kMi8zjcp1kISkwTb8Tf2k6eFhSekbu8CNtpw5hohij -PHxXgoDQYeUhiBNqAtiVy1Bpt78LducUBxYudx94bvPV8cvrLnHH2yI89tO/VGf3VRkrXK2UF42F -Alera8BR6cLk4myjjxv1cTRuE8pcwS5SfPj4WSAhOBK7jjdPm3rD8IjNg3PyPgZ10GsPkqs1O2IX -QAS1IjLKYfh0jnw8sk+d3I6JPQHIkxhmx6IYSJpP/hU4uxYKxjiYbzKMo7VVBn7g9TdfT3oioy6S -33w9eGCUFjH6xH7Y8gTtyJQGIHqnbbqUMk7J13A6UVQxa3jHtilGrNBp/6eZ7LrH6dR4UTwzvlfJ -71J8J472918e9bfFj4GH8XAJ7sLzcUPB7qzx43tWW+Fpk7UDWGfjaj57NAXY5ufTX2GzrHR87S5O -UjoUADIcHKCeNft8Dl30KxIP0k5d45Cgbyumrp4DY4QcWBh1p6P9slMTe+7ZEJtPEasuKns6AaA5 -v/IO9d2zyy5UveyGh5/zScNRj5byZtznV3yJhsXPH6KMLDCPBoM+sm9lx/+PWT7/90zykVMxx85/ -oGF8IqA/aiZsRxiatiM+rP5ld02wAfYIS7XFA93hIXaH5oPGhfHzWCUpsY+6a1+sKdeAwqx4aARQ -5uwC9sDBZdQn1m/qsuRzZ1KBhSwP8Cx1LDDNyjiBlL3VBXP4XlaIiW02o7C1k5ST96mRUAei7UzC -ZgvRL2fL3ISvZHaXlNAXFO4w/OHDj2dhvwnBkC50erwVebwLgXCfwLShJk74lD5Moad0+delqr2L -8QlqjvNNcFiTrdc++DFhE1LoX4MHgkPe2S2fkeNmfbaUs9uJpHN/ZFPs6sTH3+BrxMSmA/jJWype -UAYazGSW1kgr9sExdXBRZzM6KqkkuFo6zxfzfug0nyOBizS+EUPqPMcolOZGClTdxaV2RIsyx8xS -USfzw5tkLuRvdZziDl8uFoALnmPpVxEPT8Eo8ZYTEjjjUMlZXSbVBkgQq1wfA1LugtNwuuGJDj0k -+cSHCYjZDMfiI04b3zPh5oZcJk7gH37gJHELjh3MOS1yFz2H91k+wVEnlKA7ZqS6R/T0OGiPkAOA -AQCF+Q9GOojnv5H0yj1rpDV3iYpa0iOlG3TIyRlDKMMRBj34N/30GdHlrS1Y3mzH8mY3ljdtLG96 -sbzxsbzZjaUrEriwNn5lJKEvhtU+4ehNlnHDTzzMWTxbcjtM3MQETYAoCrPXNjLF+ctekIuP+ggI -qW3n7JkeNskvCWeEljlHwzVI5H48z9L7epN57nSmVBrdmadi3NltCUB+38MoojyvKXVneZvHVRx5 -cnGT5lMQW4vuuAEwFu1cIA6bZneTKQd6W5ZqcPlfn3748B6bI6iByXSgbriIaFhwKsP9uLxRXWlq -9oEFsCO19HNyqJsGuPfIIBuPssf/vKVkD2QcsaZkhVwU4AFQSpZt5iYuhWHruc5w0s+Zyfnc6UQM -smrQTGoLkU4vL9+efjodUPRv8L8DV2AMbX3pcPExLWyDrv/mNrcUxz4g1DrM1Rg/d04erRuOeNjG -GrAdH7714OgxBrs3YuDP8t9KKVgSIFSk48BPIdSj90BftE3o0McwYidzzz1kY2fFvnNkz3FRHNHv -O4FoD+Cfe+IeYwIE0C7U0OwMms1US+lb87qDog7QR/p6X7wFa2+92jsZn6J2Ej0OoENZ22y7++cd -2bDRU7J6ffb9+fuL89eXp59+cFxAdOU+fDw8Emc/fhaUKoIGjH2iGLMkKkxKAsPiVimJeQ7/1Rj5 -mdcVx4uh19uLC31os8I6FUxcRpsTwXPOaLLQOHzGAWn7UKciIUap3iA5BUGUuUMFQ7hfWnExisp1 -cjPVGU3RWa311ksXepmCMDrijkD6oLFLCgbB2WbwilLQK7MrLPkwUBdJ9SClbbTNEUkpPNjJHHCO -wsxBixczpc7wpOmsFf1V6OIaXkeqSBPYyb0KrSzpbpgp0zCOfmjPuhmvPg3odIeRdUOe9VYs0Gq9 -Cnluuv+oYbTfasCwYbC3MO9MUqYIpU9jnpsIsREf6oTyHr7apddroGDB8MyvwkU0TJfA7GPYXItl -AhsI4MklWF/cJwCE1kr4ZwPHTnRA5pioEb5ZzQ/+FmqC+K1/+aWneVWmB/8QBeyCBGcVhT3EdBu/ -hY1PJCNx9uHdKGTkKEtX/K3G3H5wSCgA6kg7pTLxYfpkqGS60Kkmvj7AF9pPoNet7qUsSt293zUO -UQKeqSF5Dc+UoV+ImV8W9hinMmqBxrIFixmW/7kZCeazJz4uZZrqZPXztxdn4DtiJQVKEB/BncFw -HC/B03Sdh8fliS1QeNYOr0tk4xJdWMq3mEdes96gNYoc9fZSNOw6UWC426sTBS7jRLloD3HaDMvU -AkTIyrAWZlmZtVttkMJuG6I4ygyzxOSypFxWnyeAl+lpzFsi2CthnYaJwPOBcpJVJnkxTWagR0Hl -gkIdg5AgcbEYkTgvzzgGnpfK1DDBw2JTJjfLCs85oHNE9RPY/MfTzxfn76mm4Ohl43X3MOeYdgJj -zic5wWxBjHbAFzcDELlqMunjWf0KYaD2gT/tV5yocsIDdPpxYBH/tF9xEdmJsxPkGYCCqou2eOAG -wOnWJzeNLDCudh+MHzcbsMHMB0OxSKxZ0Tkf7vy6nGhbtkwJxX3Myycc4CwKm52mO7vZae2PnuOi -wBOv+bC/Ebztky3zmULX286bbXlw7qcjhVjPChh1W/tjmESxTlM9HYfZtnELbWu1jf0lc2KlTrtZ -hqIMRBy6nUcuk/UrYd2cOdDLqO4AE99qdI0k9qrywS/ZQHsYHiaW2J19iulIFS1kBDCSIXXhTg0+ -FFoEUCCUCDx0JHc82j/y5uhYg4fnqHUX2MYfQBHqtFwq98hL4ET48hs7jvyK0EI9eixCx1PJZJbb -lDH8rJfoVb7w59grAxTERLEr4+xGDhnW2MD8yif2lhAsaVuP1FfJdZ9hEefgnN5v4fCuXPQfnBjU -WozQaXcrN2115JMHG/RWhewkmA++jNeg+4u6GvJKbjmH7i2E2w71YYiYiAhN9Tn8MMRwzG/hlvVp -APdSQ8NCD++3LaewvDbGkbX2sVXgFNoX2oOdlbA1qxQdyziVhcYXtV5AY3BPGpM/sE91zpD93VMy -5sSELFAe3AXpzW2gG7TCCQOuXOKyz4Qy45vCGv1uLu9kCkYDjOwQCx9+tYUPo8iGU3pTwr4Yu8vN -5aYfN3rTYHZsKjPQM1MFrF+UyeoQ0emN+OzCrEEGl/oXvSWJs1vykt/8/Xws3rz/Cf59LT+AKcXK -xbH4B6Ah3uQl7C+59JbuRMCijoo3jnmtsLyRoNFRBV8fgW7bpUdnPBbR1SZ+mYnVlAITbMsV31kC -KPIEqRy98RNMDQX8NkVeLW/UeIp9izLQL5EG2+tesFbkULeMltUqRXvhREma1bwaXJy/OXv/8Syq -7pHDzc+BE0Xxc7NwOvqMuMTzsLGwT2Y1Prl2HOcfZFr0+M1602lqaHDTKULYlxR2o8n3YcR2cxGX -GDkQxWaezyJsCSzPZXvVGhzpkbO/fNDQe1YWYQ1H+hSt8ebxMVBD/NJWRANoSH30nKgnIRRPsX6M -H0eDflM8FhTahj/7t+u5GxnXhUA0wTamzayHfnerC5dMZw3PchLhdWKXwdSGpkmsVtOZWzP4IRP6 -OhPQcnTOIRdxnVZCZiC5tMmneyVA07tlfiwhzCpszqj2jcI06TreKCcJKVZigKMOqDQeD2QoYgh7 -8B/jW7YHWH8oai5kBuiEKO2fcqerqmdLlmDeEhH1ehIP1kn20s3n0RTmQXmHPGscWZgnuo2M0Y2s -9Pz5wXB09aLJdKCo9Mwr8p0VYPVcNtkD1Vns7+8PxH887P0wKlGa57fglgHsXq/lgl5vsdx6cna1 -up69eRMBP86W8goeXFP03D6vMwpN7uhKCyLtXwMjxLUJLTOa9i27zEG7kg+auQUfWGnL8XOW0KVF -GFqSqGz13U8YdjLSRCwJiiGM1SxJQg5TwHps8hrr8zDMqPlF3gPHJwhmjG/xhIy32kv0MCmX1nKP -RedEDAjwgHLLeDQqcKYKNcBzcrnRaE7Os6RqSkueu4enupC/sncRab4S8Rolw8yjRQyn1NNj1cbD -zneyqLdjyWdXbsCxNUt+/RDuwNogafliYTCFh2aRZrksZ8ac4ools6RywJh2CIc70xVMZH2ioAel -Aah3sgpzK9H27Z/suriYfqBz5AMzkk4fquy1VhwcirNWgmEUNeNTGMoS0vKt+TKCUd5TWFt7At5Y -4k86qIp1Bd7tG26JY53pWzU4f6O5agPg0E1OVkFadvR0hHN9mIXPTLvlLgz80BadcLtLyqqO04m+ -vGGCDtvEHqxrPG1p3M6iT+utgJOfgwd8oLP4wXEwWTZIT0zCNVUaJ2KhQxSRW23mF2YVOXp5R+wr -gU+BlJlPTI20CSJdWXa1xac6Z9NR8QjqK1PQtMUzN5U0nSIUF/Mx5TmZEogtXrTBpX2nhfjuRAxf -jMWfWxuhWbHBW5kA5Wfz6Nk89H0y6np1fNTYme7GswVhK5CX10+ebppMaXphX/r5w3110iFuAFcg -O4tEzg+eKcSOcf5SqBpKM6/tnEIzxur0PZv1pAuzm3IVqkqbgle/bhSKo1qM/2kHMRXfWg9wcSwK -LVsgW9BvEk9ayX/20jVMDNTo+SuLnsuk73AKv+HFKfBeE9R1dLYeWuoMewu2Z0+uyyj5CKpp2HD8 -gx7Vk0SpnSPeaYXHk43Euaz/BB4O6ZIZYpqvWsfC/07m4aT9bYeLHSy/+XoXnq6C6a2Y6FnQx1Yx -8KK3SxeahTef/qCXxzJ9Xf940dkqGE9d/kdkBTwsZY+XsF3S9WQq6V79tMING6ZLL2N+g4a3Lo5t -QMMoHjxwGrpJdPipbnsrf1jpoAaubsNd0+f+u+auWwR25uYMuTN3v8LPpYHuu51f+mjAm0lNiEdl -pjdqoV/juMpirFMX+gOj+oPkdzvhTLfonofAmEQJDLMSm2rsjW1YxTP3O+bhHPAltm5BZ69Fak27 -o1jaHP8Yc8I5B/jc1nhTIslccyB7p3Qr2YRTEyfy5kZNYrwRb0JbGkqj6fiqxkl+RxeayVhtjG+L -18YACMNNOuHRzWkGxoBtE9/My1kozv0ggoamXE0n+VMlc45TaUcawEUcn6L+Jv7J2ZuDVGJYUdVl -UcLeY6Dvb+X0iL6M0gaoCZesYnVrUDc9xvo6TxyCc3JMEShHxXg/41EHCME63rmcitOhJxP7Dvjl -eVPsnowtQ8isXskyrpqLXvzD2ATsSzMClf7iAjsBkUYyW5ziIpZY/nCQwpCE/f6VduW9rcyOCveR -1XqPZyvqoQNtTymed2yP4ebk3l705l4wNKdrgV1XwjZruM9ebgNLYW4tI12pIxT8Vt+kxPdzcvwU -nRGHj0Du3cI3PwnYqjV2hSwazjNXMXSvzsHabbLFfTfidbige/ddaztjx/f1hmWWjhOypbGlonbg -ehVPM9qo2bdjvt4D+3Y/J/uJ+3YP/iP37fr+QjA4Gh+tD3qztB/Y4LOacC8DbBgB+kyASHh+2LpK -zpjMoZvzDJvr5H5gL+NlnekUUhkzgRzZvSWKQPClf8pNEPUu5dq1b/elix5/f/Hh9ekF0WJyefrm -P0+/p5wYDFK3bNajAxtZfsDUPvCyb90gh85j6Bu8wbbndk0uIdEQOu87R8A9EPrLhfoWtK3I3Nfb -OnTKLrqdAPHd025B3aayeyF3/DOd4u9mL7TSZAP9lHMazS/nYNg8MucjLA7N+Yd534SstYx2Inrb -Fs7JLuyqE+236vsYt0QbRzbHlVYAI9XIXzZQbQoWbDiUHZX2/yKBEnOx2MvcZQJSOJPOnXp0nR6D -qvz/F0MJyi7G0zZ2GMf2XmNqx0F5ZS/sxhO3mYwMQbxqq0F3fq6wz2W6hQpBwApP3xjHiBj9p4+x -7KHvMyWuDqiu8wCVzbX9hWumndy/J3i0W9mblxTnh/DhFjRe1Kl7XGv7dDqQ80dnAPnCKSQAzXcI -dG7EUwF7o8/ECnG6ESFsJPWxJOYmEh31tWkO8mg3HewNrZ6Lg21Vf27VmxAvtjectwrrdI8j7qEe -6KFqU1vlWGBMkttWzie+I8h8iCToqiXP+cCTS33DL3y9u3pxbEO6yO/42lEklMwzcAz7lZMMt/N6 -P6c7MUs5pmwp3LM5xaC6xbUDlX2CbXucTkXAln2QOV1mSAPvfX9UxfTwri0ftDG1rHcMUxLDZ2pE -03JqKDTu9smoO91GbXWBcD3II4B0VCDAQjAd3ejk5204yXb4XO8KpzVdjOrG9UNHKihXx+cI7mF8 -vwa/dneq43xUd0bR9OcGbQ7USw7Czb4Dtxp5IZHtJqE99YYPtrgAXBLb3//FI/p3s8hs96NdfrVt -9bK3DIt9WUw8xHyMFonM4wiMDOjNIWlrzFY3go63gDR0dBmqmRvyBTp+lMyI1x7TBoOc2Yn2AKxR -CP4PNIke9w== +eJzFPf1z2zaWv/OvwMqToZTIdOK0vR2nzo2TOK3v3MTbpLO5dT1aSoIs1hTJEqRl7c3d337vAwAB +kpLtTXdO04klEnh4eHhfeHgPHQwGJ0Uhs7lY5fM6lULJuJwtRRFXSyUWeSmqZVLO94u4rDbwdHYT +X0slqlyojYqwVRQET7/yEzwVn5eJMijAt7iu8lVcJbM4TTciWRV5Wcm5mNdlkl2LJEuqJE6Tf0CL +PIvE06/HIDjLBMw8TWQpbmWpAK4S+UJcbKplnolhXeCcX0Tfxi9HY6FmZVJU0KDUOANFlnEVZFLO +AU1oWSsgZVLJfVXIWbJIZrbhOq/TuSjSeCbF3//OU6OmYRiofCXXS1lKkQEyAFMCrALxgK9JKWb5 +XEZCvJGzGAfg5w2xAoY2xjVTSMYsF2meXcOcMjmTSsXlRgyndUWACGUxzwGnBDCokjQN1nl5o0aw +pLQea3gkYmYPfzLMHjBPHL/LOYDjxyz4JUvuxgwbuAfBVUtmm1IukjsRI1j4Ke/kbKKfDZOFmCeL +BdAgq0bYJGAElEiT6UFBy/G9XqHXB4SV5coYxpCIMjfml9QjCs4qEacK2LYukEaKMH8np0mcATWy +WxgOIAJJg75x5omq7Dg0O5EDgBLXsQIpWSkxXMVJBsz6UzwjtP+aZPN8rUZEAVgtJX6rVeXOf9hD +AGjtEGAc4GKZ1ayzNLmR6WYECHwG7Eup6rRCgZgnpZxVeZlIRQAAtY2Qd4D0WMSl1CRkzjRyOyb6 +E02SDBcWBQwFHl8iSRbJdV2ShIlFApwLXPH+48/i3embs5MPmscMMJbZ6xXgDFBooR2cYABxUKvy +IM1BoKPgHP+IeD5HIbvG8QGvpsHBvSsdDGHuRdTu4yw4kF0vrh4G5liBMqGxAur339BlrJZAn/+5 +Z72D4GQbVWji/G29zEEms3glxTJm/kLOCL7XcF5HRbV8BdygEE4FpFK4OIhggvCAJC7NhnkmRQEs +liaZHAVAoSm19VcRWOFDnu3TWrc4ASCUQQYvnWcjGjGTMNEurFeoL0zjDc1MNwnsOq/ykhQH8H82 +I12UxtkN4aiIofjbVF4nWYYIIS8E4V5IA6ubBDhxHolzakV6wTQSIWsvbokiUQMvIdMBT8q7eFWk +cszii7p1txqhwWQlzFqnzHHQsiL1SqvWTLWX9w6jLy2uIzSrZSkBeD31hG6R52MxBZ1N2BTxisWr +WufEOUGPPFEn5AlqCX3xO1D0RKl6Je1L5BXQLMRQwSJP03wNJDsKAiH2sJExyj5zwlt4B/8CXPw3 +ldVsGQTOSBawBoXIbwOFQMAkyExztUbC4zbNym0lk2SsKfJyLksa6mHEPmDEH9gY5xp8yCtt1Hi6 +uMr5KqlQJU21yUzY4mVhxfrxFc8bpgGWWxHNTNOGTiucXlos46k0LslULlAS9CK9sssOYwY9Y5It +rsSKrQy8A7LIhC1Iv2JBpbOoJDkBAIOFL86Sok6pkUIGEzEMtCoI/ipGk55rZwnYm81ygAqJzfcM +7A/g9g8Qo/UyAfrMAAJoGNRSsHzTpCrRQWj0UeAbfdOfxwdOPVto28RDLuIk1VY+zoIzenhaliS+ +M1lgr7EmhoIZZhW6dtcZ0BHFfDAYBIFxhzbKfM1VUJWbI2AFYcaZTKZ1goZvMkFTr3+ogEcRzsBe +N9vOwgMNYTp9ACo5XRZlvsLXdm6fQJnAWNgj2BMXpGUkO8geJ75C8rkqvTBN0XY77CxQDwUXP5++ +P/ty+kkci8tGpY3b+uwKxjzNYmBrsgjAVK1hG10GLVHxJaj7xHsw78QUYM+oN4mvjKsaeBdQ/1zW +9BqmMfNeBqcfTt6cn05++XT68+TT2edTQBDsjAz2aMpoHmtwGFUEwgFcOVeRtq9Bpwc9eHPyyT4I +JomafPcNsBs8GV7LCpi4HMKMxyJcxXcKGDQcU9MR4thpABY8HI3Ea3H49OnLQ4JWbIoNAAOz6zTF +hxNt0SdJtsjDETX+jV36Y1ZS2n+7PPrmShwfi/C3+DYOA/ChmqbMEj+ROH3eFBK6VvBnmKtREMzl +AkTvRqKADp+SXzziDrAk0DLXdvq3PMnMe+ZKdwjSH0PqAThMJrM0VgobTyYhEIE69HygQ8TONUrd +EDoWG7frSKOCn1LCwmbYZYz/9KAYT6kfosEoul1MIxDX1SxWklvR9KHfZII6azIZ6gFBmEliwOFi +NRQK0wR1VpmAX0uchzpsqvIUfyJ81AIkgLi1Qi2Ji6S3TtFtnNZSDZ1JARGHwxYZUdEmivgRXJQh +WOJm6UajNjUNz0AzIF+agxYtW5TDzx74O6CuzCYON3q892KaIab/wTsNwgFczhDVvVItKKwdxcXp +hXj5/HAf3RnYc84tdbzmaKGTrJb24QJWy8gDI8y9jLy4dFmgnsWnR7thriK7Ml1WWOglLuUqv5Vz +wBYZ2Fll8TO9gZ05zGMWwyqCXid/gFWo8Rtj3Ify7EFa0HcA6q0Iill/s/R7HAyQmQJFxBtrIrXe +9bMpLMr8NkFnY7rRL8FWgrJEi2kcm8BZOI/J0CSChgAvOENKrWUI6rCs2WElvBEk2ot5o1gjAneO +mvqKvt5k+Tqb8E74GJXucGRZFwVLMy82aJZgT7wHKwRI5rCxa4jGUMDlFyhb+4A8TB+mC5SlvQUA +AkOvaLvmwDJbPZoi7xpxWIQxeiVIeEuJ/sKtGYK2WoYYDiR6G9kHRksgJJicVXBWNWgmQ1kzzWBg +hyQ+151HvAX1AbSoGIHZHGpo3MjQ7/IIlLM4d5WS0w8t8pcvX5ht1JLiK4jYFCeNLsSCjGVUbMCw +JqATjEfG0RpigzU4twCmVpo1xf4nkRfsjcF6XmjZBj8AdndVVRwdHKzX60hHF/Ly+kAtDr7983ff +/fk568T5nPgHpuNIiw61RQf0Dj3a6HtjgV6blWvxY5L53EiwhpK8MnJFEb8f6mSei6P9kdWfyMWN +mcZ/jSsDCmRiBmUqA20HDUZP1P6T6KUaiCdknW3b4Yj9Em1SrRXzrS70qHLwBMBvmeU1muqGE5R4 +BtYNduhzOa2vQzu4ZyPND5gqyunQ8sD+iyvEwOcMw1fGFE9QSxBboMV3SP8zs01M3pHWEEheNFGd +3fOmX4sZ4s4fLu/W13SExswwUcgdKBF+kwcLoG3clRz8aNcW7Z7j2pqPZwiMpQ8M82rHcoiCQ7jg +WoxdqXO4Gj1ekKY1q2ZQMK5qBAUNTuKUqa3BkY0MESR6N2azzwurWwCdWpFDEx8wqwAt3HE61q7N +Co4nhDxwLF7QEwku8lHn3XNe2jpNKaDT4lGPKgzYW2i00znw5dAAGItB+cuAW5ptysfWovAa9ADL +OQaEDLboMBO+cX3Awd6gh506Vn9bb6ZxHwhcpCHHoh4EnVA+5hFKBdJUDP2e21jcErc72E6LQ0xl +lolEWm0Rrrby6BWqnYZpkWSoe51FimZpDl6x1YrESM1731mgfRA+7jNmWgI1GRpyOI2OydvzBDDU +7TB8dl1joMGNwyBGq0SRdUMyLeEfcCsovkHBKKAlQbNgHipl/sT+AJmz89VftrCHJTQyhNt0mxvS +sRgajnm/J5CMOhoDUpABCbvCSK4jq4MUOMxZIE+44bXcKt0EI1IgZ44FITUDuNNLb4ODTyI8ASEJ +Rch3lZKFeCYGsHxtUX2Y7v5DudQEIYZOA3IVdPTi2I1sOFGN41aUw2doP75BZyVFDhw8BZfHDfS7 +bG6Y1gZdwFn3FbdFCjQyxWEGIxfVK0MYN5j8p2OnRUMsM4hhKG8g70jHjDQK7HJr0LDgBoy35u2x +9GM3YoF9h2GuDuXqDvZ/YZmoWa5Cipm0YxfuR3NFlzYW2/NkOoA/3gIMRlceJJnq+AVGWf6JQUIP +etgH3ZsshkXmcblOspAUmKbfsb80HTwsKT0jd/CJtlMHMFGMeB68L0FA6OjzAMQJNQHsymWotNvf +BbtzigMLl7sPPLf58ujlVZe4420RHvvpX6rTu6qMFa5WyovGQoGr1TXgqHRhcnG20YeX+nAbtwll +rmAXKT5++iKQEBzXXcebx029YXjE5t45eR+DOui1e8nVmh2xCyCCWhEZ5SB8PEc+HNnHTm7HxB4B +5FEMs2NRDCTNJ/8MnF0LBWPszzcZxtHaKgM/8Pq7byY9kVEXye++GdwzSosYfWI/bHmCdmROKtg1 +21LGKbkaTh8KKmYN69g2xYj1OW3/NI9d9ficGi0b++5vgR8DBUPqEnyE5+OGbN2p4sd3p7bC03Zq +B7DObtV89mgRYG+fT3+DHbLSQbXbOEnpXAEmv7+PytVs7jle0a89PEg7FYxDgr79l7p8DtwQcjRh +1J2OdsZOTMC5ZxdsPkWsuqjs6RyC5gjMywtwjz+7ULUFM4z7nI8XDntUkzfjPmfia9Qqfv4QDWSB +eTQY9JF9Kzv+f8zy+b9mkg+cijm5/gOt4SMB/VEzYePB0LTx8GH1L7trdw2wB5inLW7nDrewOzSf +VS6Mc8cqSYmnqLueijWlK1BsFU+KAMqc/b4eOLiM+tD7bV2WfHRNKrCQ5T4ex44FZmoZz6/XxOyJ +gw+yQkxssxnFqp28nrxPjYQ6+mxnEjb7hn45W+YmZiWz26SEvqBwh+GPH386DftNCMZxodPDrcjD +/QaE+wimDTVxwsf0YQo9pss/L1XtrYtPUJMRYCLCmmy99sEPBJs4Qv8a3BMR8g5s+Zgdd+izpZzd +TCSlDiCbYlcnKP4WXyMmNqPAz/9S8YKS2GAms7RGWrHjjdmHizqb0flIJcG/0qnCmDpECQEc/luk +8bUYUuc5hp40N1J06jYutfdZlDkmp4o6mR9cJ3Mhf6/jFLf1crEAXPDwSr+KeHiKQIl3nNPASYtK +zuoyqTZAgljl+uyP0h+chtMNT3ToIcnHPExATIg4Ep9w2vieCTc35DLBAf/EAyeJ+27s4CQrRPQc +3mf5BEedUI7vmJHqnsvT46A9Qg4ABgAU5j8Y6cid/0bSK/eAkdbcJSpqSY+UbqQhJ2cMoQxHGOng +3/TTZ0SXt7Zgeb0dy+vdWF63sbzuxfLax/J6N5auSODC2qCVkYS+wFX7WKM338aNOfEwp/Fsye0w +9xNzPAGiKMwG28gUp0B7kS0+3yMgpLadA2d62OTPJJxUWuYcAtcgkfvxEEtv5k3yutOZsnF0Z56K +cWe35RD5fQ+iiFLFptSd5W0eV3HkycV1mk9BbC264wbAWLTTiThWmt1OphzdbVmqwcV/ff7x4wds +jqAGJr2BuuEiomHBqQyfxuW16kpTs/krgB2ppZ+IQ900wL0HRtZ4lD3+5x1leCDjiDVlKOSiAA+A +srpsMzf3KQxbz3WSlH7OTM6HTcdikFWDZlJbiHRycfHu5PPJgEJ+g/8duAJjaOtLh4uPaWEbdP03 +t7mlOPYBodaxrcb4uXPyaN1wxP021oDt+PCtB4cPMdi9YQJ/lv9SSsGSAKEiHfx9DKEevAf6qm1C +hz6GETvJf+7JGjsr9p0je46L4oh+37FDewD/sBP3GBMggHahhmZn0GymWkrfmtcdFHWAPtDX++ot +WHvr1d7J+BS1k+hxAB3K2mbb3T/vnIaNnpLVm9Mfzj6cn725OPn8o+MCoiv38dPBoTj96Yug/BA0 +YOwTxZgaUWEmEhgWt9BJzHP4r8bIz7yuOEgMvd6dn+uTmhWWumDuM9qcCJ5zGpOFxkEzjkLbhzr/ +CDFK9QbJqSmidB2qOcL90orrWVSu86OpVGmKzmqtt166VszUlNG5dgTSB41dUjAITjGDV5TFXpld +YckngLrOqgcpbaNtYkhKQcFOuoBz/mVOV7xAKXWGJ01nregvQxfX8CpSRZrATu5VaGVJd8P0mIZx +9EN7wM149WlApzuMrBvyrLdigVbrVchz0/1HDaP9XgOGDYO9g3lnktJDKAMbk9tEiI34JCeUd/DV +Lr1eAwULhgd9FS6iYboEZh/D5losE9hAAE8uwfriPgEgtFbCPxA4cqIDMsfsjPDtar7/l1ATxG/9 +6689zasy3f+bKGAXJDiVKOwhptv4HWx8IhmJ04/vRyEjR6m54i81lgeAQ0IBUEfaKX+JT9AnQyXT +hc4v8fUBvtB+Ar1udS9lUeru/a5xiBLwRA3Ja3iiDP1CTPeysMc4lVELNFY+WMywgtBNQzCfPfFp +KdNU57ufvTs/Bd8RizFQgvjc7RSG43gJHqHr5DuucGyBwgN2eF0iG5fowlKSxTzymvUGrVHkqLeX +l2HXiQLD3V6dKHAZJ8pFe4jTZlimnCBCVoa1MMvKrN1qgxR22xDFUWaYJSYXJSWw+jwBvExPY94S +wV4JSz1MBJ5PkZOsMhmLaTIDPQoqFxTqGIQEiYv1jMR5ecYx8LxUpgwKHhabMrleVni6AZ0jKsHA +5j+dfDk/+0BlCYcvG6+7hznHtBMYcxLJMaYIYrQDvrhpf8hVk0kfz+pXCAO1D/xpv+LslGMeoNOP +A4v4p/2K69COnZ0gzwAUVF20xQM3AE63PrlpZIFxtftg/LgpgA1mPhiKRWLZi070cOfX5UTbsmVK +KO5jXj7iAGdR2JQ03dlNSWt/9BwXBZ5zzYf9jeBtn2yZzxS63nTebEt+cz8dKcSSWMCo29ofw2SH +dZrq6TjMto1baFurbeyvmRMrddrNMhRlIOLQ7TxymaxfCevmzIFeGnUHmPheo2sksVeVD37NBtrD +8DCxxO7sU0xHKmMhI4CRDKlrf2rwodAigAKh7N+hI7nj0dNDb46ONbh/jlp3gW38ERShzsWlGo+8 +BE6EL7+z48ivCC3Uo0cidDyVTGa5zRPDz3qJXuULf469MkBBTBS7Ms6u5ZBhjQ3MZz6xt4RgSdt6 +pL5MrvoMizgD5/RuC4d35aL/4MSg1mKETrsbuWmrI5882KC3FGQnwXzwZbwG3V/U1ZBXcss5dG8t +3Xao90PE7ENoqk/fhyGGY34Pt6xPA7iXGhoWeni/bzmF5bUxjqy1j62qptC+0B7srIStWaXoWMYp +TjS+qPUCGoN73Jj8gX2qE4Xs7546MScmZIHy4C5Ib24D3aAVThhwuRJXjiaUDt9U0+h3c3krUzAa +YGSHWO3wm612GEU2nNKbB/bV2F1sLjb9uNGbBrMjU46BnpkqYP2iTFYHiE5vxGcXZg0yuNS/6i1J +nN2Ql/z2r2dj8fbDz/DvG/kRTCkWP47F3wAN8TYvYX/J1bt0rQJWclS8ccxrhRWSBI2OKvgGCnTb +Ljw647GILjHxa0usphSYVVuu+NoTQJEnSBXtjZ9gCifgt6nsanmjxlPsW5SBfok02F7sggUiB7pl +tKxWKdoLJ0rSrObl4Pzs7emHT6dRdYccbn4OnCiKn5CF09FnxCWeh42FfTKr8cmV4zj/KNOix2/W +m05TOIObThHCvqSwG02+UiO2m4u4xMiBKDbzfBZhS2B5rtWr1uBIj5z95b2G3rOyCGs40qdojTeP +j4Ea4te2IhpAQ+qj50Q9CaF4ikVj/Dga9JvisaDQNvx5erOeu5FxXf1DE2xj2sx66He3unDJdNbw +LCcRXsd2GUxBaJrEajWduYWCHzOhb0QBLUfnHHIR12klZAaSS5t8upoCNL1b28cSwqzC5owK3ihM +k67jjXKSkGIlBjjqgKrr8UCGIoawB/8pvmF7gEWHouZaaIBOiNL+KXe6qnq2ZAnmLRFRryfxYJ1k +L918Hk1hHpR3yLPGkYV5otvIGF3LSs+fHwxHly+aTAeKSs+8yt5ZAVbPZZM9UJ3F06dPB+Lf7/d+ +GJUozfMbcMsAdq/Xck6vt1huPTm7Wl3P3ryJgB9nS3kJD64oem6f1xmFJnd0pQWR9q+BEeLahJYZ +TfuWXeagXckHzdyCD6y05fglS+jeIwwtSVS2+vooDDsZaSKWBMUQxmqWJCGHKWA9NnmNRXkYZtT8 +Iu+A4xMEM8a3eELGW+0lepiUQGu5x6JzLAYEeEC5ZTwaVTVTWRrgObnYaDQnZ1lSNfUkz93DU30X +QGWvM9J8JeI1SoaZR4sYTn2nx6qNh53vZFFvx5LPLt2AY2uW/Po+3IG1QdLyxcJgCg/NIs1yWc6M +OcUVS2ZJ5YAx7RAOd6ZbnMj6REEPSgNQ72QV5lai7ds/2XVxMf1I58j7ZiSdPlTZm7E4OBRnrQTD +KGrGpzCUJaTlW/NlBKN8oLC29gS8scSfdFAViwm8CzzcusY60xdzcP5Gc1sHwKHLoKyCtOzo6Qjn +BjILn5l2y3Ua+KEtOuF2m5RVHacTff/DBB22iT1Y13jaeridlZ7WWwEnPwcPeF+n7oPjYLJskJ6Y +emtKM47FQocoIrfEzK/GKnL08g7ZVwKfAikzn5jCaBNEurTsaitOdc6mo+IR1DNTxbTFMzflM53K +ExfzMeU5mbqHLV60waV9kYV4fSyGL8bi29ZGaFZs8GInQPnJPHoyD32fjLpeHh02dqa78WxB2Ark +5dWjp5smU5pe2Jdzfn9fnXSIG8AVyM4ikfP9JwqxY5y/FqqG0sxrO6fQjLEkfc9mPelq7KZGhUrR +puDVrxuF4qgW43/aQUyZt9YDXBGLQssWyFbxm8STVvKfvbcNEwM1ev7Koucy6Tucwm94Wwq81wR1 +HZ2th5Y6rd6C7dmT69pJPoJqGjYcf69H9ShRaueId1rh8WQjcS7rP4KHQ7pZhpjmWetY+F/JPJy0 +v+1wsYPld9/swtNVML1lEj0Lurt2gZe6XbDQLLf59Ie6PEbp6/pVAuNAaUQHvD5z+SP5a0eYD8y3 +uuQ2L3iF1yvSWS/allS6/gfvSfkeLXQIaBNO6VmwFuCS1As8mr2l2yJPFKWR4aUv3xy+GJtaWwak +J/AyevlMX6pI3cx1Ar6zOtabIHip+x1G/+YASyq/t33V2RbQtI5btyv5g4UUjxpFE0uHxnLcX1nR +rFks8BbChpjspNorNd6D2zAFh8FcJ5qD5wM7u6gPXVdjNNK7TbVtEeCtwUP72SY5D+raKFJEepew +bVOeuxTno0VB9+q3ILgXR85fxvwGfaq6OLKxKmNT8Cxx6OZH4qe66a3kYnuCxrW6CXdNn/vvmrtu +EdiZm/SAztz9ik2XBrrvdivaRwOOE2hCPKjooNH4/cbEtQNjnZXSH/PWHyS/2wlnusWs3AfG5MBg +BJ3YU2NvzP4qnrnfMcVqn684dgt0e52N1rQ7NqPN8Q/xFDidBJ/bmn3KEZprDuSNB91ZN+Gs04m8 +vlaTGO9LnNBulTKkOtsQs/95T9fdyVhtzLYFrwECEIabdC6rm64OjAG6ku9t5gQj574XQUNTGq6T +16uSOZsEvUcCcBGHHqm/CW1zYu4glRgxVnVZlLCtHOjbfTnzpS9ZuAFqImGrWN0Y1E2Psb7slRQr +pVuZol4OeLbSZoAIbMQ7pmEyse+AV543FxckY8sMMqtXsoyr5tIe/4w9Ea+dEaiMGxfXiXM1Utni +EhexxPKGgxRGmuz3Z7BD83anO24qGFlt93B2oh46dvqYSxAcY2S4OLmzF/a5F0XN6bJo1zu0zRqu +s5cUwTKY2+dIR+qgE7/VN2Lxra0cEkf/0uEfkHe3ltHP67bqjL1bi4bzzFUI3SuQsAafjHPfzYYd +DujeYdjaodrxfX1hGaXjYW5pbKmoffJehdOMNmpCMZiCeU8oxk+zf2QoxoP/wFCMvocSDI3GR+uB +3sT7e2I2rB7cSx0bRoA+EyASHgm3rgQ0pnLoprEXuUruBvaKZtaVTm2cMQ/Ikd3bvggEX96o3Jxf +73K1XaEYX7ro8Q/nH9+cnBMtJhcnb//z5AdKc8Jzh5atenCsKsv3mdr7XkK1G7fSqSl9gzfY9ty5 +ylVBGkLnfedUvwdCfwVY34K2FZn7eluHTiVNtxMgvnvaLajbVHYv5I5fpqs23ISUVuZzoJ9ymqr5 +5Zz1m0fmyIvFoTnSMu+bUwgto50g7baFcxJGu+pE+6v6Xs0tAeSRTVumFcDDB+Qve/ZgalBshJsd +lPb/OINyrbF+z9xJA1I4k87diHQtIoOq/P9DRwnKLsa9HTuKY3vbNbXjcxZlr3HHQ9SZjAxBvAK6 +QXd+rrDPZbqFCkHACk/f/MeIGP2nTybtOf4TJS73qVR3H5XNlf2Fa6ad278meFpf2Ru0FKf88Hkl +NF7UqXsCb/t0OpDTR8c6+cKpDQHNdwB0bsRTAXujv8QKcboRIWwctUuG6aZER339nYM82k0He0Or +52J/WyGnW8goxIvtDeetWknd45B7qHt6qNqUyzkWGPMet1VoitcEmc8FBV2Z5TkfeBitt/3w9fby +xZGN0iO/42tHkVB+1sAx7JdOfuPOaxqd7sQs5ZgS4HCv5tT36hZXDlT2CbbtbTpFHlv2PyZhgCEN +vPf9ITPTw7vMftDG1LLeEUxJDJ+oEU3LKYvRuNsno+50G7XVBcIlPg8A0lGBAAvBdHSjk3K54bzp +4XO9G5zWdMGte1QTOlJB6Vc+R3AP4/s1+LW7U2nug7oziqY/N2hzoF5yEG72HbjVyAuFbDcJ7ak3 +fLDFBeAq5/7+Lx7Qv5sYaLsf7vKrbauXvZV17MtiLimm2LRIZB5HYGRAbw5JW2MBghF0vNiloaPL +UM3ckC/Q8aP8VLy+mjYY5MxOtAdgjULwf2RtvCc= """) ##file ez_setup.py @@ -2121,93 +2138,93 @@ ##file distribute_setup.py DISTRIBUTE_SETUP_PY = convert(""" -eJztPF1z2ziS7/oVOLlcpHISE2fm5q5cp6nKTDyzrs0mqTjZfUhcMkRCEsf8GpC0ov31190ACICk -ZOdm9uGqzrtjS0Sj0ejvboA5+7fq0OzKYjKdTn8qy6ZuJK9YksLfdN02gqVF3fAs400KQJPrDTuU -LdvzomFNydpasFo0bdWUZVYDLI5KVvH4nm9FUKvBqDrM2W9t3QBAnLWJYM0urSebNEP08AWQ8FzA -qlLETSkPbJ82O5Y2c8aLhPEkoQm4IMI2ZcXKjVrJ4L+8nEwY/GxkmTvUr2icpXlVygapXVlqCd5/ -FM4GO5Ti9xbIYpzVlYjTTRqzByFrYAbSYKfO8TNAJeW+yEqeTPJUylLOWSmJS7xgPGuELDjw1ADZ -Hc9p0RigkpLVJVsfWN1WVXZIi+0EN82rSpaVTHF6WaEwiB93d/0d3N1Fk8lHZBfxN6aFEaNgsoXP -NW4llmlF29PSJSqrreSJK88IlWKimVfW5lO9a5s0674duoEmzYX5vCly3sS7bkjkFdLTfefS/Qo7 -qrisxWTSCRDXqI3ksnI7mTTycGmFXKeonGr4083Vh9XN9cerifgaC9jZNT2/QgmoKR0EW7K3ZSEc -bGYf7Ro4HIu6VpqUiA1bKdtYxXkSPuNyW8/UFPzBr4AshP1H4quI24avMzGfsX+noQ5OAjtl4aCP -YmB4SNjYcsleTI4SfQZ2ALIByYGQE7YBISmC2Mvouz+VyDP2e1s2oGv4uM1F0QDrN7B8AapqweAR -YqrAGwAxOZIfAMx3LwO7pCELEQrc5swf03gC+B/YPowPhx22BdPzehqwcwQcwGmY/pDe9GdLAbEO -PugV69u+dMo6qisORhnCp/erf7y6/jhnPaaxZ67MXl/98urTm4+rv199uLl+9xbWm76Ifoi+u5h2 -Q58+vMHHu6apLp8/rw5VGilRRaXcPtc+sn5egx+LxfPkuXVbz6eTm6uPn95/fPfuzc3ql1d/vXrd -Wyi+gIVcoPd//XV1/faXdzg+nX6Z/E00POENX/xdeatLdhG9mLwFN3vpWPikGz2vJzdtnnOwCvYV -fiZ/KXOxqIBC+j551QLl0v28EDlPM/XkTRqLotagr4XyL4QXHwBBIMFjO5pMJqTG2hWF4BrW8Hdu -fNMK2b4MZzNjFOIrxKiYtJXCgYKnwSavwKUCD4y/ifL7BD+DZ8dx8CPRnssiDK4sElCK8zqY68kK -sMyS1T4BRKAPW9HE+0Rj6NwGQYEx72BO6E4lKE5EKCcXlZUozLYszErvQ+/ZmxzFWVkLDEfWQrel -JhY33QWODgAcjNo6EFXxZhf9BvCasDk+zEC9HFo/v7idDTeisNgBy7C35Z7tS3nvcsxAO1RqoWHY -GuK47gbZ607Zg5nrX4qy8TxaYCI8LBdo5PDxmascPQ9j17sBHYbMAZbbg0tje1nCx6SVRnXc3CZy -6OhhEYKgBXpmloMLB6tgfF0+iP4kVM60iUsIo8Z1v/QAtL9RDzdpAauP6ZNSP4tbhdxI5o0UotM2 -bTjrNgVwsd2G8N+cdfbTlCsE+3+z+T9gNiRDir8FAymOIPqpg3BsB2GtIJS8LaeOmdHid/y9xniD -akOPFvgNfkkH0Z+ipGp/Su+N7klRt1njqxYQooC1EzDyAIOqm5qGLQ2Sp5BTX7+jZCkMfi7bLKFZ -xEdlrdstWqe2kQS2pJPuUOfv8y4NX615Lcy2nceJyPhBr4qM7iuJhg9s4F6c14vqcJ5E8H/k7Ghq -Az/nzFKBaYb+AjFwU4KGjTy8uJ09nT3aaIDgbi9OiXBk/8do7f0c4ZLVukfcEQFSFonkgwcWsglf -zJmVv87H/ULNqUrWpkw1KcOKCoIlGY6Sd68o0jte9pK2HgeWTuI2yg21gyUaQCtHmLC8+I85CGe1 -4fdi+VG2ovO9OScHULdQSe4pnScd5eu6zNCMkRcTu4SjaQCCf0OXe3terxSXBPraoLrfrsCkKI+s -Ka1G/uZl0maixtLuS7ebwHKlDzj0094XRzTeej6AUs4dr3nTyNADBENZJU7UHy0LcLbm4HhdQEN+ -yd4H0c7BVlMdxLFCq5upovMf8RbHmecxI9J9hXBqWfLjcgp1mV5vNkJYfx8+Rp3K/1wWmyyNG39x -AXqi6pmY/Ek4A4/SF52rV0Pu43QIhZAFRXsJxXc4gJh+JN9OG0vcNonTTgp/XJ5DEZXWJGr+ACUE -VVdfiukQH3Z/Yl4EDSZS2tgB836HnQ1qCelOBnySbYHxJWLvMwECGsVnuh2c5aVEUmNMCw2hm1TW -zRyME9CMTg8A8cE4Hbb45OwriEbgvxRfivDnVkpYJTsoxOxczgC5FwFEhFksZhZDZVZCS5vwpT8m -snrEQkAHWc/oHAv/3PMUtzgFYzP1osr7YwX2t9jDk6LIMZsZ1esu24FV35bNL2VbJH/YbB8lc4zE -QSp0ymGtYil4I/r+aoWbIwvssiyKWCcC9R8NW/QzErt0yNKOGIr017Yt2dkrhdau+QnGl5Ux1UvU -mtWcTxvVbSx4LlTWeKdpv4OskJKzNbZQH3iWetiN6RVtvhYSTJqTLXdugXBhy5KyYmrjdL1TUAOa -Itidx487ho2XEJxEvDOriyJRkRP7ypwFz4NZxO4UT+5wRa84AAcjpDBZZFfJmVVEEqk9Ege76XoP -1BWOyyKh/mzFMdavxQb9DbZi46blme0S0/4aLLWayIjhX5IzeOGIhNpKqMTXFIgEtuZ1j1xmWHdN -HHMcDZcOipdjc5vtP1eoDtiP8vLjCOu07T/RA2rpq0a89NJVFCQEQ4NFpYD8QQBLj2ThBlQnmDJG -dLAv3e91zLWXOiu0s0vk+auHMkWtrtB0k44cm+QMonpXv3TWQ06+ns5xS77PVkRpLoWD4TP2QfDk -OQVXhhEG8jMgna3B5O7neCqwRyXEcKh8C2hyXEoJ7oKsr4cMdktabewlxfOZRhC8UWHzg51CzBBk -DPrAk15SpdhIRCtmzdl0v54OgHRegMjs2MBpaknAWiM5BhBgavgePOAfiXewqAtv27kkYdhLRpag -ZWyqQXDYNbivdfk13LRFjO5Me0Eadsep6Ttnz57d72cnMmN1JGFrFD3dWMZr41pu1PNTSXMfFvNm -KLXHEmak9iEtVQNr0Px3fype14OB/koRrgOSHj7vFnkCjg4WMB2fV+HpEJUvWCg9IbWxE37hAPDk -nL4/77gMtfIYjfBE/6g662WGdJ9m0KgIRtO6cUhX6129NZpOZK3QO4RoCHNwGOADisYG/X9QdOPx -fVuRv9io3FoUaksQ201IIn8J3m2lcRifgIhnrt8Adgxhl2Zpy6Iz8HI47WC4N9L2euVDuA1XvW2r -DnbWe4TGaiAyEyChxOiwIndAFKuUzt0EWNo+GAuX2rEZ3o0ng5sxT0TKPXHEAOu57sUZ6bwTnoUb -vo1KzXi5PvMdJhtcg10rDIXYm+iMTyHSBtG7N6+j8xrP2vAcN8Jfg/bvB0SnAhxmN9R2VBQajLoP -jAUufg3HRjX95qGlNS8fIGEG41i5nfmwyngsdqDuwnSze5E8rbEfOQTzif9U3EMs9Jr+kHvpTThz -jyvYBmsPzwNhRmruMTjN4nFSgGp9LB7pvyHOnbtdmWfYN1xggdB3+Gbxgb9cg/TvXbZs/BLJcsD2 -SSmLd8/63XV7DJj0lOBv5QOqgMiEOigu2wazXnQee36wJmcqnX7G5jBnzpTma+J78tTzHT5YZ64N -B4heebDKU3kRZDBJuUM9Y85GTlF171vzc+DbLS/ADnjfQ82ZT82oKp0B5j3LRBPUDNW+8719fnZq -pvmNmha6bbx5rwGom/x4PwI/OtwzGE7JQ8N4Z3L9XrMG6dW7rqsZYBnG9DGtBJ+qmvfAVkOs5sSR -VnpwY28fJU6jIOjtxHfHxzxN3zkfg+tcNd9AQt2dXCMBmitOAEOQ7p5N17vujMQyHwsWwIAHZ+D+ -8xyoWJXr38Lu2HMWmYZ3BUUhVF4qsj3WaPB8myb8W+Z4LtelF5RypJ56zA2PiNtwx/QWhi6IWHV4 -ICaB0elAFT757EQVhXajOhQ7dqSPbmrrB2GBL57WhceuMMwVbd/g9nqkDDyg4eXQBY76HgV+wvP0 -ffjPKH8VyAez/NynS5A6f9klSTr1vioeUlkWaGy9/NstjrFs3UEZxioh87SuzQ02Ve6eY6fyPq0q -oGl6YhtD+nRuNurECeB4nqbE1XSJ2XFxOXoSwYSgnxf12NnsHKlaDurHj6WZHhlOw66vM4/v7zEz -7/m7J7mTycyvLboIbLPLMx3XIBzG96jVKX4by/WP2orKxq9+/XWBksR4BlJVn7/BVtJBNn0y6B8L -UE8N8lZPnUB/pPAA4vP7jm/+o5OsmD3iZR7l3CmL/tNMy2GFVwJpbRmvgvSgvdhCbdMuvA5C60+q -rXo0to6cFWrM1DteVVJs0q+hiTo20HURl8KUPiblcvtw2fNHNhnXlw4N4GfzAUJ2Ir46MRxqrYvL -2y6ro+G5uZwoijYXkqtri24vB0HVtV+V/y0WEnarbm6obfTLBdgG4IhgVdnU2PdGPV5iUFN4RhpF -TVlp4dDMKkubMMB1lsHs86J3XugwwTDQXUzj6h9aKaqwUFVUjB4CZ6Cc6q7lj4o/4z0tj9z6M0Ei -d4d0fiutlkpgb1sLGdBph71ErI8vsbM82kMaW6WbPWIdSisH6tpX+JuY0yGncxZqrpGOGfDR4/pT -PbMzthcBWFUMJIwkHU6+DSrp3ERKSqGYUguRY2B3j2yHbRv6ukeT8YsXfVcK2TDckBOOMFOGyfs6 -wizSP4v2MX5QB9KYnkR0ybxXPUlBoR7Hl+S2fZ31Up2Ph0oM+IVNU+dM69X7638lwZY6W6T2lwH1 -9FXTvY/mvrDhlkyqbTAuqDOWiEboe38Yz/GuQBcUUW+TfobdnRMu++RFZqiv3e6LJE5RppYGXTfN -mpFVNC/o1EP5RlRP8o3pVyK2kuVDmohEvVOSbjS8+/ZK7bRGEn1lMJ/bUxfTEHXrIT+UjFE2LgWN -DRg67xMMiNRhzdhl2aFvU/fogZYdVEfHKygvMwMbVXKs3QuHeksjm4hEkeggQvfajmyqWKj7iFZ4 -Hh1o7ce7fKNSNZM1aYBjzN+ONH2cK6vHSTqWRI2Qcjqn0iSGx1JS1Dm/W/INaenRvPREb7zHG3/e -sDvu6kZ3tohmTQfgykPSYbTj/QvRF61fEPxReQ7phZiUV0CkcJr6GW+LeGczO/ukHzw/6BFv4xjt -VFlK73opCOpJmJeBFFSVVizn8h5vHJSM0zExtxPW7VYXT3lyge+eBIvYv7AOiQRe/8nEQrcmFuIr -vQ4GCfQi5wXE8CS47ZC8PIZEiriUBlK/j0MJ5+V3t5iwKArAlYwNvHRCqRl+cdv1QbBd6Cazn/03 -YG4huTLTJgYH3U0afbmpE4lzYbsW2UadGCynEdT5ucA7E/USo5U9ktKXzOkMXEOoA1a6/yBBhEpe -+DVW16vMHWuzP3uXA709vppX7gus5PMywZf4VGTBMw4CcHsS9rDSIElBvanTB4qU1BG7ww0E3Z0Y -fKMOkG4EETK4Yg6Eag7AR5isdxSgj1dJMM+IiBzfkKR7MsBPIplanwYPni1o+4DotD6wrWg0rnDm -Xx7RiV9cVgf3O1R9UFvo+5CKoeqqvQHQjLeXJl0OgD7cdhmHEcsg0zADGPWzzaSrc2Al8rQQqzSI -V6brYd3573m8M0OYR4++y1PzjUCpit6NBgsZ8QrK3STUa/hO0tC1JG5F+OskIN6lw17R99//l0qL -4jQH+VF9BgS++M8XL5zsL9tEWvYGqdL+Ll35INAdCFYj+12aXft2m5nsv1n4cs6+d1iERobzhQwB -w8Uc8bycjdYlcV4RTIQtCQUY2XO5Pt8QaagwjwNIRX04duoyQHQvDkujgRHedAD9RZoDJCCYYSJO -2NTNacMgSArpkgvg6ky4M1vUXZIHZol95vW0zhn3iKTzz9EmipG4z6DBtQGScrwD4qyMNd7ZELCl -c9UnAMY72NkJQNN8dUz2f3HlV6koTs6A+xkU3BfDYpsuVPcK+bErGoRslay3ISjhVPsWfLUQL3uJ -3vtK7gtcoX6j2YYA+vtT9zKHfSsVvGmgX4I1MYt13ZrSvOXTFWO6PPa9o7Oy8mqaGZqKCCt+Q5/n -pY4Y4w/HMrSp6h6YO9E1e29e3/0BQzTko0L2rlGpy+s3h7oR+RXG1gsnaXIIN07NNCi8poIL2DVr -wbQUs3tcfo8jKpaqQyeINIVwOk61B06I6Lahfmc7ekdQhEZqV6CAIp4kK4XD1ruGYLyAWjfLwGU2 -POR092YZ1A22/hpwBQS54W2my3N7x3Unsmpp0iO0cWI2vRiu5c7CU6yfBU+h1lygW+CdxI5s76Zi -gJlMwx+4XE4/fXgztSQaykfv6Cr6zT8LgEkN3lylwKxvoJb2+t64YusdaEHNTeamd+QK3SSyJfBH -5xydUXHsom4L4HjiqpERP2lQzsExHrmRbDXq+tS/J0A++4rXBw1lVMr8ewZLX01V/+fkq0z+RWhj -v95TzzCGLxmf8kbgsVK6Doi12oragasV8mG10i+8dxkwcQcm/A9nRa43 +eJztPGtz2ziS3/UrcHK5SOUkxs7MzV25TlOVmTizrs0mKdvZ/ZC4aIiEJI75GpC0ov311403SEp2 +LrMfruq8O7ZENBqNfncDzMm/1ft2W5WT6XT6S1W1TctpTdIM/marrmUkK5uW5jltMwCaXK3JvurI +jpYtaSvSNYw0rO3qtqryBmBxlJOaJg90w4JGDkb1fk5+75oWAJK8Sxlpt1kzWWc5oocvgIQWDFbl +LGkrvie7rN2SrJ0TWqaEpqmYgAsibFvVpFrLlTT+i4vJhMDPmleFQ30sxklW1BVvkdrYUivg/Ufh +bLBDzv7ogCxCSVOzJFtnCXlkvAFmIA126hw/A1Ra7cq8oumkyDiv+JxUXHCJloTmLeMlBZ5qILvj +uVg0Aai0Ik1FVnvSdHWd77NyM8FN07rmVc0znF7VKAzBj/v7/g7u76PJ5BbZJfibiIURIyO8g88N +biXhWS22p6QrqKw3nKauPCNUioliXtXoT822a7PcfNubgTYrmP68LgvaJlszxIoa6THfKXe/wo5q +yhs2mRgB4hqNllxebSaTlu8vrJCbDJVTDn+6ubyOb65uLyfsa8JgZ1fi+SVKQE4xEGRJ3lclc7Dp +fXQr4HDCmkZqUsrWJJa2ESdFGr6gfNPM5BT8wa+ALIT9R+wrS7qWrnI2n5F/F0MGjgM7eemgjxJg +eCiwkeWSnE0OEn0CdgCyAcmBkFOyBiFJgsir6Ic/lcgT8kdXtaBr+LgrWNkC69ewfAmqasHgEWKq +wRsAMQWSHwDMD68Cu6QmCxEy3ObMH1N4Avgf2D6MD4cdtgXT02YakFMEHMApmP6Q2vRnS4FgHXxQ +KzZ3felUTdTUFIwyhE8f43+8vrqdkx7TyAtXZm8u377+9O42/vvl9c3Vh/ew3vQs+in64cepGfp0 +/Q4fb9u2vnj5st7XWSRFFVV881L5yOZlA34sYS/Tl9ZtvZxObi5vP328/fDh3U389vVfL9/0FkrO +z6cTF+jjX3+Lr96//YDj0+mXyd9YS1Pa0sXfpbe6IOfR2eQ9uNkLx8InZvS0mdx0RUHBKshX+Jn8 +pSrYogYKxffJ6w4o5+7nBStolssn77KElY0CfcOkfxF48QEQBBI8tKPJZCLUWLmiEFzDCv7OtW+K +ke3LcDbTRsG+QoxKhLaKcCDhxWBb1OBSgQfa30TFQ4qfwbPjOPiRaEd5GQaXFgkoxWkTzNVkCVjl +abxLARHow4a1yS5VGIzbEFBgzFuYE7pTBRQVREgnF1U1K/W2LEys9qH27E2OkrxqGIYja6GbShGL +mzaBwwCAg5FbB6Jq2m6j3wFeETbHhzmol0Pr57O72XAjEosdsAx7X+3IruIPLsc0tEOlEhqGrSGO +KzNI3hhlD2aufymr1vNogY7wsFygkMPHF65y9DyMXe8GdBgyB1huBy6N7HgFH9OOa9Vxc5vIoaOH +hTEBzdAzkwJcOFgFoavqkfUnoXJmbVJBGNWu+5UHoPyNfLjOSlh9TJ+k+lncMuRGvGg5Y0bblOGs +ugzA2WYTwn9zYuynrWIE+3+z+T9gNkKGIv6WBKQ4gugXA+HYDsJaQUh5W04dMqPFH/h7hfEG1UY8 +WuA3+MUdRH+Kksr9Sb3XusdZ0+Wtr1pAiARWTkDLAwyqaRsxbGngNIOc+uqDSJbC4Neqy1MxS/BR +Wutmg9apbCSFLamkO1T5+9yk4fGKNkxv23mcspzu1arI6L6SKPjABu7FabOo96dpBP9Hzo6mNvBz +SiwVmGaoLxAD1xVo2MjD87vZ89mjjAYINntxSoQD+z9Ea+/nAJes1j3hjgSgyCKRfPDAjLfh2ZxY ++at83C/UnKpkpctUnTLEoiBYCsOR8u4VRWrHy17S1uPA0kncRrkhd7BEA+j4CBOW5/8xB+HEa/rA +lre8Y8b3FlQ4gKaDSnIn0nmho3TVVDmaMfJiYpdwNA1A8G/ocm9Hm1hyiaGvDeqHTQwmJfLIRqTV +yN+iSrucNVjafTG7CSxX+oBDP+19cUTjrecDSOXc0oa2LQ89QDCUOHWi/mhZgLMVB8frAjHkl+x9 +EOUcbDVlIA4VWmamjM7f4y0OM89jRqT6CuHUsuTn5RTqMrXebISw/j58jCqV/7Uq13mWtP7iDPRE +1jOJ8CfhDDxKX3SuXg25j9MhFEIWFO04FN/hAGJ6K3y72FjqtkmcdlL48/IUiqisEaKmj1BCiOrq +Szkd4sPuT0LLoMVEShk7YN5tsbMhWkKqkwGfeFdifInIx5yBgEbx6W4HJUXFkdQE00JN6DrjTTsH +4wQ0o9MDQLzXTocsPjn7CqIR+C/llzL8teMcVsn3EjE55TNA7kUAFmEWi5nFUJml0LI2fOWPsbwZ +sRDQQdIzOsfCP/c8xR1OwdgselHVw6EC+1vs4VlR5JDNjOq1yXZg1fdV+7bqyvS7zfZJMsdIHKRC +xxxWnHBGW9b3VzFuTligybJExDoSqL83bImfkdilQpZyxFCkv7FtSWOvIrSa5icYX14lol4SrVnF ++ayV3caSFkxmjfeK9nvICkVytsIW6iPNMw+7Nr2yK1aMg0lTYcvGLQhc2LIUWbFo45jeKaiBmMLI +vcePe4KNlxCcRLLVq7MylZET+8qUBC+DWUTuJU/ucUWvOAAHwzjTWaSp5PQqLI3kHgUHzXS1B9EV +TqoyFf3ZmmKsX7E1+htsxSZtR3PbJRb7a7HUaiMthn9JzuCFIyHUjkMlvhKBiGFrXvXIeY5118Qx +x9Fw6aB4NTa33fwzRnXAfpSXH0dYp23+iR5QSV824rmXrqIgIRhqLDIFpI8MWHogC9egKsHkCaKD +fal+r2OuvdRZop1dIM9fP1YZanWNppsacmySM4jqpn4x1iOcfDOd45Z8ny2JUlwKB8Mn5JrR9KUI +rgQjDORnQDpZgck9zPFUYIdKiOFQ+hbQ5KTiHNyFsL4eMtit0GptLxmez7RMwGsV1j/YKcQMgSeg +DzTtJVWSjYJoyaw5me5W0wGQygsQmR0bOE0lCVhrJMcAAnQN34MH/CPxDhZ14W07V0gY9pILS1Ay +1tUgOOwG3Neq+hquuzJBd6a8oBh2x0XTd05evHjYzY5kxvJIwtYoarq2jDfatdzI58eS5j4s5s1Q +ao8lzEjtY1bJBtag+e/+1LRpBgP9lSJcByQ9fG4WeQYOAwuYDs+r8XRIlC9YKD0jtbET3lIAeHZO +3593WIZKebRGeKJ/Up3VMkO6jzNoVASjad04pKv1rt5qTRdkxegdQjSEOTgM8AFla4P+P0R0o8lD +Vwt/sZa5NSvlliC265C01k4AMc1UhAAXCg4vVmgBYu16kLVnncCm4YSlJsmy7gS8HyLZa66OtMNe ++xBuI1axw6qJnfURobFKiPQESDQxasTCTdiNeXsFC9wFY2FUOTzN0/EkcT3moYTSTxzxwHqu23FG +jNfCM3LNt1FpfreAFHFHhKRpGXBNUlCynY76+BQieBB9ePcmOm3wDA/PhyP8NWgrXyM6GTgxaxLt +TLlDjVH1l7Fwxq/h2KgiXz+0tBbVIyTiYHSx2/EP65wmbAtmxHSXvJchZA32OYdgPvGfygeIsd5h +AuR0ahPO3MMKusaaxvNsmOnq+xFOE3qcFKBaHbdH6m+Ic+dut+cF9iMXWHj0A4lefOCHV6AnDy5b +1n7pZTlg+6+iOnDvELjr9hgw6SnB36pHVAGWM3kAXXUtZtPolHZ0b01WV1D9TNBhzpxIy1HE9+Sp +5jt8sEFCGR4QHXuw0pq8yDSYJN2smjEnI6ezqqeu+DmIGZYXYAe07+HmxKdmVJVOAPOO5KwNGoJq +b3x6n59GzRS/UdNCtz047zUW1eEB3rvAjw73NIZj8lAw3llfv4etQHp1tOtqBliGucKYVoJPlocC +wFZNrOLEgRZ9cGNvNaVOAyLo7cR354c8Td+5H4Izrp6uIVE3J+JIgOKKEwARxNzfMT1xYySW+VgI +AQY8kAOPXhRARVytfg/Nceos0o30GopNqOhkZHyqgeH5NkX4t8zxXK5LLyjlSJ32lBseEbfmju5Z +DF2QYNX+UTAJjE4FqvDZZzKy2LQbVaHcsSN1JNRYPwgLfPG0Ljx0NWIuafsGt9cjZeABNS+HLnDU +90jwI56n78N/RfnLQD6Y5edOJlcx/tIkWSqlvywfM16VaGy9vN4turEc3kJ5R2rGi6xp9M04WUaf +Ygf0IatroGl6ZBtD+lRuN+rEBcDhPE+KqzWJ3WFxOXoSwYSgnxf12NluHalaDqrHT6WpHhlOI7Cv +M0/v7ykz7/m7Z7mTycyvWUwEttnliYprEA6TB9TqDL+N1QoHbUVm85e//bZASWI8A6nKz99gK9kg +Gz8a9A8FqOcGeaunTqA/ULgA8cWD4Zv/6CgrZk94mSc5d8yi/zTTcljhlVBKW8arKDVoL8yIdqwJ +r4PQ+ots1x6MrSNnkAqz6EnHNWfr7Guoo44NdCbiijCljl8p3zxe9PyRTcbVZUYN+Fl/gJCdsq9O +DIda6/zizmR1YniuLz2ysisYp/I6pNsjQlB5nVjmf4sFh93KGyFyG/1yAbYBOCJYlbcN9tNRj5cY +1CSekQZUW9VKOGJmnWdtGOA6y2D2edE7h3SYoBnoLqZw9Q/DJFVYqEoqRg+Xc1BOeYfzZ8mf8V6Z +R27zWUAid4d0fiutlkpgb9cwHohTFHs5WR2LYsd6tDc1toqZPWIdUisH6tpX+JuEisNT54xVX08d +M+CD1wCO9eJOyI4FYFUJkDCSdDj5Nqikc8MprZhkSsNYgYHdPQoetn3E1x2ajF+8qDtYyIbhhpxw +hJkyTN41EWaR/hm3j/FaHnRjehKJy+u96okzEepxfCnctq+zXqpzu6/ZgF/YjHXOyl5/vPpXEmyp +s0VqfxlQT1813Xtu7osgbskk2wbjgjohKWuZuk+I8RzvIJigiHqb9jNsc/647JMX6aG+drsvqDhF +mVwadF03a0ZWUbwQpynSN6J6Ct+YfRXE1rx6zFKWyndVsrWCd9+KaZzWSKquIhZze5qjG61uPeSH +kjHKxqWgsAFD532CAZE8BBq7hDv0bfJ+PtCyherocAXlZWZgo1KOjXuRUW1pZBMRK1MVRMR9uQOb +KhfynqMVnkcHWvvhLt+oVPVkRRrgGPO3I00f5yrsYZIOJVEjpBzPqRSJ4aGUFHXO75Z8Q1p6MC89 +0lvv8cafN+yuu7phzizRrMXBuvSQ4pDb8f4l64vWLwi+V55DeiEmFTUQyZxDgZx2ZbK1mZ190g+e +12rE2zhGO1mWinfIJIToSeiXjCRUndWkoPwBbzJUhIrjZ2onrLqNKp6K9BzfaQkWiX8RHhIJvFaU +s4VqTSzYV/GaGSTQi4KWEMPT4M4geXUICWdJxTWkes9HJJwXP9xhwiIpAFcyNvDKCaV6+OzO9EGw +Xegms5/9N2vuILnS0yYah7jzNPrSlBGJcxG8YflanhgspxHU+QXDuxjNEqOVPepSl9fF2bqCkAe3 +4l4FBxFKeeHXRF7b0ne39f7sHRH09vjKX7UrsZIvqhRfDpSRBc84BIDbk7CHoBpJBuotOn2gSGkT +kXvcQGDu2uCbeoB0zQQhg6vrQKjiAHyEyWpHAfp4mQTTXBBR4JuX4v4N8FOQLFqfGg+eLSj7gOi0 +2pMNaxWucOZfSlGJX1LVe/c7VH1QW6h7lpKh8gq/BlCMt5cxXQ6APtyZjEOLZZBp6AGM+vl6Yuoc +WEl4WohVCsQr09Ww6vz3PN6JJsyjR90RauiaoVRZ76aEhYxoDeVuGqo1fCep6VoKbkX46ygg3tHD +XtGPP/6XTIuSrAD5ifoMCDz7z7MzJ/vL15GSvUYqtd+kK9cM3QEjDbLfpdm1b7eZSf6bhK/m5EeH +RWhkOJ/xEDCczxHPq9loXZIUtYCJsCUhASN7LtfnGyINJeZxAC6pD8dOXQaIHth+qTUwwhsUoL9I +c4AEBDNMxAU2eSNbMwiSQnF5BnAZEzZmi7or5IFZYp95Pa1zxj0ixfnnaBNFS9xn0OA6gpBysgXi +rIwV3tkQsBPnqs8ATLawsyOAuvnqmOz/4iqxVFGcnAP3cyi4z4fFtrio3Svkx65+CGRxutqEoIRT +5VvwlUW8RMZ670G5L4aF6k1pGwLE31/MSyL2bVfwpoF6uVbHLGK6NZV+e8gUY6o89r2js7L0aooZ +iooIK35Nn+elDhjjT4cytKnsHui71g35qF8L/glDNOSjjPeuZ8lL8Tf7pmXFJcbWcydpcgjXTk03 +KLymggtomrVgWpLZPS5/xBEZS+WhE0Sakjkdp8YDF4jELUb1Lnj0QUAJNFy5AgkU0TSNJQ5b72qC +8WJr0y4Dl9nwkIo7PcugabH114IrEJBr2uWqPLd3Z7csr5c6PUIbF8wWL5wruZPwGOtnwXOo1Rfz +FnjX0ZDt3YAMMJNp6SPly+mn63dTS6KmfPTur6Rf/3MDmNTgjVgRmNXN1speCxxXbLUDJai5ztzU +jlyh60S2Av6onMMYFcUu6qYEjqeuGmnxCw0qKDjGAzedrUZdHft3CoTPvqTNXkFpldL/TsLSV1PZ +/zn6ipR/wVrbr/fUM4zhy8vHvBF4rExcM8RaLRbtwDhGPsSxepHeZMCCOzDhfwBqDMd7 """) ##file activate.sh @@ -2273,9 +2290,9 @@ ##file deactivate.bat DEACTIVATE_BAT = convert(""" -eJxzSE3OyFfIT0vj4spMU0hJTcvMS01RiPf3cYkP8wwKCXX0iQ8I8vcNCFHQ4FIAguLUEgUliIit -KhZlqkpcnCA1WKRsuTTxWBIZ4uHv5+Hv64piEVwU3TK4BNBCmHIcKvDb6xjigWIjkI9uF1AIu7dA -akGGW7n6uXABALCXXUI= +eJxzSE3OyFfIT0vj4ipOLVEI8wwKCXX0iXf1C7Pl4spMU0hJTcvMS01RiPf3cYmHyQYE+fsGhCho +cCkAAUibEkTEVhWLMlUlLk6QGixStlyaeCyJDPHw9/Pw93VFsQguim4ZXAJoIUw5DhX47XUM8UCx +EchHtwsohN1bILUgw61c/Vy4AJYPYm4= """) ##file activate.ps1 diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/script/virtualenv_support/distribute-0.6.31.tar.gz Binary file script/virtualenv/script/virtualenv_support/distribute-0.6.31.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/script/virtualenv_support/distribute-0.6.34.tar.gz Binary file script/virtualenv/script/virtualenv_support/distribute-0.6.34.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/script/virtualenv_support/pip-1.2.1.tar.gz Binary file script/virtualenv/script/virtualenv_support/pip-1.2.1.tar.gz has changed diff -r 41ce1c341abe -r 10a19dd4e1c9 script/virtualenv/script/virtualenv_support/pip-1.3.1.tar.gz Binary file script/virtualenv/script/virtualenv_support/pip-1.3.1.tar.gz has changed