diff -r 8ae3d91ea4ae -r 07f1c6854df9 script/lib/iri_tweet/iri_tweet/stream.py --- a/script/lib/iri_tweet/iri_tweet/stream.py Tue Dec 18 12:26:05 2012 +0100 +++ b/script/lib/iri_tweet/iri_tweet/stream.py Sun Apr 21 10:05:16 2013 +0200 @@ -9,7 +9,7 @@ ''' import time import requests -from requests.utils import stream_untransfer, stream_decode_response_unicode +from requests.utils import stream_decode_response_unicode import anyjson import select @@ -37,7 +37,7 @@ continue try: - chunk = req.raw.read(chunk_size) + chunk = req.raw.read(chunk_size, decode_content=True) if not chunk: break if len(chunk) >= chunk_size and chunk_size < max_chunk_size: @@ -54,7 +54,7 @@ req._content_consumed = True - gen = stream_untransfer(generate(), req) + gen = generate() if decode_unicode: gen = stream_decode_response_unicode(gen, req) @@ -312,6 +312,7 @@ self._max_wait = max_wait self._retry_wait = 0 self._retry_nb = 0 + self._reconnects = 0 def __post_process_error(self,e): # Note: error_cb is not called on the last error since we @@ -323,7 +324,9 @@ def __process_tcp_error(self,e): - if self._logger: self._logger.debug("connection error :" + str(e)) + if self._logger: self._logger.debug("connection error type :" + repr(type(e))) + if self._logger: self._logger.debug("connection error :" + repr(e)) + self._reconnects += 1 if self._max_reconnects >= 0 and self._reconnects > self._max_reconnects: raise ConnectionError("Too many retries") @@ -336,7 +339,9 @@ def __process_http_error(self,e): + if self._logger: self._logger.debug("http error type %s" % (repr(type(e)))) if self._logger: self._logger.debug("http error on %s : %s" % (e.response.url,e.message)) + if self._retry_wait < self._max_wait: self._retry_wait = 2*self._retry_wait if self._retry_wait > 0 else self._initial_http_wait if self._retry_wait > self._max_wait: @@ -360,15 +365,15 @@ if self._logger: self._logger.debug("Empty Tweet received : PING") continue yield tweet - except (ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.RequestException) as e: - self.__process_tcp_error(e) except requests.exceptions.HTTPError as e: - if e.response.status_code == 401 and self._retry_nb <= 1: + if e.response.status_code == 401: raise AuthenticationError("Error connecting to %s : %s" % (e.response.url,e.message)) if e.response.status_code > 200: self.__process_http_error(e) else: self.__process_tcp_error(e) + except (ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.RequestException) as e: + self.__process_tcp_error(e)