script/lib/tweetstream/tweetstream/streamclasses.py
author Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
Fri, 01 Jul 2011 10:15:32 +0200
changeset 13 79b6e132e3d7
child 14 10e7a0c7c64f
child 207 621fa6caec0c
permissions -rw-r--r--
upgrade tweetstream to 1.0.0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
13
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     1
import time
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     2
import urllib
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     3
import urllib2
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     4
import socket
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     5
import anyjson
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     6
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     7
from . import AuthenticationError, ConnectionError, USER_AGENT
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     8
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     9
class BaseStream(object):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    10
    """A network connection to Twitters streaming API
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    11
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    12
    :param username: Twitter username for the account accessing the API.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    13
    :param password: Twitter password for the account accessing the API.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    14
    :keyword count: Number of tweets from the past to get before switching to
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    15
      live stream.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    16
    :keyword url: Endpoint URL for the object. Note: you should not
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    17
      need to edit this. It's present to make testing easier.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    18
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    19
    .. attribute:: connected
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    20
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    21
        True if the object is currently connected to the stream.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    22
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    23
    .. attribute:: url
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    24
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    25
        The URL to which the object is connected
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    26
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    27
    .. attribute:: starttime
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    28
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    29
        The timestamp, in seconds since the epoch, the object connected to the
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    30
        streaming api.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    31
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    32
    .. attribute:: count
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    33
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    34
        The number of tweets that have been returned by the object.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    35
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    36
    .. attribute:: rate
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    37
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    38
        The rate at which tweets have been returned from the object as a
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    39
        float. see also :attr: `rate_period`.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    40
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    41
    .. attribute:: rate_period
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    42
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    43
        The ammount of time to sample tweets to calculate tweet rate. By
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    44
        default 10 seconds. Changes to this attribute will not be reflected
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    45
        until the next time the rate is calculated. The rate of tweets vary
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    46
        with time of day etc. so it's usefull to set this to something
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    47
        sensible.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    48
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    49
    .. attribute:: user_agent
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    50
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    51
        User agent string that will be included in the request. NOTE: This can
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    52
        not be changed after the connection has been made. This property must
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    53
        thus be set before accessing the iterator. The default is set in
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    54
        :attr: `USER_AGENT`.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    55
    """
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    56
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    57
    def __init__(self, username, password, catchup=None, url=None):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    58
        self._conn = None
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    59
        self._rate_ts = None
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    60
        self._rate_cnt = 0
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    61
        self._username = username
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    62
        self._password = password
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    63
        self._catchup_count = catchup
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    64
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    65
        self.rate_period = 10  # in seconds
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    66
        self.connected = False
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    67
        self.starttime = None
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    68
        self.count = 0
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    69
        self.rate = 0
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    70
        self.user_agent = USER_AGENT
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    71
        if url: self.url = url
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    72
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    73
    def __iter__(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    74
        return self
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    75
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    76
    def __enter__(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    77
        return self
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    78
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    79
    def __exit__(self, *params):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    80
        self.close()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    81
        return False
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    82
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    83
    def _init_conn(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    84
        """Open the connection to the twitter server"""
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    85
        headers = {'User-Agent': self.user_agent}
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    86
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    87
        postdata = self._get_post_data() or {}
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    88
        if self._catchup_count:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    89
            postdata["count"] = self._catchup_count
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    90
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    91
        poststring = urllib.urlencode(postdata) if postdata else None
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    92
        req = urllib2.Request(self.url, poststring, headers)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    93
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    94
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    95
        password_mgr.add_password(None, self.url, self._username, self._password)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    96
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    97
        opener = urllib2.build_opener(handler)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    98
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    99
        try:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   100
            self._conn = opener.open(req)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   101
        except urllib2.HTTPError, exception:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   102
            if exception.code == 401:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   103
                raise AuthenticationError("Access denied")
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   104
            elif exception.code == 404:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   105
                raise ConnectionError("URL not found: %s" % self.url)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   106
            else:  # re raise. No idea what would cause this, so want to know
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   107
                raise
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   108
        except urllib2.URLError, exception:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   109
            raise ConnectionError(exception.reason)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   110
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   111
        self.connected = True
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   112
        if not self.starttime:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   113
            self.starttime = time.time()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   114
        if not self._rate_ts:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   115
            self._rate_ts = time.time()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   116
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   117
    def _get_post_data(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   118
        """Subclasses that need to add post data to the request can override
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   119
        this method and return post data. The data should be in the format
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   120
        returned by urllib.urlencode."""
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   121
        return None
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   122
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   123
    def next(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   124
        """Return the next available tweet. This call is blocking!"""
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   125
        while True:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   126
            try:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   127
                if not self.connected:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   128
                    self._init_conn()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   129
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   130
                rate_time = time.time() - self._rate_ts
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   131
                if not self._rate_ts or rate_time > self.rate_period:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   132
                    self.rate = self._rate_cnt / rate_time
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   133
                    self._rate_cnt = 0
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   134
                    self._rate_ts = time.time()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   135
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   136
                data = self._conn.readline()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   137
                if data == "":  # something is wrong
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   138
                    self.close()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   139
                    raise ConnectionError("Got entry of length 0. Disconnected")
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   140
                elif data.isspace():
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   141
                    continue
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   142
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   143
                data = anyjson.deserialize(data)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   144
                if 'text' in data:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   145
                    self.count += 1
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   146
                    self._rate_cnt += 1
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   147
                return data
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   148
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   149
            except ValueError, e:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   150
                self.close()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   151
                raise ConnectionError("Got invalid data from twitter",
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   152
                                      details=data)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   153
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   154
            except socket.error, e:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   155
                self.close()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   156
                raise ConnectionError("Server disconnected")
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   157
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   158
    def close(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   159
        """
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   160
        Close the connection to the streaming server.
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   161
        """
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   162
        self.connected = False
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   163
        if self._conn:
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   164
            self._conn.close()
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   165
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   166
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   167
class SampleStream(BaseStream):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   168
    url = "http://stream.twitter.com/1/statuses/sample.json"
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   169
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   170
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   171
class FilterStream(BaseStream):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   172
    url = "http://stream.twitter.com/1/statuses/filter.json"
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   173
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   174
    def __init__(self, username, password, follow=None, locations=None,
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   175
                 track=None, catchup=None, url=None):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   176
        self._follow = follow
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   177
        self._locations = locations
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   178
        self._track = track
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   179
        # remove follow, locations, track
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   180
        BaseStream.__init__(self, username, password, url=url)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   181
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   182
    def _get_post_data(self):
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   183
        postdata = {}
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   184
        if self._follow: postdata["follow"] = ",".join([str(e) for e in self._follow])
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   185
        if self._locations: postdata["locations"] = ",".join(self._locations)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   186
        if self._track: postdata["track"] = ",".join(self._track)
79b6e132e3d7 upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
   187
        return postdata