| author | Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com> |
| Fri, 01 Jul 2011 10:15:32 +0200 | |
| changeset 13 | 79b6e132e3d7 |
| child 14 | 10e7a0c7c64f |
| child 207 | 621fa6caec0c |
| permissions | -rw-r--r-- |
|
13
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
1 |
import time |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
2 |
import urllib |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
3 |
import urllib2 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
4 |
import socket |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
5 |
import anyjson |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
6 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
7 |
from . import AuthenticationError, ConnectionError, USER_AGENT |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
8 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
9 |
class BaseStream(object): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
10 |
"""A network connection to Twitters streaming API |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
11 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
12 |
:param username: Twitter username for the account accessing the API. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
13 |
:param password: Twitter password for the account accessing the API. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
14 |
:keyword count: Number of tweets from the past to get before switching to |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
15 |
live stream. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
16 |
:keyword url: Endpoint URL for the object. Note: you should not |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
17 |
need to edit this. It's present to make testing easier. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
18 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
19 |
.. attribute:: connected |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
20 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
21 |
True if the object is currently connected to the stream. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
22 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
23 |
.. attribute:: url |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
24 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
25 |
The URL to which the object is connected |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
26 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
27 |
.. attribute:: starttime |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
28 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
29 |
The timestamp, in seconds since the epoch, the object connected to the |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
30 |
streaming api. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
31 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
32 |
.. attribute:: count |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
33 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
34 |
The number of tweets that have been returned by the object. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
35 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
36 |
.. attribute:: rate |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
37 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
38 |
The rate at which tweets have been returned from the object as a |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
39 |
float. see also :attr: `rate_period`. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
40 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
41 |
.. attribute:: rate_period |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
42 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
43 |
The ammount of time to sample tweets to calculate tweet rate. By |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
44 |
default 10 seconds. Changes to this attribute will not be reflected |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
45 |
until the next time the rate is calculated. The rate of tweets vary |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
46 |
with time of day etc. so it's usefull to set this to something |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
47 |
sensible. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
48 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
49 |
.. attribute:: user_agent |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
50 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
51 |
User agent string that will be included in the request. NOTE: This can |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
52 |
not be changed after the connection has been made. This property must |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
53 |
thus be set before accessing the iterator. The default is set in |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
54 |
:attr: `USER_AGENT`. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
55 |
""" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
56 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
57 |
def __init__(self, username, password, catchup=None, url=None): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
58 |
self._conn = None |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
59 |
self._rate_ts = None |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
60 |
self._rate_cnt = 0 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
61 |
self._username = username |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
62 |
self._password = password |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
63 |
self._catchup_count = catchup |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
64 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
65 |
self.rate_period = 10 # in seconds |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
66 |
self.connected = False |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
67 |
self.starttime = None |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
68 |
self.count = 0 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
69 |
self.rate = 0 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
70 |
self.user_agent = USER_AGENT |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
71 |
if url: self.url = url |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
72 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
73 |
def __iter__(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
74 |
return self |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
75 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
76 |
def __enter__(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
77 |
return self |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
78 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
79 |
def __exit__(self, *params): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
80 |
self.close() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
81 |
return False |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
82 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
83 |
def _init_conn(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
84 |
"""Open the connection to the twitter server""" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
85 |
headers = {'User-Agent': self.user_agent} |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
86 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
87 |
postdata = self._get_post_data() or {} |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
88 |
if self._catchup_count: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
89 |
postdata["count"] = self._catchup_count |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
90 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
91 |
poststring = urllib.urlencode(postdata) if postdata else None |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
92 |
req = urllib2.Request(self.url, poststring, headers) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
93 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
94 |
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
95 |
password_mgr.add_password(None, self.url, self._username, self._password) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
96 |
handler = urllib2.HTTPBasicAuthHandler(password_mgr) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
97 |
opener = urllib2.build_opener(handler) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
98 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
99 |
try: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
100 |
self._conn = opener.open(req) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
101 |
except urllib2.HTTPError, exception: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
102 |
if exception.code == 401: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
103 |
raise AuthenticationError("Access denied") |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
104 |
elif exception.code == 404: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
105 |
raise ConnectionError("URL not found: %s" % self.url) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
106 |
else: # re raise. No idea what would cause this, so want to know |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
107 |
raise |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
108 |
except urllib2.URLError, exception: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
109 |
raise ConnectionError(exception.reason) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
110 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
111 |
self.connected = True |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
112 |
if not self.starttime: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
113 |
self.starttime = time.time() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
114 |
if not self._rate_ts: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
115 |
self._rate_ts = time.time() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
116 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
117 |
def _get_post_data(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
118 |
"""Subclasses that need to add post data to the request can override |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
119 |
this method and return post data. The data should be in the format |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
120 |
returned by urllib.urlencode.""" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
121 |
return None |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
122 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
123 |
def next(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
124 |
"""Return the next available tweet. This call is blocking!""" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
125 |
while True: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
126 |
try: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
127 |
if not self.connected: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
128 |
self._init_conn() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
129 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
130 |
rate_time = time.time() - self._rate_ts |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
131 |
if not self._rate_ts or rate_time > self.rate_period: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
132 |
self.rate = self._rate_cnt / rate_time |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
133 |
self._rate_cnt = 0 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
134 |
self._rate_ts = time.time() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
135 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
136 |
data = self._conn.readline() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
137 |
if data == "": # something is wrong |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
138 |
self.close() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
139 |
raise ConnectionError("Got entry of length 0. Disconnected") |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
140 |
elif data.isspace(): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
141 |
continue |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
142 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
143 |
data = anyjson.deserialize(data) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
144 |
if 'text' in data: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
145 |
self.count += 1 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
146 |
self._rate_cnt += 1 |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
147 |
return data |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
148 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
149 |
except ValueError, e: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
150 |
self.close() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
151 |
raise ConnectionError("Got invalid data from twitter", |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
152 |
details=data) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
153 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
154 |
except socket.error, e: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
155 |
self.close() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
156 |
raise ConnectionError("Server disconnected") |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
157 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
158 |
def close(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
159 |
""" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
160 |
Close the connection to the streaming server. |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
161 |
""" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
162 |
self.connected = False |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
163 |
if self._conn: |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
164 |
self._conn.close() |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
165 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
166 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
167 |
class SampleStream(BaseStream): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
168 |
url = "http://stream.twitter.com/1/statuses/sample.json" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
169 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
170 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
171 |
class FilterStream(BaseStream): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
172 |
url = "http://stream.twitter.com/1/statuses/filter.json" |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
173 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
174 |
def __init__(self, username, password, follow=None, locations=None, |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
175 |
track=None, catchup=None, url=None): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
176 |
self._follow = follow |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
177 |
self._locations = locations |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
178 |
self._track = track |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
179 |
# remove follow, locations, track |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
180 |
BaseStream.__init__(self, username, password, url=url) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
181 |
|
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
182 |
def _get_post_data(self): |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
183 |
postdata = {} |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
184 |
if self._follow: postdata["follow"] = ",".join([str(e) for e in self._follow]) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
185 |
if self._locations: postdata["locations"] = ",".join(self._locations) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
186 |
if self._track: postdata["track"] = ",".join(self._track) |
|
79b6e132e3d7
upgrade tweetstream to 1.0.0
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
187 |
return postdata |