1 from sqlalchemy import Column, Integer, String, ForeignKey, create_engine |
|
2 from sqlalchemy.ext.declarative import declarative_base |
|
3 from sqlalchemy.orm import relationship, backref |
|
4 import unittest #@UnresolvedImport |
|
5 from sqlalchemy.orm import sessionmaker |
|
6 from iri_tweet.utils import ObjectsBuffer, TwitterProcessor |
|
7 from iri_tweet import models |
|
8 import tempfile #@UnresolvedImport |
|
9 import os |
|
10 |
|
11 Base = declarative_base() |
|
12 |
|
13 class User(Base): |
|
14 __tablename__ = 'users' |
|
15 |
|
16 id = Column(Integer, primary_key=True) |
|
17 name = Column(String) |
|
18 fullname = Column(String) |
|
19 password = Column(String) |
|
20 |
|
21 def __init__(self, name, fullname, password): |
|
22 self.name = name |
|
23 self.fullname = fullname |
|
24 self.password = password |
|
25 |
|
26 def __repr__(self): |
|
27 return "<User('%s','%s', '%s')>" % (self.name, self.fullname, self.password) |
|
28 |
|
29 |
|
30 class Address(Base): |
|
31 __tablename__ = 'addresses' |
|
32 id = Column(Integer, primary_key=True) |
|
33 email_address = Column(String, nullable=False) |
|
34 user_id = Column(Integer, ForeignKey('users.id')) |
|
35 |
|
36 user = relationship("User", backref=backref('addresses', order_by=id)) |
|
37 |
|
38 def __init__(self, user_id, email_address): |
|
39 self.email_address = email_address |
|
40 self.user_id = user_id |
|
41 |
|
42 def __repr__(self): |
|
43 return "<Address('%s')>" % self.email_address |
|
44 |
|
45 |
|
46 |
|
47 class TestObjectBuffer(unittest.TestCase): |
|
48 |
|
49 def setUp(self): |
|
50 self.engine = create_engine('sqlite:///:memory:', echo=False) |
|
51 Base.metadata.create_all(self.engine) |
|
52 sessionMaker = sessionmaker(bind=self.engine) |
|
53 self.session = sessionMaker() |
|
54 |
|
55 def tearDown(self): |
|
56 self.session.close() |
|
57 self.engine.dispose() |
|
58 |
|
59 |
|
60 def testCreateUser(self): |
|
61 ed_user = User('ed', 'Ed Jones', 'edspassword') |
|
62 self.session.add(ed_user) |
|
63 self.assertTrue(ed_user.id is None) |
|
64 self.session.commit() |
|
65 self.assertTrue(ed_user.id is not None) |
|
66 |
|
67 |
|
68 def testSimpleBuffer(self): |
|
69 obj_buffer = ObjectsBuffer() |
|
70 obj_proxy = obj_buffer.add_object(User, ['ed1', 'Ed1 Jones', 'edspassword'], None, False) |
|
71 self.assertTrue(obj_proxy.id() is None) |
|
72 obj_buffer.persists(self.session) |
|
73 self.assertTrue(obj_proxy.id() is None) |
|
74 self.session.commit() |
|
75 self.assertTrue(obj_proxy.id() is not None) |
|
76 |
|
77 |
|
78 def testSimpleBufferKwargs(self): |
|
79 obj_buffer = ObjectsBuffer() |
|
80 obj_proxy = obj_buffer.add_object(User, None, {'name':'ed1b', 'fullname':'Ed1b Jones', 'password':'edspassword'}, False) |
|
81 self.assertTrue(obj_proxy.id() is None) |
|
82 obj_buffer.persists(self.session) |
|
83 self.assertTrue(obj_proxy.id() is None) |
|
84 self.session.commit() |
|
85 self.assertTrue(obj_proxy.id() is not None) |
|
86 |
|
87 |
|
88 def testSimpleBufferFlush(self): |
|
89 obj_buffer = ObjectsBuffer() |
|
90 obj_proxy = obj_buffer.add_object(User, ['ed2', 'Ed2 Jones', 'edspassword'], None, True) |
|
91 self.assertTrue(obj_proxy.id() is None) |
|
92 obj_buffer.persists(self.session) |
|
93 self.assertTrue(obj_proxy.id() is not None) |
|
94 self.session.commit() |
|
95 self.assertTrue(obj_proxy.id() is not None) |
|
96 |
|
97 def testRelationBuffer(self): |
|
98 obj_buffer = ObjectsBuffer() |
|
99 user1_proxy = obj_buffer.add_object(User, ['ed3', 'Ed3 Jones', 'edspassword'], None, True) |
|
100 obj_buffer.add_object(Address, [user1_proxy.id,'ed3@mail.com'], None, False) |
|
101 obj_buffer.add_object(Address, [user1_proxy.id,'ed3@other.com'], None, False) |
|
102 user2_proxy = obj_buffer.add_object(User, ['ed4', 'Ed3 Jones', 'edspassword'], None, True) |
|
103 obj_buffer.add_object(Address, [user2_proxy.id,'ed4@mail.com'], None, False) |
|
104 obj_buffer.persists(self.session) |
|
105 self.session.commit() |
|
106 ed_user = self.session.query(User).filter_by(name='ed3').first() |
|
107 self.assertEquals(2, len(ed_user.addresses)) |
|
108 ed_user = self.session.query(User).filter_by(name='ed4').first() |
|
109 self.assertEquals(1, len(ed_user.addresses)) |
|
110 |
|
111 |
|
112 def testGet(self): |
|
113 obj_buffer = ObjectsBuffer() |
|
114 user1_proxy = obj_buffer.add_object(User, None, {'name':'ed2', 'fullname':'Ed2 Jones', 'password':'edspassword'}, True) |
|
115 adress_proxy = obj_buffer.add_object(Address, None, {'user_id':user1_proxy.id,'email_address':'ed2@other.com'}, False) |
|
116 user2_proxy = obj_buffer.add_object(User, None, {'name':'ed3', 'fullname':'Ed3 Jones', 'password':'edspassword'}, True) |
|
117 obj_buffer.add_object(Address, None, {'user_id':user2_proxy.id,'email_address':'ed3@other.com'}, False) |
|
118 self.assertEquals(user1_proxy, obj_buffer.get(User, name='ed2')) |
|
119 self.assertEquals(adress_proxy, obj_buffer.get(Address, email_address='ed2@other.com')) |
|
120 self.assertEquals(user2_proxy, obj_buffer.get(User, name='ed3')) |
|
121 self.assertTrue(obj_buffer.get(User, name='ed3', fullname='Ed2 Jones') is None) |
|
122 |
|
123 original_json = u'{"in_reply_to_user_id_str":null,"text":"RT @BieberEagle: \\"I love my haters. They spend so much time thinking about me. Aren\u2019t they sweet?\\" - Justin Bieber","contributors":null,"retweeted":false,"coordinates":null,"retweeted_status":{"in_reply_to_user_id_str":null,"text":"\\"I love my haters. They spend so much time thinking about me. Aren\u2019t they sweet?\\" - Justin Bieber","contributors":null,"retweeted":false,"coordinates":null,"retweet_count":"100+","source":"web","entities":{"user_mentions":[],"hashtags":[],"urls":[]},"truncated":false,"place":null,"id_str":"96638597737889792","in_reply_to_user_id":null,"in_reply_to_status_id":null,"favorited":false,"in_reply_to_status_id_str":null,"user":{"is_translator":false,"profile_background_tile":true,"profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/298443445\/355584171.jpg","listed_count":5040,"friends_count":8477,"profile_link_color":"ff0000","profile_sidebar_border_color":"000000","url":"http:\/\/twitpic.com\/photos\/BieberEagle","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1465491672\/355584171_normal.jpg","profile_image_url":"http:\/\/a2.twimg.com\/profile_images\/1465491672\/355584171_normal.jpg","description":"1 name, 1 inspiration, 1 hero, 1 smile, 1 singer, 1 boy who changed my life. B.\u0130.E.B.E.R-Believe In Everything Because Everything\'s Reachable. #NEVERSAYNEVER","default_profile":false,"notifications":null,"time_zone":"Paris","followers_count":14506,"default_profile_image":false,"lang":"en","profile_use_background_image":true,"screen_name":"BieberEagle","show_all_inline_media":false,"geo_enabled":false,"profile_background_color":"ffffff","location":"\u2665 Albania \u2665 ","id_str":"229067923","profile_background_image_url":"http:\/\/a2.twimg.com\/profile_background_images\/298443445\/355584171.jpg","favourites_count":89,"protected":false,"follow_request_sent":null,"following":null,"name":"truebelieber","statuses_count":24279,"verified":false,"created_at":"Tue Dec 21 12:35:18 +0000 2010","profile_text_color":"000000","id":229067923,"contributors_enabled":false,"utc_offset":3600,"profile_sidebar_fill_color":""},"id":96638597737889792,"created_at":"Thu Jul 28 17:50:11 +0000 2011","geo":null,"in_reply_to_screen_name":null},"retweet_count":"100+","source":"web","entities":{"user_mentions":[{"indices":[3,15],"screen_name":"BieberEagle","id_str":"229067923","name":"truebelieber","id":229067923}],"hashtags":[],"urls":[]},"truncated":false,"place":null,"id_str":"96965037637382145","in_reply_to_user_id":null,"in_reply_to_status_id":null,"favorited":false,"in_reply_to_status_id_str":null,"user":{"is_translator":false,"profile_background_tile":true,"profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/300419382\/ipod.7.14_054.JPG","listed_count":3,"friends_count":1150,"profile_link_color":"00cccc","profile_sidebar_border_color":"c8ff00","url":"http:\/\/www.facebook.com\/blovedbecca180","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1466752962\/block_party_7.27.11_015_normal.JPG","profile_image_url":"http:\/\/a3.twimg.com\/profile_images\/1466752962\/block_party_7.27.11_015_normal.JPG","description":"if ya wanna know something about me, then get to know me. \\n\\r\\n\\ri promise, you wont regret it. (:\\r\\ni love justin bieber with an extreme burning passion!","default_profile":false,"notifications":null,"time_zone":"Central Time (US & Canada)","followers_count":361,"default_profile_image":false,"lang":"en","profile_use_background_image":true,"screen_name":"beccaxannxx","show_all_inline_media":false,"geo_enabled":false,"profile_background_color":"ff0066","location":"","id_str":"65624607","profile_background_image_url":"http:\/\/a3.twimg.com\/profile_background_images\/300419382\/ipod.7.14_054.JPG","favourites_count":266,"protected":false,"follow_request_sent":null,"following":null,"name":"beccaxannxx","statuses_count":2512,"verified":false,"created_at":"Fri Aug 14 12:36:35 +0000 2009","profile_text_color":"6a39d4","id":65624607,"contributors_enabled":false,"utc_offset":-21600,"profile_sidebar_fill_color":"ff00bb"},"id":96965037637382145,"created_at":"Fri Jul 29 15:27:21 +0000 2011","geo":null,"in_reply_to_screen_name":null}' |
|
124 original_json_media = u'{"user": {"follow_request_sent": null, "profile_use_background_image": true, "id": 34311537, "verified": false, "profile_image_url_https": "https://si0.twimg.com/profile_images/1452959211/63440_1494505009634_1444332638_31074099_4058882_n_normal.jpg", "profile_sidebar_fill_color": "DAECF4", "is_translator": false, "geo_enabled": false, "profile_text_color": "663B12", "followers_count": 29, "profile_sidebar_border_color": "C6E2EE", "id_str": "34311537", "default_profile_image": false, "location": "", "utc_offset": -25200, "statuses_count": 813, "description": "i like joe jonas, justin bieber, ashley tisdale, selena gomez, megen fox, kim kardashian and demi lovoto and many more.", "friends_count": 101, "profile_link_color": "1F98C7", "profile_image_url": "http://a1.twimg.com/profile_images/1452959211/63440_1494505009634_1444332638_31074099_4058882_n_normal.jpg", "notifications": null, "show_all_inline_media": false, "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/298244445/tour2011.jpg", "profile_background_color": "C6E2EE", "profile_background_image_url": "http://a1.twimg.com/profile_background_images/298244445/tour2011.jpg", "name": "mikayla", "lang": "en", "profile_background_tile": true, "favourites_count": 231, "screen_name": "bieberfever17ya", "url": null, "created_at": "Wed Apr 22 16:04:28 +0000 2009", "contributors_enabled": false, "time_zone": "Mountain Time (US & Canada)", "protected": false, "default_profile": false, "following": null, "listed_count": 1}, "favorited": false, "entities": {"user_mentions": [], "media": [{"media_url_https": "https://p.twimg.com/AWea5Z-CQAAvfvK.jpg", "expanded_url": "http://twitter.com/bieberfever17ya/status/101219827649232896/photo/1", "sizes": {"small": {"h": 240, "w": 201, "resize": "fit"}, "large": {"h": 240, "w": 201, "resize": "fit"}, "medium": {"h": 240, "w": 201, "resize": "fit"}, "thumb": {"h": 150, "w": 150, "resize": "crop"}}, "url": "http://t.co/N7yZ8hS", "display_url": "pic.twitter.com/N7yZ8hS", "id_str": "101219827653427200", "indices": [31, 50], "type": "photo", "id": 101219827653427200, "media_url": "http://p.twimg.com/AWea5Z-CQAAvfvK.jpg"}], "hashtags": [], "urls": []}, "contributors": null, "truncated": false, "text": "i love you justin bieber <3 http://t.co/N7yZ8hS", "created_at": "Wed Aug 10 09:14:22 +0000 2011", "retweeted": false, "in_reply_to_status_id": null, "coordinates": null, "id": 101219827649232896, "source": "web", "in_reply_to_status_id_str": null, "place": null, "in_reply_to_user_id": null, "in_reply_to_screen_name": null, "retweet_count": 0, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "id_str": "101219827649232896"}' |
|
125 original_json_media_others = u'{"user": {"utc_offset": -25200, "statuses_count": 813, "default_profile_image": false, "friends_count": 101, "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/298244445/tour2011.jpg", "profile_use_background_image": true, "profile_sidebar_fill_color": "DAECF4", "profile_link_color": "1F98C7", "profile_image_url": "http://a1.twimg.com/profile_images/1452959211/63440_1494505009634_1444332638_31074099_4058882_n_normal.jpg", "time_zone": "Mountain Time (US & Canada)", "is_translator": false, "screen_name": "bieberfever17ya", "url": null, "show_all_inline_media": false, "geo_enabled": false, "profile_background_color": "C6E2EE", "id": 34311537, "profile_background_image_url": "http://a1.twimg.com/profile_background_images/298244445/tour2011.jpg", "description": "i like joe jonas, justin bieber, ashley tisdale, selena gomez, megen fox, kim kardashian and demi lovoto and many more.", "lang": "en", "profile_background_tile": true, "favourites_count": 231, "name": "mikayla", "notifications": null, "follow_request_sent": null, "created_at": "Wed Apr 22 16:04:28 +0000 2009", "verified": false, "contributors_enabled": false, "location": "", "profile_text_color": "663B12", "followers_count": 29, "profile_sidebar_border_color": "C6E2EE", "id_str": "34311537", "default_profile": false, "following": null, "protected": false, "profile_image_url_https": "https://si0.twimg.com/profile_images/1452959211/63440_1494505009634_1444332638_31074099_4058882_n_normal.jpg", "listed_count": 1}, "favorited": false, "contributors": null, "source": "web", "text": "i love you justin bieber <3 http://t.co/N7yZ8hS", "created_at": "Wed Aug 10 09:14:22 +0000 2011", "truncated": false, "retweeted": false, "in_reply_to_status_id_str": null, "coordinates": null, "in_reply_to_user_id_str": null, "entities": {"user_mentions": [], "media": [], "hashtags": [], "urls": [], "others": [{"url": "http://t.co/N7yZ8hS", "text": "comments", "indices": [31, 50]}]}, "in_reply_to_status_id": null, "in_reply_to_screen_name": null, "id_str": "101219827649232896", "place": null, "retweet_count": 0, "geo": null, "id": 101219827649232896, "possibly_sensitive": false, "in_reply_to_user_id": null}' |
|
126 |
|
127 class TestTwitterProcessor(unittest.TestCase): |
|
128 |
|
129 def setUp(self): |
|
130 self.engine, self.metadata, sessionMaker = models.setup_database('sqlite:///:memory:', echo=True) |
|
131 self.session = sessionMaker() |
|
132 file, self.tmpfilepath = tempfile.mkstemp() |
|
133 os.close(file) |
|
134 |
|
135 |
|
136 def testTwitterProcessor(self): |
|
137 tp = TwitterProcessor(None, original_json, None, self.session, self.tmpfilepath) |
|
138 tp.process() |
|
139 self.session.commit() |
|
140 |
|
141 self.assertEquals(1, self.session.query(models.TweetSource).count()) |
|
142 self.assertEquals(1, self.session.query(models.Tweet).count()) |
|
143 self.assertEquals(2, self.session.query(models.User).count()) |
|
144 tweet = self.session.query(models.Tweet).first() |
|
145 self.assertFalse(tweet.user is None) |
|
146 self.assertEqual(u"beccaxannxx",tweet.user.name) |
|
147 self.assertEqual(65624607,tweet.user.id) |
|
148 self.assertEqual(1,len(tweet.entity_list)) |
|
149 entity = tweet.entity_list[0] |
|
150 self.assertEqual(u"BieberEagle", entity.user.screen_name) |
|
151 self.assertTrue(entity.user.created_at is None) |
|
152 self.assertEqual("entity_user", entity.type) |
|
153 self.assertEqual("user_mentions", entity.entity_type.label) |
|
154 |
|
155 |
|
156 def testTwitterProcessorMedia(self): |
|
157 tp = TwitterProcessor(None, original_json_media, None, self.session, self.tmpfilepath) |
|
158 tp.process() |
|
159 self.session.commit() |
|
160 |
|
161 self.assertEquals(1, self.session.query(models.TweetSource).count()) |
|
162 self.assertEquals(1, self.session.query(models.Tweet).count()) |
|
163 self.assertEquals(1, self.session.query(models.User).count()) |
|
164 tweet = self.session.query(models.Tweet).first() |
|
165 self.assertFalse(tweet.user is None) |
|
166 self.assertEqual(u"mikayla",tweet.user.name) |
|
167 self.assertEqual(34311537,tweet.user.id) |
|
168 self.assertEqual(1,len(tweet.entity_list)) |
|
169 entity = tweet.entity_list[0] |
|
170 self.assertEqual(101219827653427200, entity.media.id) |
|
171 self.assertEqual("photo", entity.media.type.label) |
|
172 self.assertEqual("entity_media", entity.type) |
|
173 self.assertEqual("media", entity.entity_type.label) |
|
174 |
|
175 |
|
176 def testTwitterProcessorMediaOthers(self): |
|
177 tp = TwitterProcessor(None, original_json_media_others, None, self.session, self.tmpfilepath) |
|
178 tp.process() |
|
179 self.session.commit() |
|
180 |
|
181 self.assertEquals(1, self.session.query(models.TweetSource).count()) |
|
182 self.assertEquals(1, self.session.query(models.Tweet).count()) |
|
183 tweet = self.session.query(models.Tweet).first() |
|
184 self.assertEqual(1,len(tweet.entity_list)) |
|
185 entity = tweet.entity_list[0] |
|
186 self.assertEqual("entity_entity", entity.type) |
|
187 self.assertEqual("others", entity.entity_type.label) |
|
188 |
|
189 |
|
190 |
|
191 def tearDown(self): |
|
192 self.session.close() |
|
193 self.engine.dispose() |
|
194 os.remove(self.tmpfilepath) |
|
195 |
|
196 if __name__ == '__main__': |
|
197 unittest.main() |
|