| author | Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com> |
| Tue, 07 May 2013 18:57:54 +0200 | |
| changeset 888 | 6fc6637d8403 |
| parent 529 | 99215db3da25 |
| child 889 | c774bdf7d3dd |
| permissions | -rw-r--r-- |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
1 |
from sqlalchemy import (Boolean, Column, Enum, BigInteger, Integer, String, |
|
888
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
2 |
ForeignKey, DateTime, create_engine, event) |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
3 |
from sqlalchemy.engine import Engine |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
4 |
from sqlalchemy.ext.declarative import declarative_base |
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
5 |
from sqlalchemy.orm import relationship, sessionmaker |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
6 |
import anyjson |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
7 |
import datetime |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
8 |
import email.utils |
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
9 |
import iri_tweet |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
10 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
11 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
12 |
Base = declarative_base() |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
13 |
|
|
11
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
14 |
APPLICATION_NAME = "IRI_TWITTER" |
|
888
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
15 |
#CONSUMER_KEY = "54ThDZhpEjokcMgHJOMnQA" |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
16 |
#CONSUMER_SECRET = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA" |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
17 |
ACCESS_TOKEN_KEY = None |
|
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
18 |
ACCESS_TOKEN_SECRET = None |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
19 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
20 |
def adapt_date(date_str): |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
21 |
ts = email.utils.parsedate_tz(date_str) #@UndefinedVariable |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
22 |
return datetime.datetime(*ts[0:7]) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
23 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
24 |
def adapt_json(obj): |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
25 |
if obj is None: |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
26 |
return None |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
27 |
else: |
|
18
bd595ad770fc
- replace json with anyjson
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
12
diff
changeset
|
28 |
return anyjson.serialize(obj) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
29 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
30 |
class TweetMeta(type(Base)): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
31 |
|
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
32 |
def __init__(cls, name, bases, ns): #@NoSelf |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
33 |
def init(self, **kwargs): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
34 |
for key, value in kwargs.items(): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
35 |
if hasattr(self, key): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
36 |
setattr(self, key, value) |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
37 |
super(cls, self).__init__() |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
38 |
setattr(cls, '__init__', init) |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
39 |
super(TweetMeta, cls).__init__(name, bases, ns) |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
40 |
|
|
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
41 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
42 |
class ProcessEvent(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
43 |
__metaclass__ = TweetMeta |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
44 |
__tablename__ = "tweet_process_event" |
|
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
45 |
id = Column(Integer, primary_key=True, autoincrement=True) |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
46 |
ts = Column(DateTime, default=datetime.datetime.utcnow, index=True) |
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
47 |
type = Column(Enum("start","pid","shutdown","error", "start_worker", "stop_worker", "model_version", "application_name", "application_version", name="process_event_type_enum"), nullable=False) |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
48 |
args = Column(String) |
|
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
49 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
50 |
class EntityType(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
51 |
__metaclass__ = TweetMeta |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
52 |
__tablename__ = "tweet_entity_type" |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
53 |
id = Column(Integer, primary_key=True, autoincrement=True) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
54 |
label = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
55 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
56 |
class Entity(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
57 |
__metaclass__ = TweetMeta |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
58 |
__tablename__ = "tweet_entity" |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
59 |
id = Column(Integer, primary_key=True) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
60 |
tweet_id = Column(BigInteger, ForeignKey('tweet_tweet.id')) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
61 |
type = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
62 |
entity_type_id = Column(Integer, ForeignKey('tweet_entity_type.id'), nullable=False) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
63 |
entity_type = relationship("EntityType", backref="entities") |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
64 |
indice_start = Column(Integer) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
65 |
indice_end = Column(Integer) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
66 |
source = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
67 |
__mapper_args__ = {'polymorphic_on': type, 'polymorphic_identity': 'entity_entity', 'with_polymorphic':'*'} |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
68 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
69 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
70 |
class TweetSource(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
71 |
__metaclass__ = TweetMeta |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
72 |
__tablename__ = 'tweet_tweet_source' |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
73 |
id = Column(Integer, primary_key=True, autoincrement=True) |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
74 |
original_json = Column(String) |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
75 |
received_at = Column(DateTime, default=datetime.datetime.utcnow, index=True) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
76 |
|
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
77 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
78 |
class TweetLog(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
79 |
|
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
80 |
TWEET_STATUS = { |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
81 |
'OK' : 1, |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
82 |
'ERROR' : 2, |
|
255
500cd0405c7a
improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
254
diff
changeset
|
83 |
'NOT_TWEET': 3, |
|
888
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
84 |
'DELETE': 4, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
85 |
'SCRUB_GEO': 5, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
86 |
'LIMIT': 6, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
87 |
'STATUS_WITHHELD': 7, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
88 |
'USER_WITHHELD': 8, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
89 |
'DISCONNECT': 9, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
90 |
'STALL_WARNING': 10, |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
91 |
'DELETE_PENDING': 4 |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
92 |
} |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
93 |
__metaclass__ = TweetMeta |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
94 |
|
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
95 |
__tablename__ = 'tweet_tweet_log' |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
96 |
id = Column(Integer, primary_key=True, autoincrement=True) |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
97 |
ts = Column(DateTime, default=datetime.datetime.utcnow, index=True) |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
98 |
tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id')) |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
99 |
tweet_source = relationship("TweetSource", backref="logs") |
|
888
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
100 |
status_id = Column(BigInteger, index=True, nullable=True, default=None) |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
101 |
status = Column(Integer) |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
102 |
error = Column(String) |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
103 |
error_stack = Column(String) |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
104 |
|
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
105 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
106 |
class Tweet(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
107 |
__metaclass__ = TweetMeta |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
108 |
__tablename__ = 'tweet_tweet' |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
109 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
110 |
id = Column(BigInteger, primary_key=True, autoincrement=False) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
111 |
id_str = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
112 |
contributors = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
113 |
coordinates = Column(String) |
|
392
aa445cd7300e
Nouvelle seance museo
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
289
diff
changeset
|
114 |
created_at = Column(DateTime, index=True) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
115 |
favorited = Column(Boolean) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
116 |
geo = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
117 |
in_reply_to_screen_name = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
118 |
in_reply_to_status_id = Column(BigInteger) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
119 |
in_reply_to_status_id_str = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
120 |
in_reply_to_user_id = Column(BigInteger) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
121 |
in_reply_to_user_id_str = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
122 |
place = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
123 |
retweet_count = Column(String) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
124 |
retweeted = Column(Boolean) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
125 |
source = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
126 |
text = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
127 |
truncated = Column(Boolean) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
128 |
user_id = Column(Integer, ForeignKey('tweet_user.id')) |
|
243
9213a63fa34a
- debug multithread (still database lock problem)
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
242
diff
changeset
|
129 |
user = relationship("User", backref="tweets") |
|
242
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
130 |
tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id')) |
|
cdd7d3c0549c
Starting 'parallel_twitter' branch
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
122
diff
changeset
|
131 |
tweet_source = relationship("TweetSource", backref="tweet") |
|
888
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
132 |
entity_list = relationship(Entity, backref='tweet', cascade="all, delete-orphan") |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
133 |
received_at = Column(DateTime, default=datetime.datetime.utcnow, index=True) |
|
243
9213a63fa34a
- debug multithread (still database lock problem)
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
242
diff
changeset
|
134 |
|
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
135 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
136 |
class UserMessage(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
137 |
__metaclass__ = TweetMeta |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
138 |
__tablename__ = "tweet_user_message" |
|
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
139 |
|
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
140 |
id = Column(Integer, primary_key=True) |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
141 |
user_id = Column(Integer, ForeignKey('tweet_user.id')) |
|
243
9213a63fa34a
- debug multithread (still database lock problem)
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
242
diff
changeset
|
142 |
user = relationship("User", backref="messages") |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
143 |
created_at = Column(DateTime, default=datetime.datetime.utcnow) |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
144 |
message_id = Column(Integer, ForeignKey('tweet_message.id')) |
|
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
145 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
146 |
class Message(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
147 |
__metaclass__ = TweetMeta |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
148 |
__tablename__ = "tweet_message" |
|
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
149 |
|
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
150 |
id = Column(Integer, primary_key=True) |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
151 |
created_at = Column(DateTime, default=datetime.datetime.utcnow) |
|
98
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
152 |
text = Column(String) |
|
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
153 |
users = relationship(UserMessage, backref='message') |
|
6e8930a1b8f7
add tools to track tweeter messaging
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
18
diff
changeset
|
154 |
|
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
155 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
156 |
class User(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
157 |
__metaclass__ = TweetMeta |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
158 |
__tablename__ = "tweet_user" |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
159 |
|
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
160 |
id = Column(BigInteger, primary_key=True, autoincrement=False) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
161 |
id_str = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
162 |
contributors_enabled = Column(Boolean) |
|
409
f7ceddf99d6d
improve the user management. try to complete user information whenever possible.
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
392
diff
changeset
|
163 |
created_at = Column(DateTime, index=True) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
164 |
description = Column(String) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
165 |
favourites_count = Column(Integer) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
166 |
follow_request_sent = Column(Boolean) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
167 |
followers_count = Column(Integer) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
168 |
following = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
169 |
friends_count = Column(Integer) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
170 |
geo_enabled = Column(Boolean) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
171 |
is_translator = Column(Boolean) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
172 |
lang = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
173 |
listed_count = Column(Integer) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
174 |
location = Column(String) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
175 |
name = Column(String) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
176 |
notifications = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
177 |
profile_background_color = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
178 |
profile_background_image_url = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
179 |
profile_background_tile = Column(Boolean) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
180 |
profile_image_url = Column(String) |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
181 |
profile_image_url_https = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
182 |
profile_link_color = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
183 |
profile_sidebar_border_color = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
184 |
profile_sidebar_fill_color = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
185 |
profile_text_color = Column(String) |
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
186 |
default_profile_image = Column(String) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
187 |
profile_use_background_image = Column(Boolean) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
188 |
protected = Column(Boolean) |
|
255
500cd0405c7a
improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
254
diff
changeset
|
189 |
screen_name = Column(String, index=True) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
190 |
show_all_inline_media = Column(Boolean) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
191 |
statuses_count = Column(Integer) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
192 |
time_zone = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
193 |
url = Column(String) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
194 |
utc_offset = Column(Integer) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
195 |
verified = Column(Boolean) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
196 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
197 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
198 |
class Hashtag(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
199 |
__metaclass__ = TweetMeta |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
200 |
__tablename__ = "tweet_hashtag" |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
201 |
id = Column(Integer, primary_key=True) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
202 |
text = Column(String, unique=True, index=True) |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
203 |
|
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
204 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
205 |
class Url(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
206 |
__metaclass__ = TweetMeta |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
207 |
__tablename__ = "tweet_url" |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
208 |
id = Column(Integer, primary_key=True) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
209 |
url = Column(String, unique=True) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
210 |
expanded_url = Column(String) |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
211 |
|
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
212 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
213 |
class MediaType(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
214 |
__metaclass__ = TweetMeta |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
215 |
__tablename__ = "tweet_media_type" |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
216 |
id = Column(Integer, primary_key=True, autoincrement=True) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
217 |
label = Column(String, unique=True, index=True) |
|
261
d84c4aa2a9eb
add process event for start and shutdown
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
255
diff
changeset
|
218 |
|
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
219 |
|
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
220 |
|
|
263
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
221 |
class Media(Base): |
|
6671e9a4c9c5
correct model ans improve event tracking
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
261
diff
changeset
|
222 |
__metaclass__ = TweetMeta |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
223 |
__tablename__ = "tweet_media" |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
224 |
id = Column(BigInteger, primary_key=True, autoincrement=False) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
225 |
id_str = Column(String, unique=True) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
226 |
media_url = Column(String, unique=True) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
227 |
media_url_https = Column(String, unique=True) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
228 |
url = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
229 |
display_url = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
230 |
expanded_url = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
231 |
sizes = Column(String) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
232 |
type_id = Column(Integer, ForeignKey("tweet_media_type.id")) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
233 |
type = relationship(MediaType, primaryjoin=type_id == MediaType.id) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
234 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
235 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
236 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
237 |
class EntityHashtag(Entity): |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
238 |
__tablename__ = "tweet_entity_hashtag" |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
239 |
__mapper_args__ = {'polymorphic_identity': 'entity_hashtag'} |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
240 |
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
241 |
hashtag_id = Column(Integer, ForeignKey("tweet_hashtag.id")) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
242 |
hashtag = relationship(Hashtag, primaryjoin=hashtag_id == Hashtag.id) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
243 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
244 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
245 |
class EntityUrl(Entity): |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
246 |
__tablename__ = "tweet_entity_url" |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
247 |
__mapper_args__ = {'polymorphic_identity': 'entity_url'} |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
248 |
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
249 |
url_id = Column(Integer, ForeignKey("tweet_url.id")) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
250 |
url = relationship(Url, primaryjoin=url_id == Url.id) |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
251 |
|
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
252 |
class EntityUser(Entity): |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
253 |
__tablename__ = "tweet_entity_user" |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
254 |
__mapper_args__ = {'polymorphic_identity': 'entity_user'} |
|
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
255 |
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True) |
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
256 |
user_id = Column(BigInteger, ForeignKey('tweet_user.id')) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
257 |
user = relationship(User, primaryjoin=(user_id == User.id)) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
258 |
|
|
11
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
259 |
|
|
254
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
260 |
class EntityMedia(Entity): |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
261 |
__tablename__ = "tweet_entity_media" |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
262 |
__mapper_args__ = {'polymorphic_identity': 'entity_media'} |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
263 |
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
264 |
media_id = Column(BigInteger, ForeignKey('tweet_media.id')) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
265 |
media = relationship(Media, primaryjoin=(media_id == Media.id)) |
|
2209e66bb50b
multiple debugging and corrections
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
253
diff
changeset
|
266 |
|
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
267 |
def add_model_version(session, must_commit=True): |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
268 |
pe = ProcessEvent(args=iri_tweet.get_version(), type="model_version") |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
269 |
session.add(pe) |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
270 |
if must_commit: |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
271 |
session.commit() |
|
11
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
272 |
|
|
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
273 |
def setup_database(*args, **kwargs): |
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
274 |
|
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
275 |
session_argname = [ 'autoflush','binds', "class_", "_enable_transaction_accounting","expire_on_commit", "extension", "query_cls", "twophase", "weak_identity_map", "autocommit"] |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
276 |
|
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
277 |
kwargs_ce = dict((k, v) for k,v in kwargs.items() if (k not in session_argname and k != "create_all")) |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
278 |
|
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
279 |
engine = create_engine(*args, **kwargs_ce) |
|
888
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
280 |
|
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
281 |
if engine.name == "sqlite": |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
282 |
@event.listens_for(Engine, "connect") |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
283 |
def set_sqlite_pragma(dbapi_connection, connection_record): |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
284 |
cursor = dbapi_connection.cursor() |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
285 |
cursor.execute("PRAGMA foreign_keys=ON") |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
286 |
cursor.close() |
|
6fc6637d8403
update listener. add support for twitter regulation messages. update virtualenv
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
529
diff
changeset
|
287 |
|
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
288 |
metadata = Base.metadata |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
289 |
|
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
290 |
kwargs_sm = {'bind': engine} |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
291 |
|
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
292 |
kwargs_sm.update([(argname, kwargs[argname]) for argname in session_argname if argname in kwargs]) |
|
11
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
293 |
|
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
294 |
Session = sessionmaker(**kwargs_sm) |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
295 |
#set model version |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
296 |
|
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
297 |
if kwargs.get('create_all', True): |
|
11
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
298 |
metadata.create_all(engine) |
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
299 |
session = Session() |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
300 |
try: |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
301 |
add_model_version(session) |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
302 |
finally: |
|
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
303 |
session.close() |
|
11
54d7f1486ac4
implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
9
diff
changeset
|
304 |
|
|
289
a5eff8f2b81d
improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
263
diff
changeset
|
305 |
return (engine, metadata, Session) |
|
9
bb44692e09ee
script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff
changeset
|
306 |