from sqlalchemy import (Boolean, Column, BigInteger, Integer, String, ForeignKey,
DateTime, create_engine)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
import anyjson
import datetime
import email.utils
Base = declarative_base()
APPLICATION_NAME = "IRI_TWITTER"
CONSUMER_KEY = "54ThDZhpEjokcMgHJOMnQA"
CONSUMER_SECRET = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA"
ACCESS_TOKEN_KEY = None
ACCESS_TOKEN_SECRET = None
#ACCESS_TOKEN_KEY= "47312923-LiNTtz0I18YXMVIrFeTuhmH7bOvYsK6p3Ln2Dc"
#ACCESS_TOKEN_SECRET = "r3LoXVcjImNAElUpWqTu2SG2xCdWFHkva7xeQoncA"
def adapt_date(date_str):
ts = email.utils.parsedate_tz(date_str) #@UndefinedVariable
return datetime.datetime(*ts[0:7])
def adapt_json(obj):
if obj is None:
return None
else:
return anyjson.serialize(obj)
class EntityType(Base):
__tablename__ = "tweet_entity_type"
id = Column(Integer, primary_key=True, autoincrement=True)
label = Column(String)
class Entity(Base):
__tablename__ = "tweet_entity"
id = Column(Integer, primary_key=True)
tweet_id = Column(BigInteger, ForeignKey('tweet_tweet.id'))
type = Column(String)
entity_type_id = Column(Integer, ForeignKey('tweet_entity_type.id'), nullable=False)
entity_type = relationship("EntityType", backref="entities")
indice_start = Column(Integer)
indice_end = Column(Integer)
source = Column(String)
__mapper_args__ = {'polymorphic_on': type, 'polymorphic_identity': 'entity_entity', 'with_polymorphic':'*'}
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class TweetSource(Base):
__tablename__ = 'tweet_tweet_source'
id = Column(Integer, primary_key=True, autoincrement=True)
original_json = Column(String)
received_at = Column(DateTime, default=datetime.datetime.now(), index=True)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class TweetLog(Base):
TWEET_STATUS = {
'OK' : 1,
'ERROR' : 2,
'NOT_TWEET': 3,
}
__tablename__ = 'tweet_tweet_log'
id = Column(Integer, primary_key=True, autoincrement=True)
tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id'))
tweet_source = relationship("TweetSource", backref="logs")
status = Column(Integer)
error = Column(String)
error_stack = Column(String)
class Tweet(Base):
__tablename__ = 'tweet_tweet'
id = Column(BigInteger, primary_key=True, autoincrement=False)
id_str = Column(String)
contributors = Column(String)
coordinates = Column(String)
created_at = Column(DateTime)
favorited = Column(Boolean)
geo = Column(String)
in_reply_to_screen_name = Column(String)
in_reply_to_status_id = Column(BigInteger)
in_reply_to_status_id_str = Column(String)
in_reply_to_user_id = Column(BigInteger)
in_reply_to_user_id_str = Column(String)
place = Column(String)
retweet_count = Column(String)
retweeted = Column(Boolean)
source = Column(String)
text = Column(String)
truncated = Column(Boolean)
user_id = Column(Integer, ForeignKey('tweet_user.id'))
user = relationship("User", backref="tweets")
tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id'))
tweet_source = relationship("TweetSource", backref="tweet")
entity_list = relationship(Entity, backref='tweet')
received_at = Column(DateTime, default=datetime.datetime.now(), index=True)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class UserMessage(Base):
__tablename__ = "tweet_user_message"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey('tweet_user.id'))
user = relationship("User", backref="messages")
created_at = Column(DateTime, default=datetime.datetime.now())
message_id = Column(Integer, ForeignKey('tweet_message.id'))
class Message(Base):
__tablename__ = "tweet_message"
id = Column(Integer, primary_key=True)
created_at = Column(DateTime, default=datetime.datetime.now())
text = Column(String)
users = relationship(UserMessage, backref='message')
class User(Base):
__tablename__ = "tweet_user"
id = Column(BigInteger, primary_key=True, autoincrement=False)
id_str = Column(String)
contributors_enabled = Column(Boolean)
created_at = Column(DateTime)
description = Column(String)
favourites_count = Column(Integer)
follow_request_sent = Column(Boolean)
followers_count = Column(Integer)
following = Column(String)
friends_count = Column(Integer)
geo_enabled = Column(Boolean)
is_translator = Column(Boolean)
lang = Column(String)
listed_count = Column(Integer)
location = Column(String)
name = Column(String)
notifications = Column(String)
profile_background_color = Column(String)
profile_background_image_url = Column(String)
profile_background_tile = Column(Boolean)
profile_image_url = Column(String)
profile_link_color = Column(String)
profile_sidebar_border_color = Column(String)
profile_sidebar_fill_color = Column(String)
profile_text_color = Column(String)
profile_use_background_image = Column(Boolean)
protected = Column(Boolean)
screen_name = Column(String, index=True)
show_all_inline_media = Column(Boolean)
statuses_count = Column(Integer)
time_zone = Column(String)
url = Column(String)
utc_offset = Column(Integer)
verified = Column(Boolean)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class Hashtag(Base):
__tablename__ = "tweet_hashtag"
id = Column(Integer, primary_key=True)
text = Column(String, unique=True, index=True)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class Url(Base):
__tablename__ = "tweet_url"
id = Column(Integer, primary_key=True)
url = Column(String, unique=True)
expanded_url = Column(String)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class MediaType(Base):
__tablename__ = "tweet_media_type"
id = Column(Integer, primary_key=True, autoincrement=True)
label = Column(String, unique=True, index=True)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class Media(Base):
__tablename__ = "tweet_media"
id = Column(BigInteger, primary_key=True, autoincrement=False)
id_str = Column(String, unique=True)
media_url = Column(String, unique=True)
media_url_https = Column(String, unique=True)
url = Column(String)
display_url = Column(String)
expanded_url = Column(String)
sizes = Column(String)
type_id = Column(Integer, ForeignKey("tweet_media_type.id"))
type = relationship(MediaType, primaryjoin=type_id == MediaType.id)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class EntityHashtag(Entity):
__tablename__ = "tweet_entity_hashtag"
__mapper_args__ = {'polymorphic_identity': 'entity_hashtag'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
hashtag_id = Column(Integer, ForeignKey("tweet_hashtag.id"))
hashtag = relationship(Hashtag, primaryjoin=hashtag_id == Hashtag.id)
def __init__(self, **kwargs):
super(EntityHashtag, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class EntityUrl(Entity):
__tablename__ = "tweet_entity_url"
__mapper_args__ = {'polymorphic_identity': 'entity_url'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
url_id = Column(Integer, ForeignKey("tweet_url.id"))
url = relationship(Url, primaryjoin=url_id == Url.id)
def __init__(self, **kwargs):
super(EntityUrl, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class EntityUser(Entity):
__tablename__ = "tweet_entity_user"
__mapper_args__ = {'polymorphic_identity': 'entity_user'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
user_id = Column(BigInteger, ForeignKey('tweet_user.id'))
user = relationship(User, primaryjoin=(user_id == User.id))
def __init__(self, **kwargs):
super(EntityUser, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
class EntityMedia(Entity):
__tablename__ = "tweet_entity_media"
__mapper_args__ = {'polymorphic_identity': 'entity_media'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
media_id = Column(BigInteger, ForeignKey('tweet_media.id'))
media = relationship(Media, primaryjoin=(media_id == Media.id))
def __init__(self, **kwargs):
super(EntityMedia, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self, key):
setattr(self, key, value)
def setup_database(*args, **kwargs):
create_all = True
if "create_all" in kwargs:
create_all = kwargs["create_all"]
del(kwargs["create_all"])
engine = create_engine(*args, **kwargs)
metadata = Base.metadata
if create_all:
metadata.create_all(engine)
return (engine, metadata)
rest_tweet_tweet = {
u'iso_language_code': 'unicode',
u'text': 'unicode',
u'from_user_id_str': 'unicode',
u'profile_image_url': 'unicode',
u'to_user_id_str': 'NoneType',
u'created_at': 'unicode',
u'source': 'unicode',
u'to_user': 'unicode',
u'id_str': 'unicode',
u'from_user': 'unicode',
u'place': {u'type': 'unicode', u'id': 'unicode', u'full_name': 'unicode'},
u'from_user_id': 'int',
u'to_user_id': 'NoneType',
u'geo': 'NoneType',
u'id': 'int',
u'metadata': {u'result_type': 'unicode'}
}
tweet_tweet = {
'contributors': None,
'coordinates': None,
'created_at': 'date',
'entities': "tweet_entity",
'favorited': "bool",
'geo': None,
'id': "long",
'id_str': "string",
'in_reply_to_screen_name': "string",
'in_reply_to_status_id': "long",
'in_reply_to_status_id_str': "string",
'in_reply_to_user_id': "int",
'in_reply_to_user_id_str': "string",
'place': "string",
'retweet_count': "int",
'retweeted': "bool",
'source': "string",
'text': "string",
'truncated': "bool",
'user': "tweet_user"
}
tweet_user = {
'contributors_enabled': 'bool',
'created_at': 'str',
'description': 'str',
'favourites_count': 'int',
'follow_request_sent': None,
'followers_count': 'int',
'following': None,
'friends_count': 'int',
'geo_enabled': 'bool',
'id': 'int',
'id_str': 'str',
'is_translator': 'bool',
'lang': 'str',
'listed_count': 'int',
'location': 'str',
'name': 'str',
'notifications': 'NoneType',
'profile_background_color': 'str',
'profile_background_image_url': 'str',
'profile_background_tile': 'bool',
'profile_image_url': 'str',
'profile_link_color': 'str',
'profile_sidebar_border_color': 'str',
'profile_sidebar_fill_color': 'str',
'profile_text_color': 'str',
'profile_use_background_image': 'bool',
'protected': 'bool',
'screen_name': 'str',
'show_all_inline_media': 'bool',
'statuses_count': 'int',
'time_zone': 'str',
'url': 'str',
'utc_offset': 'int',
'verified': 'bool',
}
tweet_entity_hashtag = {
'hashtag' : 'tweet_hashtag',
'indice_start' : 'int',
'indice_end' : 'int',
'tweet':'tweet_tweet'
}
tweet_entity_url = {
'url' : 'tweet_url',
'indice_start' : 'int',
'indice_end' : 'int',
'tweet':'tweet_tweet'
}
tweet_entity_user = {
'user' : 'tweet_user',
'indice_start' : 'int',
'indice_end' : 'int',
'tweet':'tweet_tweet'
}
#id int
#id_str str
#indices list
#name str
#screen_name str
tweet_hashtag = {
"text": "string"
}
tweet_url = {
"url": "string",
"expanded_url" : "string",
}