from sqlalchemy import Boolean, Table, Column, BigInteger, Integer, String, \
MetaData, ForeignKey, DateTime, create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, backref
import datetime
import email.utils
import anyjson
Base = declarative_base()
APPLICATION_NAME = "IRI_TWITTER"
CONSUMER_KEY = "54ThDZhpEjokcMgHJOMnQA"
CONSUMER_SECRET = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA"
#ACCESS_TOKEN_KEY= "47312923-LiNTtz0I18YXMVIrFeTuhmH7bOvYsK6p3Ln2Dc"
#ACCESS_TOKEN_SECRET = "r3LoXVcjImNAElUpWqTu2SG2xCdWFHkva7xeQoncA"
def adapt_date(date_str):
ts = email.utils.parsedate_tz(date_str)
return datetime.datetime(*ts[0:7])
def adapt_json(obj):
if obj is None:
return None
else:
return anyjson.serialize(obj)
class Entity(Base):
__tablename__ = "tweet_entity"
id = Column(Integer, primary_key = True)
tweet_id = Column(BigInteger, ForeignKey('tweet_tweet.id'))
#tweet = relationship(Tweet, primaryjoin = tweet_id == Tweet.id)
type = Column(String)
indice_start = Column(Integer)
indice_end = Column(Integer)
__mapper_args__ = {'polymorphic_on': type}
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class Tweet(Base):
__tablename__ = 'tweet_tweet'
id = Column(BigInteger, primary_key=True, autoincrement=False)
id_str = Column(String)
contributors = Column(String)
coordinates = Column(String)
created_at = Column(DateTime)
favorited = Column(Boolean)
geo = Column(String)
in_reply_to_screen_name = Column(String)
in_reply_to_status_id = Column(BigInteger)
in_reply_to_status_id_str = Column(String)
in_reply_to_user_id = Column(Integer)
in_reply_to_user_id_str = Column(String)
place = Column(String)
retweet_count = Column(Integer)
retweeted = Column(Boolean)
source = Column(String)
text = Column(String)
truncated = Column(Boolean)
user_id = Column(Integer, ForeignKey('tweet_user.id'))
original_json = Column(String)
entity_list = relationship(Entity, backref='tweet')
#user = relationship(User, primaryjoin=user_id == User.id)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class User(Base):
__tablename__ = "tweet_user"
id = Column(Integer, primary_key = True, autoincrement=False)
id_str= Column(String)
contributors_enabled= Column(Boolean)
created_at= Column(DateTime)
description= Column(String)
favourites_count = Column(Integer)
follow_request_sent = Column(Boolean)
followers_count = Column(Integer)
following = Column(String)
friends_count = Column(Integer)
geo_enabled= Column(Boolean)
is_translator= Column(Boolean)
lang = Column(String)
listed_count = Column(Integer)
location= Column(String)
name = Column(String)
notifications = Column(String)
profile_background_color= Column(String)
profile_background_image_url= Column(String)
profile_background_tile= Column(Boolean)
profile_image_url= Column(String)
profile_link_color= Column(String)
profile_sidebar_border_color= Column(String)
profile_sidebar_fill_color= Column(String)
profile_text_color= Column(String)
profile_use_background_image= Column(Boolean)
protected= Column(Boolean)
screen_name= Column(String)
show_all_inline_media= Column(Boolean)
statuses_count = Column(Integer)
time_zone= Column(String)
url= Column(String)
utc_offset = Column(Integer)
verified= Column(Boolean)
tweets = relationship(Tweet, backref='user')
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class Hashtag(Base):
__tablename__ = "tweet_hashtag"
id = Column(Integer, primary_key=True)
text = Column(String, unique = True)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class Url(Base):
__tablename__ = "tweet_url"
id = Column(Integer, primary_key=True)
url = Column(String, unique=True)
expanded_url = Column(String)
def __init__(self, **kwargs):
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class EntityHashtag(Entity):
__tablename__ = "tweet_entity_hashtag"
__mapper_args__ = {'polymorphic_identity': 'entity_hashtag'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
hashtag_id = Column(Integer, ForeignKey("tweet_hashtag.id"))
hashtag = relationship(Hashtag, primaryjoin=hashtag_id == Hashtag.id)
def __init__(self, **kwargs):
super(EntityHashtag, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class EntityUrl(Entity):
__tablename__ = "tweet_entity_url"
__mapper_args__ = {'polymorphic_identity': 'entity_url'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
url_id = Column(Integer, ForeignKey("tweet_url.id"))
url = relationship(Url, primaryjoin=url_id == Url.id)
def __init__(self, **kwargs):
super(EntityUrl, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
class EntityUser(Entity):
__tablename__ = "tweet_entity_user"
__mapper_args__ = {'polymorphic_identity': 'entity_user'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
user_id = Column(Integer, ForeignKey('tweet_user.id'))
user = relationship(User, primaryjoin=user_id == User.id)
def __init__(self, **kwargs):
super(EntityUser, self).__init__(**kwargs)
for key, value in kwargs.items():
if hasattr(self,key):
setattr(self,key,value)
def setup_database(*args, **kwargs):
create_all = True
if "create_all" in kwargs:
create_all = kwargs["create_all"]
del(kwargs["create_all"])
engine = create_engine(*args, **kwargs)
metadata = Base.metadata
if create_all:
metadata.create_all(engine)
return (engine, metadata)
rest_tweet_tweet = {
u'iso_language_code': 'unicode',
u'text': 'unicode',
u'from_user_id_str': 'unicode',
u'profile_image_url': 'unicode',
u'to_user_id_str': 'NoneType',
u'created_at': 'unicode',
u'source': 'unicode',
u'to_user': 'unicode',
u'id_str': 'unicode',
u'from_user': 'unicode',
u'place': {u'type': 'unicode', u'id': 'unicode', u'full_name': 'unicode'},
u'from_user_id': 'int',
u'to_user_id': 'NoneType',
u'geo': 'NoneType',
u'id': 'int',
u'metadata': {u'result_type': 'unicode'}
}
tweet_tweet = {
'contributors': None,
'coordinates': None,
'created_at': 'date',
'entities': "tweet_entity",
'favorited': "bool",
'geo': None,
'id': "long",
'id_str': "string",
'in_reply_to_screen_name': "string",
'in_reply_to_status_id': "long",
'in_reply_to_status_id_str': "string",
'in_reply_to_user_id': "int",
'in_reply_to_user_id_str': "string",
'place': "string",
'retweet_count': "int",
'retweeted': "bool",
'source': "string",
'text': "string",
'truncated': "bool",
'user': "tweet_user"
}
tweet_user = {
'contributors_enabled': 'bool',
'created_at': 'str',
'description': 'str',
'favourites_count': 'int',
'follow_request_sent': None,
'followers_count': 'int',
'following': None,
'friends_count': 'int',
'geo_enabled': 'bool',
'id': 'int',
'id_str': 'str',
'is_translator': 'bool',
'lang': 'str',
'listed_count': 'int',
'location': 'str',
'name': 'str',
'notifications': 'NoneType',
'profile_background_color': 'str',
'profile_background_image_url': 'str',
'profile_background_tile': 'bool',
'profile_image_url': 'str',
'profile_link_color': 'str',
'profile_sidebar_border_color': 'str',
'profile_sidebar_fill_color': 'str',
'profile_text_color': 'str',
'profile_use_background_image': 'bool',
'protected': 'bool',
'screen_name': 'str',
'show_all_inline_media': 'bool',
'statuses_count': 'int',
'time_zone': 'str',
'url': 'str',
'utc_offset': 'int',
'verified': 'bool',
}
tweet_entity_hashtag = {
'hashtag' : 'tweet_hashtag',
'indice_start' : 'int',
'indice_end' : 'int',
'tweet':'tweet_tweet'
}
tweet_entity_url = {
'url' : 'tweet_url',
'indice_start' : 'int',
'indice_end' : 'int',
'tweet':'tweet_tweet'
}
tweet_entity_user = {
'user' : 'tweet_user',
'indice_start' : 'int',
'indice_end' : 'int',
'tweet':'tweet_tweet'
}
#id int
#id_str str
#indices list
#name str
#screen_name str
tweet_hashtag = {
"text": "string"
}
tweet_url = {
"url": "string",
"expanded_url" : "string",
}