1 from sqlalchemy import Boolean, Table, Column, BigInteger, Integer, String, \

     2     MetaData, ForeignKey, DateTime, create_engine

     3 from sqlalchemy.ext.declarative import declarative_base

     4 from sqlalchemy.orm import relationship, backref

     5 import datetime

     6 import email.utils

     7 import simplejson

     8 

     9 

    10 Base = declarative_base()

    11 

    12 APPLICATION_NAME = "IRI_TWITTER" 

    13 CONSUMER_KEY = "54ThDZhpEjokcMgHJOMnQA"

    14 CONSUMER_SECRET = "wUoL9UL2T87tfc97R0Dff2EaqRzpJ5XGdmaN2XK3udA"

    15 #ACCESS_TOKEN_KEY= "47312923-LiNTtz0I18YXMVIrFeTuhmH7bOvYsK6p3Ln2Dc"

    16 #ACCESS_TOKEN_SECRET = "r3LoXVcjImNAElUpWqTu2SG2xCdWFHkva7xeQoncA"

    17 

    18 def adapt_date(date_str):

    19     ts = email.utils.parsedate_tz(date_str)

    20     return datetime.datetime(*ts[0:7])

    21 

    22 def adapt_json(obj):

    23     if obj is None:

    24         return None

    25     else:

    26         return simplejson.dumps(obj)

    27 

    28 class Entity(Base):

    29     __tablename__ = "tweet_entity"

    30     id = Column(Integer, primary_key = True)

    31     tweet_id = Column(BigInteger, ForeignKey('tweet_tweet.id'))

    32     #tweet = relationship(Tweet, primaryjoin = tweet_id == Tweet.id)

    33     type = Column(String)

    34     indice_start = Column(Integer)

    35     indice_end = Column(Integer)

    36     __mapper_args__ = {'polymorphic_on': type}

    37 

    38     def __init__(self, **kwargs):

    39         for key, value in kwargs.items():

    40             if hasattr(self,key):

    41                 setattr(self,key,value)

    42 

    43 

    44 class Tweet(Base):

    45     __tablename__ = 'tweet_tweet'

    46 

    47     id = Column(BigInteger, primary_key=True, autoincrement=False)

    48     id_str = Column(String)

    49     contributors = Column(String)

    50     coordinates =  Column(String) 

    51     created_at = Column(DateTime)

    52     favorited = Column(Boolean)

    53     geo = Column(String)

    54     in_reply_to_screen_name = Column(String)

    55     in_reply_to_status_id = Column(BigInteger) 

    56     in_reply_to_status_id_str = Column(String)

    57     in_reply_to_user_id = Column(Integer)

    58     in_reply_to_user_id_str = Column(String)

    59     place = Column(String)

    60     retweet_count = Column(Integer)

    61     retweeted = Column(Boolean)

    62     source = Column(String)

    63     text = Column(String)

    64     truncated = Column(Boolean)

    65     user_id = Column(Integer, ForeignKey('tweet_user.id'))

    66     original_json = Column(String)

    67     entity_list = relationship(Entity, backref='tweet')

    68     

    69     #user = relationship(User, primaryjoin=user_id == User.id)

    70     

    71     def __init__(self, **kwargs):

    72         for key, value in kwargs.items():

    73             if hasattr(self,key):

    74                 setattr(self,key,value)

    75     

    76 

    77 class User(Base):

    78     __tablename__ = "tweet_user"

    79     

    80     id = Column(Integer, primary_key = True, autoincrement=False)

    81     id_str= Column(String)

    82     contributors_enabled= Column(Boolean)

    83     created_at= Column(DateTime)

    84     description= Column(String)

    85     favourites_count = Column(Integer)

    86     follow_request_sent = Column(Boolean)

    87     followers_count = Column(Integer)

    88     following = Column(String)

    89     friends_count = Column(Integer)

    90     geo_enabled= Column(Boolean)

    91     is_translator= Column(Boolean)

    92     lang = Column(String)

    93     listed_count = Column(Integer)

    94     location= Column(String)

    95     name = Column(String)

    96     notifications = Column(String)

    97     profile_background_color= Column(String)

    98     profile_background_image_url= Column(String)

    99     profile_background_tile= Column(Boolean)

   100     profile_image_url= Column(String)

   101     profile_link_color= Column(String)

   102     profile_sidebar_border_color= Column(String)

   103     profile_sidebar_fill_color= Column(String)

   104     profile_text_color= Column(String)

   105     profile_use_background_image= Column(Boolean)

   106     protected= Column(Boolean)

   107     screen_name= Column(String)

   108     show_all_inline_media= Column(Boolean)

   109     statuses_count = Column(Integer)

   110     time_zone= Column(String)

   111     url= Column(String)

   112     utc_offset = Column(Integer)

   113     verified= Column(Boolean)

   114     tweets = relationship(Tweet, backref='user')

   115 

   116     def __init__(self, **kwargs):

   117         for key, value in kwargs.items():

   118             if hasattr(self,key):

   119                 setattr(self,key,value)

   120 

   121     

   122 

   123 class Hashtag(Base):

   124     __tablename__ = "tweet_hashtag"

   125     id = Column(Integer, primary_key=True)

   126     text = Column(String, unique = True)

   127     def __init__(self, **kwargs):

   128         for key, value in kwargs.items():

   129             if hasattr(self,key):

   130                 setattr(self,key,value)

   131 

   132 

   133 

   134 class Url(Base):

   135     __tablename__ = "tweet_url"

   136     id = Column(Integer, primary_key=True)

   137     url = Column(String, unique=True)

   138     expanded_url = Column(String)

   139     def __init__(self, **kwargs):

   140         for key, value in kwargs.items():

   141             if hasattr(self,key):

   142                 setattr(self,key,value)

   143 

   144     

   145 

   146 class EntityHashtag(Entity):

   147     __tablename__ = "tweet_entity_hashtag"

   148     __mapper_args__ = {'polymorphic_identity': 'entity_hashtag'}

   149     id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)

   150     hashtag_id = Column(Integer, ForeignKey("tweet_hashtag.id"))

   151     hashtag = relationship(Hashtag, primaryjoin=hashtag_id == Hashtag.id)

   152     def __init__(self, **kwargs):

   153         super(EntityHashtag, self).__init__(**kwargs)

   154         for key, value in kwargs.items():

   155             if hasattr(self,key):

   156                 setattr(self,key,value)

   157 

   158     

   159 class EntityUrl(Entity):

   160     __tablename__ = "tweet_entity_url"

   161     __mapper_args__ = {'polymorphic_identity': 'entity_url'}

   162     id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)

   163     url_id = Column(Integer, ForeignKey("tweet_url.id"))

   164     url = relationship(Url, primaryjoin=url_id == Url.id)

   165     def __init__(self, **kwargs):

   166         super(EntityUrl, self).__init__(**kwargs)

   167         for key, value in kwargs.items():

   168             if hasattr(self,key):

   169                 setattr(self,key,value)

   170 

   171 class EntityUser(Entity):

   172     __tablename__ = "tweet_entity_user"

   173     __mapper_args__ = {'polymorphic_identity': 'entity_user'}

   174     id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)

   175     user_id = Column(Integer, ForeignKey('tweet_user.id'))

   176     user = relationship(User, primaryjoin=user_id == User.id)

   177 

   178     def __init__(self, **kwargs):

   179         super(EntityUser, self).__init__(**kwargs)

   180         for key, value in kwargs.items():

   181             if hasattr(self,key):

   182                 setattr(self,key,value)

   183                 

   184                 

   185 def setup_database(*args, **kwargs):

   186         

   187     create_all = True        

   188     if "create_all" in kwargs:

   189         create_all = kwargs["create_all"]

   190         del(kwargs["create_all"])

   191 

   192     engine = create_engine(*args, **kwargs)

   193     metadata = Base.metadata

   194             

   195     if create_all:

   196         metadata.create_all(engine)

   197 

   198     return (engine, metadata)

   199 

   200 rest_tweet_tweet = {

   201     u'iso_language_code': 'unicode',

   202     u'text': 'unicode',

   203     u'from_user_id_str': 'unicode',

   204     u'profile_image_url': 'unicode',

   205     u'to_user_id_str': 'NoneType',

   206     u'created_at': 'unicode',

   207     u'source': 'unicode',

   208     u'to_user': 'unicode',

   209     u'id_str': 'unicode',

   210     u'from_user': 'unicode',

   211     u'place': {u'type': 'unicode', u'id': 'unicode', u'full_name': 'unicode'},

   212     u'from_user_id': 'int',

   213     u'to_user_id': 'NoneType',

   214     u'geo': 'NoneType',

   215     u'id': 'int',

   216     u'metadata': {u'result_type': 'unicode'}

   217 }

   218 

   219 tweet_tweet = {

   220     'contributors': None,

   221     'coordinates': None, 

   222     'created_at': 'date', 

   223     'entities': "tweet_entity", 

   224     'favorited': "bool",

   225     'geo': None,

   226     'id': "long",

   227     'id_str': "string",

   228     'in_reply_to_screen_name': "string", 

   229     'in_reply_to_status_id': "long", 

   230     'in_reply_to_status_id_str': "string",

   231     'in_reply_to_user_id': "int",

   232     'in_reply_to_user_id_str': "string",

   233     'place': "string",

   234     'retweet_count': "int",

   235     'retweeted': "bool",

   236     'source': "string",

   237     'text': "string",

   238     'truncated': "bool",

   239     'user': "tweet_user"

   240 }

   241 tweet_user = {

   242     'contributors_enabled': 'bool',

   243     'created_at': 'str',

   244     'description': 'str',

   245     'favourites_count': 'int',

   246     'follow_request_sent': None,

   247     'followers_count': 'int',

   248     'following': None,

   249     'friends_count': 'int',

   250     'geo_enabled': 'bool',

   251     'id': 'int',

   252     'id_str': 'str',

   253     'is_translator': 'bool',

   254     'lang': 'str',

   255     'listed_count': 'int',

   256     'location': 'str',

   257     'name': 'str',

   258     'notifications': 'NoneType',

   259     'profile_background_color': 'str',

   260     'profile_background_image_url': 'str',

   261     'profile_background_tile': 'bool',

   262     'profile_image_url': 'str',

   263     'profile_link_color': 'str',

   264     'profile_sidebar_border_color': 'str',

   265     'profile_sidebar_fill_color': 'str',

   266     'profile_text_color': 'str',

   267     'profile_use_background_image': 'bool',

   268     'protected': 'bool',

   269     'screen_name': 'str',

   270     'show_all_inline_media': 'bool',

   271     'statuses_count': 'int',

   272     'time_zone': 'str',

   273     'url': 'str',

   274     'utc_offset': 'int',

   275     'verified': 'bool',

   276 }

   277 

   278 

   279 tweet_entity_hashtag = {

   280     'hashtag' : 'tweet_hashtag',

   281     'indice_start' : 'int',

   282     'indice_end' : 'int',

   283     'tweet':'tweet_tweet'

   284 }

   285 

   286 tweet_entity_url = {

   287     'url' : 'tweet_url',

   288     'indice_start' : 'int',

   289     'indice_end' : 'int',

   290     'tweet':'tweet_tweet'

   291 }

   292 

   293 tweet_entity_user = {

   294     'user' : 'tweet_user',

   295     'indice_start' : 'int',

   296     'indice_end' : 'int',

   297     'tweet':'tweet_tweet'

   298 }

   299 

   300 #id int

   301 #id_str str

   302 #indices list

   303 #name str

   304 #screen_name str

   305 

   306 tweet_hashtag = {

   307     "text": "string"

   308 }

   309 

   310 tweet_url = {

   311     "url": "string",

   312     "expanded_url" : "string",    

   313 }

   314