#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gevent import monkey; monkey.patch_all()
# Importer d'abord, sinon exception
import anyjson, gevent, psycopg2
from sqlalchemy import (Boolean, Column, BigInteger, Integer, String,
ForeignKey, DateTime, create_engine, desc, func)
from sqlalchemy.orm import backref, relationship, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from gevent.pywsgi import WSGIServer
from urlparse import parse_qs
annotation_keywords = {
"positive" : '++',
"negative" : '--',
"reference" : '==',
"question" : '??'
}
Base = declarative_base()
engine = create_engine('postgresql://postgres:doiteshimashite@localhost/tweet_live')
Session = sessionmaker(bind=engine)
class EntityType(Base):
__tablename__ = "tweet_entity_type"
id = Column(Integer, primary_key=True, autoincrement=True)
label = Column(String)
class Entity(Base):
__tablename__ = "tweet_entity"
id = Column(Integer, primary_key=True)
tweet_id = Column(BigInteger, ForeignKey('tweet_tweet.id'))
type = Column(String)
entity_type_id = Column(Integer, ForeignKey('tweet_entity_type.id'), nullable=False)
entity_type = relationship("EntityType", backref="entities")
indice_start = Column(Integer)
indice_end = Column(Integer)
source = Column(String)
__mapper_args__ = {'polymorphic_on': type, 'polymorphic_identity': 'entity_entity', 'with_polymorphic':'*'}
def jsondict(self):
return {
"indice_start" : self.indice_start,
"indice_end" : self.indice_end,
"type" : self.type
}
class Tweet(Base):
__tablename__ = 'tweet_tweet'
id = Column(BigInteger, primary_key=True, autoincrement=False)
created_at = Column(DateTime)
text = Column(String)
user_id = Column(Integer, ForeignKey('tweet_user.id'))
user = relationship("User", backref="tweets")
entity_list = relationship(Entity, backref='tweet')
def annotations(self):
aa = []
for a in annotation_keywords:
n = self.text.count(annotation_keywords[a])
if n:
aa.append({
"name" : a,
"text" : annotation_keywords[a],
"count" : n
})
return aa
def jsondict(self):
return {
"id" : str(self.id),
"created_at" : str(self.created_at),
"text" : self.text,
"user" : self.user.jsondict(),
"entities" : [en.jsondict() for en in self.entity_list],
"annotations" : self.annotations()
}
class User(Base):
__tablename__ = "tweet_user"
id = Column(BigInteger, primary_key=True, autoincrement=False)
screen_name = Column(String, index=True)
profile_image_url = Column(String)
def jsondict(self):
return {
"id" : str(self.id),
"screen_name" : self.screen_name,
"profile_image_url" : self.profile_image_url
}
class Hashtag(Base):
__tablename__ = "tweet_hashtag"
id = Column(Integer, primary_key=True)
text = Column(String, unique=True, index=True)
def jsondict(self):
return {
"text" : self.text
}
class Url(Base):
__tablename__ = "tweet_url"
id = Column(Integer, primary_key=True)
url = Column(String, unique=True)
expanded_url = Column(String)
def jsondict(self):
return {
"url" : self.url,
"expanded_url" : self.expanded_url
}
class Media(Base):
__tablename__ = "tweet_media"
id = Column(BigInteger, primary_key=True, autoincrement=False)
url = Column(String)
expanded_url = Column(String)
def jsondict(self):
return {
"url" : self.url,
"expanded_url" : self.expanded_url
}
class EntityHashtag(Entity):
__tablename__ = "tweet_entity_hashtag"
__mapper_args__ = {'polymorphic_identity': 'entity_hashtag'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
hashtag_id = Column(Integer, ForeignKey("tweet_hashtag.id"))
hashtag = relationship(Hashtag, primaryjoin=hashtag_id == Hashtag.id)
def jsondict(self):
d = super(EntityHashtag, self).jsondict()
d['entity'] = self.hashtag.jsondict()
return d
class EntityUrl(Entity):
__tablename__ = "tweet_entity_url"
__mapper_args__ = {'polymorphic_identity': 'entity_url'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
url_id = Column(Integer, ForeignKey("tweet_url.id"))
url = relationship(Url, primaryjoin=url_id == Url.id)
def jsondict(self):
d = super(EntityUrl, self).jsondict()
d['entity'] = self.url.jsondict()
return d
class EntityUser(Entity):
__tablename__ = "tweet_entity_user"
__mapper_args__ = {'polymorphic_identity': 'entity_user'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
user_id = Column(BigInteger, ForeignKey('tweet_user.id'))
user = relationship(User, primaryjoin=(user_id == User.id))
def jsondict(self):
d = super(EntityUser, self).jsondict()
d['entity'] = self.user.jsondict()
return d
class EntityMedia(Entity):
__tablename__ = "tweet_entity_media"
__mapper_args__ = {'polymorphic_identity': 'entity_media'}
id = Column(Integer, ForeignKey('tweet_entity.id'), primary_key=True)
media_id = Column(BigInteger, ForeignKey('tweet_media.id'))
media = relationship(Media, primaryjoin=(media_id == Media.id))
def jsondict(self):
d = super(EntityMedia, self).jsondict()
d['entity'] = self.media.jsondict()
return d
# ranges = []
# lastid = 0L
#
# def define_ranges:
#
def webserver(env, start_response):
if env['PATH_INFO'] == '/':
httpquery = parse_qs(env['QUERY_STRING'])
print "serving tweets to", env['REMOTE_ADDR'], httpquery
query = session.query(Tweet)
if "since_id" in httpquery:
query = query.filter(Tweet.id >= long(httpquery["since_id"][0]))
if "after_id" in httpquery:
query = query.filter(Tweet.id > long(httpquery["after_id"][0]))
if "max_id" in httpquery:
query = query.filter(Tweet.id <= long(httpquery["max_id"][0]))
if "before_id" in httpquery:
query = query.filter(Tweet.id < long(httpquery["before_id"][0]))
query = query.order_by(desc(Tweet.id))
if "limit" in httpquery:
result = query[:int(httpquery["limit"][0])]
else:
result = query[:200]
start_response('200 OK', [('Content-Type', 'application/javascript' if "callback" in httpquery else 'application/json' )])
return ["%s%s%s"%(
"%s("%httpquery["callback"][0] if "callback" in httpquery else "",
anyjson.serialize({"tweets" : [t.jsondict() for t in result]}),
")" if "callback" in httpquery else ""
)]
else:
start_response('404 Not Found', [('Content-Type', 'text/html')])
return ['<h1>Not Found</h1>']
session = Session()
WSGIServer(('', 8888), webserver).serve_forever()