tweetcast/server-gevent/tweetcast.py
author Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
Mon, 19 Dec 2011 00:30:04 +0100
changeset 425 b346fd32fc34
parent 405 6626b728b142
child 438 892c3d9f635c
permissions -rwxr-xr-x
prepare for publication, add sync info
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
404
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/env python
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     2
# -*- coding: utf-8 -*-
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     3
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     4
from gevent import monkey; monkey.patch_all()
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     5
# Importer d'abord, sinon exception
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     6
import anyjson, gevent, psycopg2
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     7
from sqlalchemy import (Boolean, Column, BigInteger, Integer, String, 
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     8
    ForeignKey, DateTime, create_engine, asc, func)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     9
from sqlalchemy.orm import backref, relationship, sessionmaker, joinedload
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    10
from sqlalchemy.ext.declarative import declarative_base
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    11
from gevent.pywsgi import WSGIServer
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    12
from urlparse import parse_qs
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    13
import datetime
405
6626b728b142 Added configuration file to Tweetcast Gevent Server
Raphael Velt <raph.velt@gmail.com>
parents: 404
diff changeset
    14
from server_setup import SQL_CONNECT, WEB_PORT
404
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    15
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    16
Base = declarative_base()
405
6626b728b142 Added configuration file to Tweetcast Gevent Server
Raphael Velt <raph.velt@gmail.com>
parents: 404
diff changeset
    17
engine = create_engine(SQL_CONNECT)
404
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    18
Session = sessionmaker(bind=engine)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    19
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    20
class TweetSource(Base):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    21
    __tablename__ = 'tweet_tweet_source'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    22
    id = Column(Integer, primary_key=True, autoincrement=True)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    23
    original_json = Column(String)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    24
    received_at = Column(DateTime, default=datetime.datetime.utcnow, index=True)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    25
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    26
class Tweet(Base):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    27
    __tablename__ = 'tweet_tweet'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    28
    id = Column(BigInteger, primary_key=True, autoincrement=False)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    29
    tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id'))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    30
    tweet_source = relationship("TweetSource", backref="tweet")
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    31
    def jsondict(self):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    32
        tweetdict = anyjson.deserialize(self.tweet_source.original_json)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    33
        keys_to_delete = [
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    34
            'in_reply_to_screen_name',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    35
            'in_reply_to_user_id',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    36
            'retweeted',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    37
            'place',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    38
            'geo',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    39
            'source',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    40
            'contributors',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    41
            'coordinates',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    42
            'retweet_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    43
            'favorited',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    44
            'truncated',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    45
            'possibly_sensitive'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    46
        ]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    47
        user_keys_to_delete = [
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    48
            'default_profile_image',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    49
            'show_all_inline_media',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    50
            'contributors_enabled',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    51
            'profile_sidebar_fill_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    52
            'created_at',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    53
            'lang',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    54
            'time_zone',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    55
            'profile_sidebar_border_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    56
            'follow_request_sent',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    57
            'profile_background_image_url',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    58
            'profile_background_image_url_https',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    59
            'followers_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    60
            'description',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    61
            'url',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    62
            'geo_enabled',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    63
            'profile_use_background_image',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    64
            'default_profile',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    65
            'following',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    66
            'profile_text_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    67
            'is_translator',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    68
            'favourites_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    69
            'listed_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    70
            'friends_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    71
            'profile_link_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    72
            'protected',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    73
            'location',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    74
            'notifications',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    75
            'profile_image_url_https',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    76
            'statuses_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    77
            'verified',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    78
            'profile_background_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    79
            'profile_background_tile',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    80
            'utc_offset'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    81
        ]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    82
        
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    83
        def textids(dictionary):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    84
            idfields = [key for key in dictionary if key[-2:] == 'id']
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    85
            for key in idfields:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    86
                keystr = key + '_str'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    87
                if keystr in dictionary:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    88
                    dictionary[key] = dictionary[keystr]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    89
                    del dictionary[keystr]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    90
                        
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    91
        for key in keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    92
            if key in tweetdict:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    93
                del tweetdict[key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    94
        for key in user_keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    95
            if key in tweetdict['user']:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    96
                del tweetdict['user'][key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    97
        textids(tweetdict)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    98
        textids(tweetdict['user'])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    99
        if 'retweeted_status' in tweetdict:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   100
            for key in keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   101
                if key in tweetdict['retweeted_status']:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   102
                    del tweetdict['retweeted_status'][key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   103
            for key in user_keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   104
                if key in tweetdict['retweeted_status']['user']:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   105
                    del tweetdict['retweeted_status']['user'][key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   106
            textids(tweetdict['retweeted_status'])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   107
        return tweetdict
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   108
        
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   109
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   110
def webserver(env, start_response):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   111
	if env['PATH_INFO'] == '/':
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   112
		httpquery = parse_qs(env['QUERY_STRING'])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   113
		print "serving tweets to", env['REMOTE_ADDR'], httpquery
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   114
		query = session.query(Tweet).order_by(asc(Tweet.id)).options(joinedload(Tweet.tweet_source))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   115
		if "since_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   116
		    query = query.filter(Tweet.id >= long(httpquery["since_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   117
		if "after_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   118
		    query = query.filter(Tweet.id > long(httpquery["after_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   119
		if "max_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   120
		    query = query.filter(Tweet.id <= long(httpquery["max_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   121
		if "before_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   122
		    query = query.filter(Tweet.id < long(httpquery["before_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   123
		if "limit" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   124
			result = query[:int(httpquery["limit"][0])]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   125
		else:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   126
			result = query
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   127
		start_response('200 OK', [('Content-Type', 'application/javascript' if "callback" in httpquery else 'application/json' )])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   128
		return ["%s%s%s"%(
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   129
			"%s("%httpquery["callback"][0] if "callback" in httpquery else "",
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   130
			anyjson.serialize({"tweets" : [t.jsondict() for t in result]}),
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   131
			")" if "callback" in httpquery else ""
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   132
		)]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   133
	else:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   134
		start_response('404 Not Found', [('Content-Type', 'text/html')])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   135
		return ['<h1>Not Found</h1>']
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   136
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   137
session = Session()
425
b346fd32fc34 prepare for publication, add sync info
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 405
diff changeset
   138
b346fd32fc34 prepare for publication, add sync info
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 405
diff changeset
   139
if __name__ == "__main__":
b346fd32fc34 prepare for publication, add sync info
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 405
diff changeset
   140
    WSGIServer(('', WEB_PORT), webserver).serve_forever()