tweetcast/server-gevent/tweetcast.py
author Raphael Velt <raph.velt@gmail.com>
Mon, 12 Dec 2011 12:54:14 +0100
changeset 404 89968844eb7d
child 405 6626b728b142
permissions -rwxr-xr-x
Tweetcast: heavy refactoring !
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
404
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/env python
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     2
# -*- coding: utf-8 -*-
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     3
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     4
from gevent import monkey; monkey.patch_all()
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     5
# Importer d'abord, sinon exception
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     6
import anyjson, gevent, psycopg2
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     7
from sqlalchemy import (Boolean, Column, BigInteger, Integer, String, 
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     8
    ForeignKey, DateTime, create_engine, asc, func)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
     9
from sqlalchemy.orm import backref, relationship, sessionmaker, joinedload
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    10
from sqlalchemy.ext.declarative import declarative_base
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    11
from gevent.pywsgi import WSGIServer
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    12
from urlparse import parse_qs
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    13
import datetime
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    14
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    15
Base = declarative_base()
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    16
engine = create_engine('postgresql://raph:yusao6Kh@localhost/tweets')
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    17
Session = sessionmaker(bind=engine)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    18
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    19
class TweetSource(Base):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    20
    __tablename__ = 'tweet_tweet_source'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    21
    id = Column(Integer, primary_key=True, autoincrement=True)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    22
    original_json = Column(String)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    23
    received_at = Column(DateTime, default=datetime.datetime.utcnow, index=True)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    24
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    25
class Tweet(Base):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    26
    __tablename__ = 'tweet_tweet'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    27
    id = Column(BigInteger, primary_key=True, autoincrement=False)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    28
    tweet_source_id = Column(Integer, ForeignKey('tweet_tweet_source.id'))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    29
    tweet_source = relationship("TweetSource", backref="tweet")
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    30
    def jsondict(self):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    31
        tweetdict = anyjson.deserialize(self.tweet_source.original_json)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    32
        keys_to_delete = [
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    33
            'in_reply_to_screen_name',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    34
            'in_reply_to_user_id',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    35
            'retweeted',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    36
            'place',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    37
            'geo',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    38
            'source',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    39
            'contributors',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    40
            'coordinates',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    41
            'retweet_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    42
            'favorited',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    43
            'truncated',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    44
            'possibly_sensitive'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    45
        ]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    46
        user_keys_to_delete = [
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    47
            'default_profile_image',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    48
            'show_all_inline_media',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    49
            'contributors_enabled',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    50
            'profile_sidebar_fill_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    51
            'created_at',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    52
            'lang',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    53
            'time_zone',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    54
            'profile_sidebar_border_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    55
            'follow_request_sent',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    56
            'profile_background_image_url',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    57
            'profile_background_image_url_https',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    58
            'followers_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    59
            'description',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    60
            'url',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    61
            'geo_enabled',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    62
            'profile_use_background_image',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    63
            'default_profile',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    64
            'following',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    65
            'profile_text_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    66
            'is_translator',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    67
            'favourites_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    68
            'listed_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    69
            'friends_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    70
            'profile_link_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    71
            'protected',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    72
            'location',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    73
            'notifications',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    74
            'profile_image_url_https',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    75
            'statuses_count',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    76
            'verified',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    77
            'profile_background_color',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    78
            'profile_background_tile',
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    79
            'utc_offset'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    80
        ]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    81
        
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    82
        def textids(dictionary):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    83
            idfields = [key for key in dictionary if key[-2:] == 'id']
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    84
            for key in idfields:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    85
                keystr = key + '_str'
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    86
                if keystr in dictionary:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    87
                    dictionary[key] = dictionary[keystr]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    88
                    del dictionary[keystr]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    89
                        
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    90
        for key in keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    91
            if key in tweetdict:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    92
                del tweetdict[key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    93
        for key in user_keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    94
            if key in tweetdict['user']:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    95
                del tweetdict['user'][key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    96
        textids(tweetdict)
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    97
        textids(tweetdict['user'])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    98
        if 'retweeted_status' in tweetdict:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
    99
            for key in keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   100
                if key in tweetdict['retweeted_status']:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   101
                    del tweetdict['retweeted_status'][key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   102
            for key in user_keys_to_delete:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   103
                if key in tweetdict['retweeted_status']['user']:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   104
                    del tweetdict['retweeted_status']['user'][key]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   105
            textids(tweetdict['retweeted_status'])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   106
        return tweetdict
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   107
        
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   108
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   109
def webserver(env, start_response):
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   110
	if env['PATH_INFO'] == '/':
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   111
		httpquery = parse_qs(env['QUERY_STRING'])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   112
		print "serving tweets to", env['REMOTE_ADDR'], httpquery
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   113
		query = session.query(Tweet).order_by(asc(Tweet.id)).options(joinedload(Tweet.tweet_source))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   114
		if "since_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   115
		    query = query.filter(Tweet.id >= long(httpquery["since_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   116
		if "after_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   117
		    query = query.filter(Tweet.id > long(httpquery["after_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   118
		if "max_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   119
		    query = query.filter(Tweet.id <= long(httpquery["max_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   120
		if "before_id" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   121
		    query = query.filter(Tweet.id < long(httpquery["before_id"][0]))
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   122
		if "limit" in httpquery:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   123
			result = query[:int(httpquery["limit"][0])]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   124
		else:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   125
			result = query
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   126
		start_response('200 OK', [('Content-Type', 'application/javascript' if "callback" in httpquery else 'application/json' )])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   127
		return ["%s%s%s"%(
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   128
			"%s("%httpquery["callback"][0] if "callback" in httpquery else "",
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   129
			anyjson.serialize({"tweets" : [t.jsondict() for t in result]}),
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   130
			")" if "callback" in httpquery else ""
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   131
		)]
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   132
	else:
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   133
		start_response('404 Not Found', [('Content-Type', 'text/html')])
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   134
		return ['<h1>Not Found</h1>']
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   135
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   136
session = Session()
89968844eb7d Tweetcast: heavy refactoring !
Raphael Velt <raph.velt@gmail.com>
parents:
diff changeset
   137
WSGIServer(('', 8888), webserver).serve_forever()