script/rest/search_twitter.py
author Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
Sat, 01 Jun 2013 01:32:22 +0200
changeset 916 5dce89631093
parent 914 43876221071f
child 917 c47f290a001f
permissions -rw-r--r--
correct search twitter after api change
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
914
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
     1
from iri_tweet import models, processor, utils
891
8628c590f608 Remove old script and correct obvious script errors
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 464
diff changeset
     2
from optparse import OptionParser
18
bd595ad770fc - replace json with anyjson
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 15
diff changeset
     3
import anyjson
891
8628c590f608 Remove old script and correct obvious script errors
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 464
diff changeset
     4
import re
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
     5
import twitter
122
4c3a15877f80 clean php and python scripts
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 18
diff changeset
     6
9
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     7
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     8
def get_option():
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
     9
    
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    10
    parser = OptionParser()
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    11
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    12
    parser.add_option("-l", "--log", dest="logfile",
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    13
                      help="log to file", metavar="LOG", default="stderr")
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    14
    parser.add_option("-v", dest="verbose", action="count",
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    15
                      help="verbose", metavar="VERBOSE", default=0)
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    16
    parser.add_option("-q", dest="quiet", action="count",
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    17
                      help="quiet", metavar="QUIET", default=0)
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    18
    parser.add_option("-Q", dest="query",
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    19
                      help="query", metavar="QUERY")
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    20
    parser.add_option("-P", dest="rpp", metavar="RPP", default="50",
15
5d552b6a0e55 add oauth authentication to tweetstream
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 11
diff changeset
    21
                      help="Result per page")
5d552b6a0e55 add oauth authentication to tweetstream
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 11
diff changeset
    22
    parser.add_option("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",
5d552b6a0e55 add oauth authentication to tweetstream
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 11
diff changeset
    23
                      help="Token file name")
916
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    24
    parser.add_option("-k", "--key", dest="consumer_key",
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    25
                        help="Twitter consumer key", metavar="CONSUMER_KEY")
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    26
    parser.add_option("-s", "--secret", dest="consumer_secret",
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    27
                        help="Twitter consumer secret", metavar="CONSUMER_SECRET")
15
5d552b6a0e55 add oauth authentication to tweetstream
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 11
diff changeset
    28
    
914
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    29
    return parser.parse_args()
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    30
914
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    31
def get_auth(options, access_token):
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    32
    consumer_key = options.consumer_key
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    33
    consumer_secret = options.consumer_secret
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    34
    auth = twitter.OAuth(token=access_token[0], token_secret=access_token[1], consumer_key=consumer_key, consumer_secret=consumer_secret)
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    35
    return auth
9
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    36
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    37
if __name__ == "__main__":
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    38
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    39
    (options, args) = get_option()
914
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    40
    
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    41
    access_token = utils.get_oauth_token(consumer_key=options.consumer_key, consumer_secret=options.consumer_secret, token_file_path=options.token_filename)
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    42
    auth = get_auth(options, access_token)
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    43
914
43876221071f update search utilities
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 891
diff changeset
    44
    t = twitter.Twitter(domain="api.twitter.com",api_version="1.1",secure=True, auth=auth)
255
500cd0405c7a improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 242
diff changeset
    45
    
500cd0405c7a improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 242
diff changeset
    46
    conn_str = args[0].strip()
500cd0405c7a improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 242
diff changeset
    47
    if not re.match("^\w+://.+", conn_str):
500cd0405c7a improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 242
diff changeset
    48
        conn_str = 'sqlite:///' + conn_str
500cd0405c7a improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 242
diff changeset
    49
500cd0405c7a improve multi processing architecture
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 242
diff changeset
    50
    
289
a5eff8f2b81d improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 255
diff changeset
    51
    engine, metadata, Session = models.setup_database(conn_str, echo=((options.verbose-options.quiet)>0), create_all=True)
a5eff8f2b81d improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 255
diff changeset
    52
    session = None
9
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    53
    try:
289
a5eff8f2b81d improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 255
diff changeset
    54
        session = Session()
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    55
        #conn.row_factory = sqlite3.Row
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    56
        #curs = conn.cursor()
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    57
        #curs.execute("create table if not exists tweet_tweet (json);")
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    58
        #conn.commit()
9
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    59
        
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    60
        results = None        
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    61
        page = 1
411
0471e6eb8a1b add merge to export
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 289
diff changeset
    62
        print options.query
916
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    63
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    64
        #get current_maxid
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    65
        results = t.search.tweets(q=options.query, result_type="recent")
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    66
        max_id = results.get('search_metadata',{}).get('max_id',0)
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    67
        if max_id==0:
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    68
            print("No results, exit")
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    69
            sys.exit(0)
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    70
        
916
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    71
        while page <= int(1500/int(options.rpp)) and \
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    72
            ( results is None  or len(results.get('statuses',0)) > 0) and \
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    73
            max_id > 0:
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    74
            results = t.search.tweets(q=options.query, count=options.rpp, max_id=max_id, include_entities=True, result_type='recent')
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    75
            max_id = results.get('search_metadata',{}).get('since_id',1) - 1
411
0471e6eb8a1b add merge to export
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 289
diff changeset
    76
            
916
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    77
            for tweet in results["statuses"]:
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    78
                print tweet
18
bd595ad770fc - replace json with anyjson
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 15
diff changeset
    79
                tweet_str = anyjson.serialize(tweet)
bd595ad770fc - replace json with anyjson
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 15
diff changeset
    80
                #invalidate user id
916
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    81
                p = processor.TwitterProcessorStatus(json_dict=tweet, json_txt=tweet_str, source_id=None, session=session, consumer_token=(options.consumer_key, options.consumer_secret), access_token=access_token, token_filename=options.token_filename, user_query_twitter=False, logger=None)
5dce89631093 correct search twitter after api change
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 914
diff changeset
    82
                p.process()
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    83
                session.flush()
122
4c3a15877f80 clean php and python scripts
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 18
diff changeset
    84
                session.commit()
11
54d7f1486ac4 implement get_oauth_token
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 9
diff changeset
    85
            page += 1
122
4c3a15877f80 clean php and python scripts
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 18
diff changeset
    86
            #session.commit()
9
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    87
    finally:
289
a5eff8f2b81d improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 255
diff changeset
    88
        if session:
a5eff8f2b81d improve session maker creation + models version + add model version in db
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents: 255
diff changeset
    89
            session.close()
9
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    90
bb44692e09ee script apres traitement enmi
Yves-Marie Haussonne <1218002+ymph@users.noreply.github.com>
parents:
diff changeset
    91