script/utils/search_twitter_api.py
author ymh <ymh.work@gmail.com>
Wed, 18 Dec 2019 09:53:36 +0100
changeset 1523 53f1b28188f0
parent 1497 14a9bed2e3cd
permissions -rw-r--r--
Take into account change into twitter
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
import argparse
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
     2
import datetime
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
     3
import functools
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
     4
import json
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
import logging
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
import math
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
import re
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
import time
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
import urllib
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    10
from enum import Enum
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
import requests
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
import twitter
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    14
from blessings import Terminal
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
from iri_tweet import models, utils
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
from iri_tweet.processor import TwitterProcessorStatus
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
logger = logging.getLogger(__name__)
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
APPLICATION_NAME = "Tweet seach json"
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    24
class SearchType(Enum):
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    25
    standard = 'standard'
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    26
    _30day = '30day'
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    27
    full = 'full'
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    28
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    29
    def __str__(self):
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    30
        return self.value
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    31
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    32
def pass_kwargs_as_json(f):
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    33
    def kwargs_json_wrapper(*args, **kwargs):
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    34
        normal_kwargs = { k:v for k,v in kwargs.items() if k[0] != "_" }
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    35
        special_kwargs = { k:v for k,v in kwargs.items() if k[0] == "_" }
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    36
        new_kwargs = { **special_kwargs, '_json': normal_kwargs }
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    37
        return f(*args, **new_kwargs)
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    38
    return kwargs_json_wrapper
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    39
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
# TODO: implement some more parameters
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
# script to "scrap twitter results"
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
# Shamelessly taken from https://github.com/Jefferson-Henrique/GetOldTweets-python
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
# pyquery cssselect
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
class TweetManager:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    46
    def __init__(self, twitter_con, query, search_type, api_env):
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
        self.query = query
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    48
        self.search_type = search_type
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    49
        self.next = ""
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
        self.t = twitter_con
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    51
        self.api_env = api_env
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    52
        self.twitter_api = self.get_twitter_api()
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    53
        self.rate_limit_remaining = 0
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    54
        self.rate_limit_limit = 0
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    55
        self.rate_limit_reset = 0
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    56
        self.i = 0
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    57
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    58
    def get_twitter_api(self):
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    59
        return {
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    60
            SearchType.standard: lambda t: t.search.tweets,
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    61
            SearchType._30day:   lambda t: pass_kwargs_as_json(functools.partial(getattr(getattr(t.tweets.search,'30day'),self.api_env), _method="POST")),
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    62
            SearchType.full:     lambda t: pass_kwargs_as_json(functools.partial(getattr(t.tweets.search.fullarchive, self.api_env), _method="POST")),
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    63
        }[self.search_type](self.t)
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
    def __iter__(self):
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        while True:
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    67
            if self.next is None:
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
                break
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    69
            self.i = self.i+1
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    70
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    71
            # with open("json_dump_%s.json" % self.i, 'r') as fp:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    72
            #     jsondata = json.load(fp)
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    73
            jsondata = self.get_json_response()
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    75
            self.rate_limit_remaining = jsondata.rate_limit_remaining
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    76
            self.rate_limit_limit = jsondata.rate_limit_limit
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    77
            self.rate_limit_reset = jsondata.rate_limit_reset
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    78
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    79
            with open("json_dump_%s.json" % self.i, 'w') as fp:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    80
                json.dump(jsondata, fp)
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    82
            if self.search_type == SearchType.standard:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    83
                next_results = jsondata['search_metadata'].get('next_results', "?")[1:]
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    84
                self.next = urllib.parse.parse_qs(next_results).get('max_id', [None])[0]
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    85
                tweet_list = jsondata['statuses']
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    86
            else:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    87
                self.next = jsondata.get('next')
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    88
                tweet_list = jsondata['results']
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
            if len(tweet_list) == 0:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
                break
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
            for tweet in tweet_list:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
                yield tweet
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
    def get_json_response(self):
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    97
        if self.search_type == SearchType.standard:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    98
            return self.twitter_api(q=self.query, include_entities=True, max_id=int(self.next) if self.next else 0)
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
    99
        else:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   100
            kwargs = { "query": self.query, "maxResults": 100 }
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   101
            if self.next:
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   102
                kwargs["next"] = self.next
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   103
            return self.twitter_api(**kwargs)
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   105
def get_options():
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
    usage = "usage: %(prog)s [options] <connection_str_or_filepath>"
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
    parser = argparse.ArgumentParser(usage=usage)
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
    parser.add_argument(dest="conn_str",
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
                        help="write tweet to DATABASE. This is a connection string", metavar="CONNECTION_STR")
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
    parser.add_argument("-Q", dest="query",
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   114
                        help="query", metavar="QUERY")
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
    parser.add_argument("-k", "--key", dest="consumer_key",
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
                        help="Twitter consumer key", metavar="CONSUMER_KEY")
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
    parser.add_argument("-s", "--secret", dest="consumer_secret",
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
                        help="Twitter consumer secret", metavar="CONSUMER_SECRET")
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   119
    parser.add_argument("-t", dest="token_filename", metavar="TOKEN_FILENAME", default=".oauth_token",
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   120
                        help="Token file name")
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   121
    parser.add_argument("-a", dest="search_type", metavar="SEARCH_TYPE", default=SearchType.standard, choices=list(SearchType), type=SearchType,
1523
53f1b28188f0 Take into account change into twitter
ymh <ymh.work@gmail.com>
parents: 1497
diff changeset
   122
                        help="Twitter search type ('standard', '30day', 'full')")
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   123
    parser.add_argument("-e", dest="api_env", metavar="API_ENV", default="dev",
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   124
                        help="Twitter api dev environment")
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   125
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
    utils.set_logging_options(parser)
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
    return parser.parse_args()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
if __name__ == "__main__":
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
    options = get_options()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   135
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   136
    print("the search type is : %s" % options.search_type)
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   137
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
    utils.set_logging(options)
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   140
    bearer_token = utils.get_oauth2_token(consumer_key=options.consumer_key, consumer_secret=options.consumer_secret, token_file_path=options.token_filename, application_name=APPLICATION_NAME)
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   141
    twitter_auth = twitter.OAuth2(options.consumer_key, options.consumer_secret, bearer_token)
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   143
    t = twitter.Twitter(domain="api.twitter.com", auth=twitter_auth, secure=True)
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   144
    t.secure = True
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
    conn_str = options.conn_str.strip()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
    if not re.match(r"^\w+://.+", conn_str):
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
        conn_str = 'sqlite:///' + conn_str
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150
    engine, metadata, Session = models.setup_database(conn_str, echo=((options.verbose-options.quiet)>0), create_all=True)
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   151
    session = None
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   154
    term = Terminal()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   156
    try:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
        session = Session()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   158
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   159
        results = None
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
        print(options.query)
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   162
        tm = TweetManager(t, options.query, options.search_type, options.api_env)
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
        move_up = 0
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
        for i,tweet in enumerate(tm):
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
            # get id
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   168
            tweet_id = tweet.get("id")
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   170
            if not tweet_id:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   171
                continue
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   172
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
            if move_up > 0:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
                print((move_up+1)*term.move_up())
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
                move_up = 0
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   176
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   177
            print ("%d: %s - %r" % (i+1, tweet_id, tweet.get("text", "") ) + term.clear_eol())
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
            move_up += 1
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   179
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
            count_tweet = session.query(models.Tweet).filter_by(id_str=tweet_id).count()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   181
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   182
            if count_tweet:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   183
                continue
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   184
1497
14a9bed2e3cd Adapt recorder_stream to python 3
ymh <ymh.work@gmail.com>
parents: 1496
diff changeset
   185
            processor = TwitterProcessorStatus(tweet, None, None, session, twitter_auth=twitter_auth, logger=logger)
1496
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   186
            processor.process()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   187
            session.flush()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   188
            session.commit()
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   189
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   190
    except twitter.api.TwitterHTTPError as e:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   191
        fmt = ("." + e.format) if e.format else ""
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   192
        print("Twitter sent status %s for URL: %s%s using parameters: (%s)\ndetails: %s" % (repr(e.e.code), repr(e.uri), repr(fmt), repr(e.uriparts), repr(e.response_data)))
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   193
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   194
    finally:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   195
        if session:
184372ec27e2 upgrade to python 3 and twitter api
ymh <ymh.work@gmail.com>
parents:
diff changeset
   196
            session.close()