script/utils/dump_public_toots.py
author ymh <ymh.work@gmail.com>
Fri, 15 Nov 2024 01:29:53 +0100
changeset 1575 ce1d5b0d1479
parent 1557 7c67caaafdeb
permissions -rwxr-xr-x
Correct some details
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1557
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/env python
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
"""
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Dump public toots
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
"""
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
# import sqlite3
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
# con = sqlite3.connect("registered_app.db")
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
# cur = con.cursor()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
# cur.execute("CREATE TABLE registered_app(title, year, score)")
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    14
import argparse
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
import collections
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
from sqlalchemy import Column, Integer, String, create_engine, UniqueConstraint, select
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
from sqlalchemy.orm import declarative_base, Session
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
from mastodon import Mastodon
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
import socket
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
import requests.packages.urllib3.util.connection as urllib3_cn
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
import requests
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
Base = declarative_base()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
# Disable ipv6
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
def allowed_gai_family():
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
    """
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
     https://github.com/shazow/urllib3/blob/master/urllib3/util/connection.py
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
    """
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
    return socket.AF_INET
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
urllib3_cn.allowed_gai_family = allowed_gai_family
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
class RegisteredApp(Base):
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
    """RegisteredApp"""
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
    __tablename__ = "registered_app"
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
    __table_args__ = (UniqueConstraint("url", "name", name="_url_name_uc"),)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
    id = Column(Integer, primary_key=True)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
    url = Column(String)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
    name = Column(String)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
    key = Column(String, unique=True, index=True)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
    secret = Column(String)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
    def __repr__(self):
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
        return f"RegisteredApp(id={self.id!r}, url={self.url!r}, name={self.name!r})"
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
AppKeySecret = collections.namedtuple(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
    "AppKeySecret", ("key", "secret", "url"), defaults=(None, None, None)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
def get_application_secret(name, url, db_con):
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
    """Get the application id and secret
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
    Args:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
        name (String): The name of the application
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
        url (String): The mastodon base url (without http(s))
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
        db_con (Connection): The database connection
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
    Returns:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        AppKeySecret: The application secret
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
    """
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
    # get existing secret keys
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
    stmt = select(RegisteredApp).where(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
        RegisteredApp.url == url, RegisteredApp.name == name
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
    result = db_con.execute(stmt).fetchone()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
    if result:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
        return AppKeySecret(key=result.key, secret=result.secret, url=url)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
    (client_id, client_secret) = Mastodon.create_app(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
        name,
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        api_base_url = f'https://{url}'
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
    registered_app = RegisteredApp(name=name, url=url, key=client_id, secret=client_secret)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
    with Session(db_con) as session:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
        session.add(registered_app)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
        session.commit()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
    return AppKeySecret(key=client_id, secret=client_secret, url=url)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
def get_public_statuses(appKeySecret, tag, limit=40):
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
    mastodon = Mastodon(client_id=appKeySecret.key, client_secret=appKeySecret.secret, api_base_url=f'https://{appKeySecret.url}')
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
    results = mastodon.timeline_hashtag(hashtag=tag, limit=limit)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
    while results:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
        for status in results:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
            print("-------------------------")
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
            print(repr(status))
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
        results = mastodon.fetch_next(results)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
def parse_arg():
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
    """
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
    parse args
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
    """
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
    parser = argparse.ArgumentParser(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
        prog="dump_public_toots",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   105
        description="Dump the latest public toots",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
        epilog="dump only public toots",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
    parser.add_argument(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
        "-d",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        "--database",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
        dest="database",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   112
        default="registered_apps.sqlite",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   113
        help="The path to the registered apps db",
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   114
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   115
    parser.add_argument(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   116
        "-u", "--url", dest="url", help="The mastodon base url", required=True
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
    parser.add_argument(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   119
        "-n", "--name", dest="name", help="The application name", required=True
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   120
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
    parser.add_argument(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
        "-t", "--tag", dest="tag", help="The tag to search", required=True
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
    parser.add_argument(
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
        "-l", "--limit", dest="limit", help="page size", required=False, default=40
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   126
    )
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   127
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
    return parser.parse_args()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   131
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   132
if __name__ == "__main__":
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   133
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   134
    options = parse_arg()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   135
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   136
    # req = requests.get(f"https://{options.url}/api/v1/timelines/tag/{options.tag}/")
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
    # print(req.text)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
    app_db_eng = create_engine(f"sqlite+pysqlite:///{options.database}", future=True)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
    with app_db_eng.connect() as app_db_con:
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
        Base.metadata.create_all(app_db_con)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   144
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
        keySecret = get_application_secret(options.name, options.url, app_db_con)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
        get_public_statuses(keySecret, options.tag, options.limit)
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
        app_db_con.commit()
7c67caaafdeb enmi 22
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150