sbin/sync/fabfile.py
author ymh <ymh.work@gmail.com>
Fri, 05 Jul 2019 15:58:49 +0200
changeset 38 58f02a3ca6a8
parent 12 8895d41be7e2
permissions -rw-r--r--
Added tag 0.1.38 for changeset 7e614ca2431a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
12
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
import imp
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
import os.path
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
import io
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
# import config  # @UnusedImport
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
# from fablib import (export_version, do_sync_web, create_config,
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
#                     clean_export_folder, do_sync_comp, sync_install_build, do_create_virtualenv,
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
#                     clean_rsync_folder, rsync_export, get_src_version, sync_build,
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
#                     do_relaunch_server, install_build, do_create_virtualenv_requirement, build_src)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from core import (export_version, build_src, get_src_version, sync_build,
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
                  do_create_virtualenv_requirement, get_src_dependencies,
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
                  do_relaunch_server, clean_export_folder)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
# from fabric import task, env, run, cd, put
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    14
from fabric import Connection
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
from invoke import task
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
from blessings import Terminal
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
# from fabric.colors import green
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
# env.use_ssh_config = True
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
t = Terminal()
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
def build_source(c, key, version):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
    print(t.green("build source with version %s" % version))
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    export_path = export_version(c, **{ key: version })
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
    export_path_full = os.path.join(export_path, key, c.env.repos[key]['src_root'])
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    build_src(c, export_path_full)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
    (_,version_str) = get_src_version(c, key, export_path_full)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
    src_dep = get_src_dependencies(c, key, export_path_full)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
    return (os.path.join(export_path_full,"dist","%s-%s.tar.gz" % (key,version_str)), src_dep)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
def do_create_virtualenv(c, remote_build_path, dep_remote_build_path_list):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
    env = c.env
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
    requirements_path = os.path.join(remote_build_path, env['repos'][env.key]['requirements'])
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
    remotevirtualenvpath = env['remote_path']['virtualenv']
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
    do_create_virtualenv_requirement(c, requirements_path, remotevirtualenvpath, env['repos'][env.key]['python_version'])
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
    # add setting path to virtualenv
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
    ext_path = "import sys; sys.__plen = len(sys.path)\n"
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
    for l in env['remote_path'].get('pythonpath', []):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
        ext_path += l + "\n"
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
    ext_path += "import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)"
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
    with Connection(env['hosts'][0]) as rconnection:
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
        rconnection.put(io.StringIO(ext_path), os.path.join(env['remote_path']['virtualenv'], 'lib/python%s/site-packages/_virtualenv_path_extensions.pth'%env['repos'][env.key]['python_version']))
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
        for dep_remote_build_path in dep_remote_build_path_list:
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
            with rconnection.prefix("echo $SHELL && . \"%s\"" % os.path.join(remotevirtualenvpath, "bin/activate")):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
                rconnection.run("pip install \"%s\"" % dep_remote_build_path)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
        with rconnection.prefix("echo $SHELL && . \"%s\"" % os.path.join(remotevirtualenvpath, "bin/activate")):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
            rconnection.run("pip install \"%s.tar.gz\"" % remote_build_path)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
@task
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
def relaunch_server(c, collectstatic=True, migrate=True):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
    print("Relaunch server")
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
    do_relaunch_server(c, collectstatic, migrate)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
@task
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
def create_virtualenv(c, version):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
    print(t.green("create virtualenv for version ") + version)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
    build_path, source_dep_list = build_source(c, c.env.key, version)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
    print(t.green("BUILD PATH: ") + build_path + " - %r" % source_dep_list)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
    source_dep_build_path_list = []
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
    print("Build dependencies : %r" %  source_dep_list)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
    for source_dep_key, source_dep_version in source_dep_list:
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
        source_dep_build_path, _ = build_source(c, source_dep_key, source_dep_version)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
        source_dep_build_path_list.append(source_dep_build_path)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
    host_connection = Connection(c.env['hosts'][0])
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
    host_connection.run('mkdir -p "%s"' % c.env['remote_path']['build_export'])
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
    res_trans = sync_build(c, build_path)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
    res_trans_dep = [ sync_build(c, source_dep_build_path).remote for source_dep_build_path in source_dep_build_path_list]
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
    # untar build
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
    print("Untar %s on remote host"%res_trans.remote)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
    with host_connection.cd(c.env['remote_path']['build_export']):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        host_connection.run('tar zxf %s' % res_trans.remote)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
    do_create_virtualenv(c, res_trans.remote[0:-7], res_trans_dep)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
    host_connection.run('rm -fr "%s/*"' % (c.env['remote_path']['build_export']))
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
    clean_export_folder(c.env.remote_path['build_export'])
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
@task
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
def publish_version(c, version):
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
    create_virtualenv(c, version)
8895d41be7e2 upgrade distribution script and define version
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
    relaunch_server(c, True, True)