sbin/sync/fabfile.py
author ymh <ymh.work@gmail.com>
Thu, 01 Sep 2016 14:08:20 +0200
changeset 166 e9cd38b7cfe0
parent 162 f014c1593ade
child 171 9b019ac3b791
permissions -rw-r--r--
improve install script + correct base settings
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
import imp
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
import os.path
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
import StringIO
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
import config  # @UnusedImport
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
from fablib import (export_version, do_sync_web, create_config,
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
                    clean_export_folder, do_sync_comp, sync_install_build, do_create_virtualenv,
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
                    clean_rsync_folder, rsync_export, get_src_version, sync_build,
162
f014c1593ade smal modification to get version
ymh <ymh.work@gmail.com>
parents: 159
diff changeset
     9
                    do_relaunch_server, install_build, do_create_virtualenv_requirement, build_src)
166
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    10
from fabric.api import task, env, run, cd, put
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from fabric.colors import green
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    12
166
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    13
env.use_ssh_config = True
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    14
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
def build_source(version):
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
    print(green("build source with version %s" % version))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
    export_path = export_version(iconolab=version)
162
f014c1593ade smal modification to get version
ymh <ymh.work@gmail.com>
parents: 159
diff changeset
    18
    export_path_full = os.path.join(export_path, env.key, env.repos[env.key]['src_root'])
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
    build_src(export_path_full)
162
f014c1593ade smal modification to get version
ymh <ymh.work@gmail.com>
parents: 159
diff changeset
    20
    (_,version_str) = get_src_version(env.key, export_path_full)
166
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    21
    return os.path.join(export_path_full,"dist","%s-%s.tar.gz" % (env.key,version_str))
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
@task
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
def relaunch_server(do_collectstatic=True, do_migrate=True, do_check_folder_access=False):
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    print("Relaunch server")
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
    do_relaunch_server(do_collectstatic, do_migrate, env.get('check_folder_access',do_check_folder_access))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
@task
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
def sync_site(version):
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
    print(green("sync site with version %s" % version))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
    build_path = build_source(version)
166
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    32
    run('mkdir -p "%s"' % env.remote_path['build_export'])
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
    res_trans = sync_build(build_path)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
    # untar build
166
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    35
    with cd(env.remote_path['build_export']):
e9cd38b7cfe0 improve install script + correct base settings
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
    36
        run('tar zxf %s' % res_trans[0])
159
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
    # rebuild virtualenv
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
    requirements_path = os.path.join(res_trans[0][0:-7], env.repos[env.key]['requirements'])
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
    do_create_virtualenv_requirement(requirements_path, env.remote_path['virtualenv'], env.repos[env.key]['python_version'])
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
    # add setting path to virtualenv
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
    ext_path = "import sys; sys.__plen = len(sys.path)\n"
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
    for l in env.remote_path.get('pythonpath', []):
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
        ext_path += l + "\n"
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
    ext_path += "import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)"
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
    put(StringIO.StringIO(ext_path), os.path.join(env.remote_path['virtualenv'], 'lib/python%s/site-packages/_virtualenv_path_extensions.pth'%env.repos[env.key]['python_version']))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
    # install build
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
    install_build(res_trans[0], env.remote_path['virtualenv'])
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
    # remove build + untared folder
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
    run('rm -fr "%s" "%s" ' % (res_trans[0], res_trans[0][0:-7]))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
    clean_export_folder(export_path)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
    relaunch_server()
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
@task
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
def create_virtualenv(version):
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
    print(green("create virtualenv with version %s" % version))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
    export_path = export_version(web=version)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
    export_path_web = os.path.join(export_path,'web')
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
    venv_remote_export_path = ""
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
    try:
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
        virtualenv_path = os.path.join(export_path_web, "virtualenv")
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
        venv_remote_export_path = os.path.join(env.remote_path['venv_export'], env.export_prefix, version,"virtualenv")
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
        rsync_export(virtualenv_path, venv_remote_export_path, env.rsync_filters['venv'])
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
        do_create_virtualenv(venv_remote_export_path, env.remote_path['virtualenv'])
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
    finally:
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
        clean_export_folder(export_path)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
        if venv_remote_export_path:
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
            clean_rsync_folder(venv_remote_export_path)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    70
if len(env.repos) > 1:
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
    @task
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
    def sync_platform(version):
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
        print(green("sync platform with version web %s" % (version)))
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
        export_path = export_version(web=version)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
        export_path_web = os.path.join(export_path,'web')
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
        do_sync_web(version, export_path_web)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        comp_versions = get_comp_versions_dict(export_path_web)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
        for key in [k for k in env.repos if k != 'web']:
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
            export_path_key = export_version(**{key: comp_versions[key]})
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
            export_path_comp = os.path.join(export_path_key, key)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
            do_sync_comp(key, export_path_comp)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
            clean_export_folder(export_path_key)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
        create_config(export_path_web)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
        clean_export_folder(export_path)
a2e6d314da2f add sync scripts
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
        relaunch_server()