sbin/sync/fabfile.py
author ymh <ymh.work@gmail.com>
Tue, 21 Jan 2020 22:11:40 +0100
changeset 96 900b29a370ae
parent 86 bb39eafd5293
permissions -rw-r--r--
Force the use of https for the pypi index when installing the server dependencies.

from fabric.api import task, run, local, env, cd, put, prefix, sudo
from fabric.colors import green
from fabric.contrib.project import rsync_project
from fabric.contrib.files import exists, upload_template
from fabric.context_managers import settings
from mercurial import commands, ui, hg, cmdutil
import imp
import os, os.path
import shutil
import sys

import config

def get_export_path(version):
    base_path = os.path.join(env.base_export_path,env.export_prefix).lstrip("/")
    return os.path.expanduser(base_path) + "_%s" % (str(version))

def clean_export_folder(path):
    print("Removing %s" % path)
    if os.path.isdir(path):
        shutil.rmtree(path, ignore_errors=True)

def do_export_version(path, version):
    print("Export version %s"%str(version))

    #hgui = ui.ui()
    #repo = hg.repository(hgui, cmdutil.findrepo(os.getcwd()))
    #commands.archive(hgui, repo, path, rev=str(version))

    local("hg archive -r \'%s\' \"%s\"" % (str(version),path))
    print("Export version %s done"%str(version))

def rsync_export(path, remotepath, filters):
    print("Rsync %s to %s",(path,remotepath))

    if filters:
        filter_option_str = " ".join(["--filter \"%s\"" % (f) for f in filters])
    else:
        filter_option_str =""

    run("mkdir -p \"%s\"" % remotepath)
    rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)
    print("Rsync %s to %s done",(path,remotepath))

def clean_rsync_folder(remotepath):
    print("clean rsync folder %s" % remotepath)
    run("rm -fr \"%s\"" % remotepath)

def build_src(path):
    print("Build source dist at %s" % path)
    f = None
    try:
        f, pathname, description = imp.find_module("setup", [path])
        print(" 2 Build source dist at %s" % path)
        setup_mod = imp.load_module("setup", f, pathname, description)
        print(" 3 Build source dist at %s" % path)
    finally:
        if f:
            f.close()

    setup_mod.launch_setup("setup.py", ['sdist'])

    print("Build source dist at %s done" % path)


def get_src_version(path):
    print("get src version at %s" % path)
    f = None
    try:
        f, pathname, description = imp.find_module("ldt", [path])
        ldt_mod = imp.load_module("ldt", f, pathname, description)
    finally:
        if f:
            f.close()
    version = ldt_mod.VERSION
    version_str = ldt_mod.get_version()

    return (version, version_str)


def sync_build(path):
    print("Sync build %s" % path)
    with cd(env.remote_ldt_base_path):
        filename = os.path.basename(path)
        res_trans = put(path, os.path.join(env.remote_ldt_base_path, filename))
        print("Sync build %s to %s" % (path,repr(res_trans)))
        return res_trans

def remove_build(path):
    print("remove build build %s" % path)
    run("rm \"%s\"" % path)


def install_build(remotepath, remotevirtualenvpath):
    print("Install build %s in %s" % (remotepath, remotevirtualenvpath))
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

    with prefix("source %s" % activate_path):
        run("pip install \"%s\"" % remotepath)

def collectstatic(remotepath, remotevirtualenvpath):
    print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
        run("python manage.py collectstatic -c --noinput")

def create_config(export_path):
    print("Create config from %s" % (export_path,))
    remotepath = env.remote_web_path
    remote_config_path = os.path.join(remotepath, env.platform_web_module, "config.py")
    template_path = os.path.join(export_path, "web", env.platform_web_module, "config.py.tmpl")

    context = {
     'base_dir': os.path.join(remotepath, env.platform_web_module).rstrip("/")+"/",
     'base_url': env.base_url,
     'web_url': env.web_url,
     'stream_url': env.stream_url,
     'stream_src_prefix': env.stream_src_prefix,
     'ffmpeg_path': env.ffmpeg_path,
     'db_engine': env.db_engine,
     'db_name': env.db_name,
     'db_user': env.db_user,
     'db_password': env.db_password,
     'db_host': env.db_host,
     'db_port': env.db_port,
     'log_file': env.log_file,
     'index_path': env.index_path,
     'google_analytics_code': env.google_analytics_code,
     'email_use_tls': env.email_use_tls,
     'email_host': env.email_host,
     'email_host_user': env.email_host_user,
     'email_host_user': env.email_host_user,
     'email_port': env.email_port,
    }

    if not exists(remote_config_path, verbose=True):
        upload_template(template_path, remote_config_path, context=context)

def export_version(version):
    print("export version %s" % str(version))

    export_path = get_export_path(version)

    clean_export_folder(export_path)
    do_export_version(export_path,version)

    return export_path

def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):
    print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    if "remote_baseline_venv" in env and env.remote_baseline_venv:
        prefix_str = "source \"%s\"" % os.path.join(env.remote_baseline_venv, "bin/activate")
    else:
        prefix_str = "echo"
    run("mkdir -p \"%s\"" % remotevirtualenvpath)
    with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):
        run("python create_python_env.py")
        run("python project-boot.py --unzip-setuptools --no-site-packages --clear --type-install=local \"%s\"" % remotevirtualenvpath)
    with prefix("source \"%s\"" % activate_path):
        run("pip install -i https://pypi.python.org/simple -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))

def do_sync_ldt(version, export_path):
    print("do_sync_ldt with version %s and path %s" % (version,export_path))
    src_path = export_path + "/src/ldt"
    build_src(src_path)
    (_,version_str) = get_src_version(src_path)
    build_path = os.path.join(src_path,"dist","ldt-%s.tar.gz" % version_str)
    sync_install_build(build_path)


def sync_install_build(build_path):
    res_trans = None
    try:
        res_trans = sync_build(build_path)
        install_build(res_trans[0], env.remote_virtualenv_path)
    finally:
        if res_trans:
            remove_build(res_trans[0])


def do_sync_web(version, export_path):
    print("do_sync_web with version %s and path %s" % (version,export_path))
    web_path = os.path.join(export_path,"web/")
    rsync_export(web_path, env.remote_web_path, env.web_rsync_filters)

def relaunch_server():
    print("Relaunch server")
    collectstatic(env.remote_web_path, env.remote_virtualenv_path)
    sudo(env.web_relaunch_cmd, shell=False)

def check_folder_access():
    print("Check folder access")
    # get remote user
    for folder_path in env.folders:
        if not os.path.isabs(folder_path):
            folder_path = env.remote_web_path.rstrip("/")+ "/" + folder_path
            with settings(warn_only=True):
                if not exists(folder_path):
                    run("mkdir -p \"%s\"" % folder_path)
                run("chown -R -c :%s \"%s\"" % (env.web_group, folder_path))
                run("chown -R -c g+w \"%s\"" % folder_path)

@task
def sync_web(version):
    print(green("sync web with version %s" % version))
    export_path = export_version(version)
    do_sync_web(version, export_path)
    check_folder_access()
    create_config(export_path)
    clean_export_folder(export_path)
    relaunch_server()

@task
def sync_ldt(version):
    print(green("sync ldt with version %s" % version))
    export_path = export_version(version)
    do_sync_ldt(version, export_path)
    clean_export_folder(export_path)
    relaunch_server()

@task
def update_lib(version, package):
    print(green("update ldt with version %s" % version))
    export_path = export_version(version)
    lib_path = os.path.join(export_path, "virtualenv", "res", "lib")

    f, pathname, description = imp.find_module("patch", [lib_path])
    patch = imp.load_module("patch", f, pathname, description)
    f, pathname, description = imp.find_module("lib_create_env", [lib_path])
    lib_create_env = imp.load_module("lib_create_env", f, pathname, description)

    package_path = os.path.join(export_path, "virtualenv", "res", "src", lib_create_env.URLS[package]['local'])

    sync_install_build(package_path)
    clean_export_folder(export_path)
    relaunch_server()

@task
def sync_platform(version):
    print(green("sync platform with version %s" % version))
    export_path = export_version(version)
    do_sync_ldt(version, export_path)
    do_sync_web(version, export_path)
    check_folder_access()
    create_config(export_path)
    clean_export_folder(export_path)
    relaunch_server()

@task
def create_virtualenv(version):
    print(green("create virtualenv with version %s" % version))
    export_path = export_version(version)
    venv_remote_export_path = ""
    try:
        virtualenv_path = os.path.join(export_path, "virtualenv")

        venv_remote_export_path = os.path.join(env.remote_venv_export_path, env.export_prefix, version,"virtualenv")
        rsync_export(virtualenv_path, venv_remote_export_path, env.venv_rsync_filters)
        do_create_virtualenv(venv_remote_export_path, env.remote_virtualenv_path)
    finally:
        clean_export_folder(export_path)
        if venv_remote_export_path:
            clean_rsync_folder(venv_remote_export_path)