sbin/sync/fabfile.py
author ymh <ymh.work@gmail.com>
Wed, 28 Nov 2012 02:12:51 +0100
changeset 95 c63f2508c34d
parent 82 fa141523769a
permissions -rwxr-xr-x
improve on kc rel form to avoid having all the contents in the drop down list, only those available.

from fabric.api import task, run, local, env, cd, put, prefix, sudo
from fabric.colors import green
from fabric.contrib.project import rsync_project
from fabric.contrib.files import exists, upload_template
from fabric.context_managers import settings 
from mercurial import commands, ui, hg, cmdutil
import imp
import os, os.path
import shutil
import sys

import config

def get_export_path(version):
    base_path = os.path.join(env.base_export_path,env.export_prefix).rstrip("/")
    return os.path.expanduser(base_path) + "_%s" % (str(version))

def clean_export_folder(path):
    print("Removing %s" % path)
    if os.path.isdir(path):
        shutil.rmtree(path, ignore_errors=True)

def do_export_version(path, version):
    print("Export version %s"%str(version))
        
    local("hg archive -r \'%s\' \"%s\"" % (str(version),path))
    print("Export version %s done"%str(version))

    
def get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, settings_key):
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    res = ""
    with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
        tempfilepath = run("mktemp -t ldtplatform.XXXXXX")
        with settings(warn_only=True):
            run("echo \"import os\" > %s" % (tempfilepath))
            map(lambda str: run("echo \"%s\" >> %s" % (str, tempfilepath)),
                ["os.environ.setdefault('DJANGO_SETTINGS_MODULE', '%s.settings')" % (platform_web_module),
                 "from django.conf import settings",
                 "print settings.%s" % (settings_key)])
            res = run("python < %s" % (tempfilepath))
        run("rm -f \"%s\"" % (tempfilepath))
    return res

def rsync_export(path, remotepath, filters):
    print("Rsync %s to %s",(path,remotepath))
    
    filter_option_str = "--progress --stats"
    if filters:
        filter_option_str += " " + " ".join(["--filter \"%s\"" % (f) for f in filters])
    
    run("mkdir -p \"%s\"" % remotepath)
    rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)
    print("Rsync %s to %s done",(path,remotepath))
    
def clean_rsync_folder(remotepath):
    print("clean rsync folder %s" % remotepath)
    run("rm -fr \"%s\"" % remotepath)
    

def collectstatic(remotepath, remotevirtualenvpath, platform_web_module):
    print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))
    remotestaticsitepath = get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, "STATIC_ROOT")
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
        #remocve old files optio -c of collect static fail !
        run("rm -fr \"%s\"" % (remotestaticsitepath))
        run("python manage.py collectstatic --noinput")

def migrate(remotepath, remotevirtualenvpath, platform_web_module):
    print("migrate in %s with %s" % (remotepath, remotevirtualenvpath))
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
        #remocve old files optio -c of collect static fail !
        run("python manage.py syncdb --migrate --noinput")
        
def create_config(export_path, key):    
    print("Create config from %s with key %s" % (export_path,key))
    remotepath = env.remote_web_path[key]
    remote_config_path = os.path.join(remotepath, env.web_module[key], "config.py")
    template_path = os.path.join(export_path, env.local_folders[key], env.web_module[key], "config.py.tmpl")
    
    context = env.config[key]
    context['base_dir'] = os.path.join(remotepath, env.web_module[key]).rstrip("/")+"/"
    
    if not exists(remote_config_path, verbose=True):
        upload_template(template_path, remote_config_path, context=context)

def export_version(version):
    print("export version %s" % str(version))
    
    export_path = get_export_path(version)
    
    clean_export_folder(export_path)
    do_export_version(export_path,version)
    
    return export_path

def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):
    print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    if "remote_baseline_venv" in env and env.remote_baseline_venv:
        prefix_str = "source \"%s\"" % os.path.join(env.remote_baseline_venv, "bin/activate")
    else:
        prefix_str = "echo"
    with settings(warn_only=True):
        run("rm -fr \"%s\"" % remotevirtualenvpath)
    run("mkdir -p \"%s\"" % remotevirtualenvpath)
    with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):
        run("python create_python_env.py")
        run("python project-boot.py \"%s\"" % remotevirtualenvpath)
    with prefix("source \"%s\"" % activate_path):
        run("pip install -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))
        

def do_sync_web(version, export_path, key):
    print("do_sync_web with version %s and path %s and key %s" % (version,export_path, key))
    web_path = os.path.join(export_path,env.local_folders[key].rstrip("/")+"/") 
    rsync_export(web_path, env.remote_web_path[key], env.web_rsync_filters[key])

def remove_lib(path):
    print("remove build build %s" % path)
    run("rm \"%s\"" % path)

def install_lib(remotepath, remotevirtualenvpath):
    print("Install lib %s in %s" % (remotepath, remotevirtualenvpath))
    activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
    
    with prefix("source %s" % activate_path):
        run("pip install \"%s\"" % remotepath)

def sync_lib(path):
    print("Sync build %s" % path)
    run("mkdir -p \"%s\"" % env.remote_venv_export_path)
    with cd(env.remote_venv_export_path):
        filename = os.path.basename(path)
        dest_path = os.path.join(env.remote_venv_export_path, filename)
        res_trans = put(path, dest_path)
        print("Sync build %s to %s" % (path,repr(res_trans)))
        return res_trans

def sync_install_lib(lib_path):
    res_trans = None
    try:
        res_trans = sync_lib(lib_path)
        install_lib(res_trans[0], env.remote_virtualenv_path)        
    finally:
        if res_trans:
            remove_lib(res_trans[0])

    
def check_folder_access(key):
    print("Check folder access")
    # get remote user
    for folder_path in env.folders[key]:
        if not os.path.isabs(folder_path):
            folder_path = env.remote_web_path[key].rstrip("/")+ "/" + folder_path
            with settings(warn_only=True):
                if not exists(folder_path):
                    run("mkdir -p \"%s\"" % folder_path)
                run("chown -R -c :%s \"%s\"" % (env.web_group, folder_path))
                run("chmod -R -c g+w \"%s\"" % folder_path)
@task
def relaunch_server(key,do_collectstatic=True, do_migrate=True):
    print("Relaunch server")
    check_folder_access(key)
    if do_collectstatic:
        collectstatic(env.remote_web_path[key], env.remote_virtualenv_path, env.web_module[key])
    if do_migrate:
        migrate(env.remote_web_path[key], env.remote_virtualenv_path, env.web_module[key])
    sudo(env.relaunch_cmd[key], shell=False)


@task
def sync_web(version, key):
    print(green("sync web with version %s" % version))
    export_path = export_version(version)
    do_sync_web(version, export_path, key)
    create_config(export_path, key)
    clean_export_folder(export_path)
    relaunch_server(key)
    
@task
def update_lib(version, package):
    print(green("update ldt with version %s" % version))
    export_path = export_version(version)
    lib_path = os.path.join(export_path, env.local_folders.get("virtualenv","virtualenv"), "res", "lib")
    
    f, pathname, description = imp.find_module("patch", [lib_path])
    patch = imp.load_module("patch", f, pathname, description)
    f, pathname, description = imp.find_module("lib_create_env", [lib_path])
    lib_create_env = imp.load_module("lib_create_env", f, pathname, description)
    
    package_path = os.path.join(export_path, env.local_folders.get("virtualenv","virtualenv"), "res", "src", lib_create_env.URLS[package]['local'])
    
    sync_install_lib(package_path)
    clean_export_folder(export_path)
    relaunch_server('ldt')
    relaunch_server('hp')


@task
def create_virtualenv(version):
    print(green("create virtualenv with version %s" % version))
    export_path = export_version(version)
    venv_remote_export_path = ""
    try:
        virtualenv_path = os.path.join(export_path, env.local_folders.get("virtualenv","virtualenv"))
    
        venv_remote_export_path = os.path.join(env.remote_venv_export_path, env.export_prefix, version,"virtualenv")
        rsync_export(virtualenv_path, venv_remote_export_path, env.venv_rsync_filters)
        do_create_virtualenv(venv_remote_export_path, env.remote_virtualenv_path)
    finally:
        clean_export_folder(export_path)
        if venv_remote_export_path:
            clean_rsync_folder(venv_remote_export_path)


#sync hp
# sync files
# create virtualenv
# create config
# syncdb + migrate
# collectstatic
# relaunch web

#sync ldt
# sync files
# create virtualenv
# create config
# syncdb + migrate
# collectstatic
# relaunch web