diff -r 87104b7cb3d6 -r 5778de052a1b sbin/sync/fabfile.py --- a/sbin/sync/fabfile.py Wed Nov 28 10:05:53 2012 +0100 +++ b/sbin/sync/fabfile.py Thu Nov 29 15:57:49 2012 +0100 @@ -1,4 +1,4 @@ -from fabric.api import task, run, local, env, cd, put, prefix, sudo +from fabric.api import task, run, local, env, cd, put, prefix, sudo, lcd from fabric.colors import green from fabric.contrib.project import rsync_project from fabric.contrib.files import exists, upload_template @@ -8,6 +8,7 @@ import os, os.path import shutil import sys +import urlparse import config @@ -20,10 +21,25 @@ if os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) -def do_export_version(path, version): - print("Export version %s"%str(version)) +def do_export_version(path, version, *export_keys): + print("Export version %s with keys %s" % (str(version), repr(export_keys))) + + for export_key in export_keys: + export_path = os.path.join(path,export_key) - local("hg archive -r \'%s\' \"%s\"" % (str(version),path)) + repo_url = env.repos[export_key] + url_part = urlparse.urlparse(repo_url) + if url_part.scheme or url_part.netloc: + # this is a remote repo. Let's clone first + clone_path = os.path.join(path,'clone',export_keys) + os.makedirs(clone_path) + local("hg clone \"%s\" \"%s\"" % (repo_url,clone_path)) + else: + clone_path = repo_url + + with lcd(clone_path): + local("hg archive -r \'%s\' \"%s\"" % (str(version),export_path)) + print("Export version %s done"%str(version)) @@ -93,9 +109,9 @@ def sync_build(path): print("Sync build %s" % path) - with cd(env.remote_ldt_base_path): + with cd(env.remote_path['ldt_base']): filename = os.path.basename(path) - res_trans = put(path, os.path.join(env.remote_ldt_base_path, filename)) + res_trans = put(path, os.path.join(env.remote_path['ldt_base'], filename)) print("Sync build %s to %s" % (path,repr(res_trans))) return res_trans @@ -122,7 +138,7 @@ def create_config(export_path): print("Create config from %s" % (export_path,)) - remotepath = env.remote_web_path + remotepath = env.remote_path['web'] remote_config_path = os.path.join(remotepath, env.platform_web_module, "config.py") template_path = os.path.join(export_path, "web", env.platform_web_module, "config.py.tmpl") @@ -152,13 +168,14 @@ if not exists(remote_config_path, verbose=True): upload_template(template_path, remote_config_path, context=context) -def export_version(version): - print("export version %s" % str(version)) +def export_version(version, *args): + print("export version %s for %s" % (str(version), args)) export_path = get_export_path(version) clean_export_folder(export_path) - do_export_version(export_path,version) + + do_export_version(export_path,version, *args) return export_path @@ -180,6 +197,8 @@ def do_sync_ldt(version, export_path): print("do_sync_ldt with version %s and path %s" % (version,export_path)) + + ## src_path = export_path + "/src/ldt" build_src(src_path) (_,version_str) = get_src_version(src_path) @@ -191,7 +210,7 @@ res_trans = None try: res_trans = sync_build(build_path) - install_build(res_trans[0], env.remote_virtualenv_path) + install_build(res_trans[0], env.remote_path['virtualenv']) finally: if res_trans: remove_build(res_trans[0]) @@ -199,15 +218,20 @@ def do_sync_web(version, export_path): print("do_sync_web with version %s and path %s" % (version,export_path)) + #sync web web_path = os.path.join(export_path,"web/") - rsync_export(web_path, env.remote_web_path, env.web_rsync_filters) + rsync_export(web_path, env.remote_path['web'], env.rsync_filters['web']) + #sync src + src_path = os.path.join(export_path,"src/") + rsync_export(src_path, env.remote_path['src'], env.rsync_filters['web']) + def check_folder_access(): print("Check folder access") # get remote user for folder_path in env.folders: if not os.path.isabs(folder_path): - folder_path = env.remote_web_path.rstrip("/")+ "/" + folder_path + folder_path = env.remote_path['web'].rstrip("/")+ "/" + folder_path with settings(warn_only=True): if not exists(folder_path): run("mkdir -p \"%s\"" % folder_path) @@ -218,65 +242,73 @@ print("Relaunch server") check_folder_access() if do_collectstatic: - collectstatic(env.remote_web_path, env.remote_virtualenv_path, env.platform_web_module) + collectstatic(env.remote_path['web'], env.remote_path['virtualenv'], env.platform_web_module) sudo(env.web_relaunch_cmd, shell=False) @task def sync_web(version): print(green("sync web with version %s" % version)) - export_path = export_version(version) - do_sync_web(version, export_path) - create_config(export_path) + export_path = export_version(version, 'web') + export_path_full = os.path.join(export_path,'web') + do_sync_web(version, export_path_full) + create_config(export_path_full) clean_export_folder(export_path) relaunch_server() @task def sync_ldt(version): print(green("sync ldt with version %s" % version)) - export_path = export_version(version) - do_sync_ldt(version, export_path) + export_path = export_version(version, 'ldt') + export_path_full = os.path.join(export_path,'ldt') + do_sync_ldt(version, export_path_full) clean_export_folder(export_path) relaunch_server() @task def update_lib(version, package): print(green("update ldt with version %s" % version)) - export_path = export_version(version) - lib_path = os.path.join(export_path, "virtualenv", "res", "lib") + export_path = export_version(version,'web') + export_path_full = os.path.join(export_path,'web') + lib_path = os.path.join(export_path_full, "virtualenv", "res", "lib") f, pathname, description = imp.find_module("patch", [lib_path]) patch = imp.load_module("patch", f, pathname, description) f, pathname, description = imp.find_module("lib_create_env", [lib_path]) lib_create_env = imp.load_module("lib_create_env", f, pathname, description) - package_path = os.path.join(export_path, "virtualenv", "res", "src", lib_create_env.URLS[package]['local']) + package_path = os.path.join(export_path_full, "virtualenv", "res", "src", lib_create_env.URLS[package]['local']) - sync_install_build(package_path) + sync_install_build(package_path_full) clean_export_folder(export_path) relaunch_server() @task def sync_platform(version): print(green("sync platform with version %s" % version)) - export_path = export_version(version) - do_sync_ldt(version, export_path) - do_sync_web(version, export_path) - create_config(export_path) + export_path = export_version(version, 'ldt', 'web') + export_path_ldt = os.path.join(export_path,'ldt') + export_path_web = os.path.join(export_path,'web') + do_sync_ldt(version, export_path_ldt) + do_sync_web(version, export_path_web) + create_config(export_path_web) clean_export_folder(export_path) relaunch_server() @task def create_virtualenv(version): print(green("create virtualenv with version %s" % version)) - export_path = export_version(version) + export_path = export_version(version, 'web') + export_path_web = os.path.join(export_path,'web') venv_remote_export_path = "" try: - virtualenv_path = os.path.join(export_path, "virtualenv") + virtualenv_path = os.path.join(export_path_web, "virtualenv") - venv_remote_export_path = os.path.join(env.remote_venv_export_path, env.export_prefix, version,"virtualenv") - rsync_export(virtualenv_path, venv_remote_export_path, env.venv_rsync_filters) - do_create_virtualenv(venv_remote_export_path, env.remote_virtualenv_path) + venv_remote_export_path = os.path.join(env.remote_path['venv_export'], env.export_prefix, version,"virtualenv") + rsync_export(virtualenv_path, venv_remote_export_path, env.rsync_filters['venv']) + do_create_virtualenv(venv_remote_export_path, env.remote_path['virtualenv']) finally: clean_export_folder(export_path) if venv_remote_export_path: clean_rsync_folder(venv_remote_export_path) + +