Force the use of https for the pypi index when installing the server dependencies.
from fabric.api import task, run, local, env, cd, put, prefix, sudo
from fabric.colors import green
from fabric.contrib.project import rsync_project
from fabric.contrib.files import exists, upload_template
from fabric.context_managers import settings
from mercurial import commands, ui, hg, cmdutil
import imp
import os, os.path
import shutil
import sys
import config
def get_export_path(version):
base_path = os.path.join(env.base_export_path,env.export_prefix).lstrip("/")
return os.path.expanduser(base_path) + "_%s" % (str(version))
def clean_export_folder(path):
print("Removing %s" % path)
if os.path.isdir(path):
shutil.rmtree(path, ignore_errors=True)
def do_export_version(path, version):
print("Export version %s"%str(version))
#hgui = ui.ui()
#repo = hg.repository(hgui, cmdutil.findrepo(os.getcwd()))
#commands.archive(hgui, repo, path, rev=str(version))
local("hg archive -r \'%s\' \"%s\"" % (str(version),path))
print("Export version %s done"%str(version))
def rsync_export(path, remotepath, filters):
print("Rsync %s to %s",(path,remotepath))
if filters:
filter_option_str = " ".join(["--filter \"%s\"" % (f) for f in filters])
else:
filter_option_str =""
run("mkdir -p \"%s\"" % remotepath)
rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)
print("Rsync %s to %s done",(path,remotepath))
def clean_rsync_folder(remotepath):
print("clean rsync folder %s" % remotepath)
run("rm -fr \"%s\"" % remotepath)
def build_src(path):
print("Build source dist at %s" % path)
f = None
try:
f, pathname, description = imp.find_module("setup", [path])
print(" 2 Build source dist at %s" % path)
setup_mod = imp.load_module("setup", f, pathname, description)
print(" 3 Build source dist at %s" % path)
finally:
if f:
f.close()
setup_mod.launch_setup("setup.py", ['sdist'])
print("Build source dist at %s done" % path)
def get_src_version(path):
print("get src version at %s" % path)
f = None
try:
f, pathname, description = imp.find_module("ldt", [path])
ldt_mod = imp.load_module("ldt", f, pathname, description)
finally:
if f:
f.close()
version = ldt_mod.VERSION
version_str = ldt_mod.get_version()
return (version, version_str)
def sync_build(path):
print("Sync build %s" % path)
with cd(env.remote_ldt_base_path):
filename = os.path.basename(path)
res_trans = put(path, os.path.join(env.remote_ldt_base_path, filename))
print("Sync build %s to %s" % (path,repr(res_trans)))
return res_trans
def remove_build(path):
print("remove build build %s" % path)
run("rm \"%s\"" % path)
def install_build(remotepath, remotevirtualenvpath):
print("Install build %s in %s" % (remotepath, remotevirtualenvpath))
activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
with prefix("source %s" % activate_path):
run("pip install \"%s\"" % remotepath)
def collectstatic(remotepath, remotevirtualenvpath):
print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))
activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
run("python manage.py collectstatic -c --noinput")
def create_config(export_path):
print("Create config from %s" % (export_path,))
remotepath = env.remote_web_path
remote_config_path = os.path.join(remotepath, env.platform_web_module, "config.py")
template_path = os.path.join(export_path, "web", env.platform_web_module, "config.py.tmpl")
context = {
'base_dir': os.path.join(remotepath, env.platform_web_module).rstrip("/")+"/",
'base_url': env.base_url,
'web_url': env.web_url,
'stream_url': env.stream_url,
'stream_src_prefix': env.stream_src_prefix,
'ffmpeg_path': env.ffmpeg_path,
'db_engine': env.db_engine,
'db_name': env.db_name,
'db_user': env.db_user,
'db_password': env.db_password,
'db_host': env.db_host,
'db_port': env.db_port,
'log_file': env.log_file,
'index_path': env.index_path,
'google_analytics_code': env.google_analytics_code,
'email_use_tls': env.email_use_tls,
'email_host': env.email_host,
'email_host_user': env.email_host_user,
'email_host_user': env.email_host_user,
'email_port': env.email_port,
}
if not exists(remote_config_path, verbose=True):
upload_template(template_path, remote_config_path, context=context)
def export_version(version):
print("export version %s" % str(version))
export_path = get_export_path(version)
clean_export_folder(export_path)
do_export_version(export_path,version)
return export_path
def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):
print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))
activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
if "remote_baseline_venv" in env and env.remote_baseline_venv:
prefix_str = "source \"%s\"" % os.path.join(env.remote_baseline_venv, "bin/activate")
else:
prefix_str = "echo"
run("mkdir -p \"%s\"" % remotevirtualenvpath)
with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):
run("python create_python_env.py")
run("python project-boot.py --unzip-setuptools --no-site-packages --clear --type-install=local \"%s\"" % remotevirtualenvpath)
with prefix("source \"%s\"" % activate_path):
run("pip install -i https://pypi.python.org/simple -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))
def do_sync_ldt(version, export_path):
print("do_sync_ldt with version %s and path %s" % (version,export_path))
src_path = export_path + "/src/ldt"
build_src(src_path)
(_,version_str) = get_src_version(src_path)
build_path = os.path.join(src_path,"dist","ldt-%s.tar.gz" % version_str)
sync_install_build(build_path)
def sync_install_build(build_path):
res_trans = None
try:
res_trans = sync_build(build_path)
install_build(res_trans[0], env.remote_virtualenv_path)
finally:
if res_trans:
remove_build(res_trans[0])
def do_sync_web(version, export_path):
print("do_sync_web with version %s and path %s" % (version,export_path))
web_path = os.path.join(export_path,"web/")
rsync_export(web_path, env.remote_web_path, env.web_rsync_filters)
def relaunch_server():
print("Relaunch server")
collectstatic(env.remote_web_path, env.remote_virtualenv_path)
sudo(env.web_relaunch_cmd, shell=False)
def check_folder_access():
print("Check folder access")
# get remote user
for folder_path in env.folders:
if not os.path.isabs(folder_path):
folder_path = env.remote_web_path.rstrip("/")+ "/" + folder_path
with settings(warn_only=True):
if not exists(folder_path):
run("mkdir -p \"%s\"" % folder_path)
run("chown -R -c :%s \"%s\"" % (env.web_group, folder_path))
run("chown -R -c g+w \"%s\"" % folder_path)
@task
def sync_web(version):
print(green("sync web with version %s" % version))
export_path = export_version(version)
do_sync_web(version, export_path)
check_folder_access()
create_config(export_path)
clean_export_folder(export_path)
relaunch_server()
@task
def sync_ldt(version):
print(green("sync ldt with version %s" % version))
export_path = export_version(version)
do_sync_ldt(version, export_path)
clean_export_folder(export_path)
relaunch_server()
@task
def update_lib(version, package):
print(green("update ldt with version %s" % version))
export_path = export_version(version)
lib_path = os.path.join(export_path, "virtualenv", "res", "lib")
f, pathname, description = imp.find_module("patch", [lib_path])
patch = imp.load_module("patch", f, pathname, description)
f, pathname, description = imp.find_module("lib_create_env", [lib_path])
lib_create_env = imp.load_module("lib_create_env", f, pathname, description)
package_path = os.path.join(export_path, "virtualenv", "res", "src", lib_create_env.URLS[package]['local'])
sync_install_build(package_path)
clean_export_folder(export_path)
relaunch_server()
@task
def sync_platform(version):
print(green("sync platform with version %s" % version))
export_path = export_version(version)
do_sync_ldt(version, export_path)
do_sync_web(version, export_path)
check_folder_access()
create_config(export_path)
clean_export_folder(export_path)
relaunch_server()
@task
def create_virtualenv(version):
print(green("create virtualenv with version %s" % version))
export_path = export_version(version)
venv_remote_export_path = ""
try:
virtualenv_path = os.path.join(export_path, "virtualenv")
venv_remote_export_path = os.path.join(env.remote_venv_export_path, env.export_prefix, version,"virtualenv")
rsync_export(virtualenv_path, venv_remote_export_path, env.venv_rsync_filters)
do_create_virtualenv(venv_remote_export_path, env.remote_virtualenv_path)
finally:
clean_export_folder(export_path)
if venv_remote_export_path:
clean_rsync_folder(venv_remote_export_path)