# HG changeset patch # User ymh # Date 1472657848 -7200 # Node ID a2e6d314da2fa6fb33d1a0ef042eb908dac2060f # Parent f9d0d6a840cfecbba60ece5c5d54f573deef1b56 add sync scripts diff -r f9d0d6a840cf -r a2e6d314da2f .hgignore --- a/.hgignore Wed Aug 31 17:20:07 2016 +0200 +++ b/.hgignore Wed Aug 31 17:37:28 2016 +0200 @@ -29,3 +29,5 @@ ^log$ ^sbin/sync/config\.py$ ^src/MANIFEST +^src/iconolab.egg-info +^src/dist/ diff -r f9d0d6a840cf -r a2e6d314da2f sbin/sync/config.py.tmpl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sbin/sync/config.py.tmpl Wed Aug 31 17:37:28 2016 +0200 @@ -0,0 +1,66 @@ +from fabric.api import env +from random import choice + +env.hosts = ['iri@web.iri.centrepompidou.fr'] + +env.web_group = 'www-data' +env.folders = ['../run/log', 'static/media'] + +env.repos = {'iconolab' : {'repo':"/Users/ymh/dev/projects/iconolab", 'src_root':'src', 'requirements': 'src/requirements/prod.txt'}, 'python_version': '3.5'} +env.base_export_path = "/tmp" +env.export_prefix = "iconolab" +env.key = 'iconolab' + +env.remote_path = { + 'web':"/iridata/www/ldt/", + 'src':"/Users/ymh/dev/tmp/testfab/src", + 'virtualenv':"/iridata/virtualenv/ldt", + 'build_export':"/tmp/build", + 'pythonpath' : ['/etc/www/iconolab'] +} + +env.platform_web_module = "iconolab" +env.remote_baseline_venv = "" + +env.rsync_filters = { + 'src' : [ + "P .htpasswd", + "P .htaccess", + "P egonomy/config.py", + ], + 'web': [ + "+ core", + "P .htpasswd", + "P .htaccess", + "P robots.txt", + "P env/***", + "P log/***", + "P index/***", + "P static/media/***", + "P crossdomain.xml", + ], + 'venv': [ + "+ core", + ] +} +env.web_relaunch_cmd = "supervisorctl restart iconolab" + + +env.config = { + 'web': { + 'base_url': "/", + 'web_url': 'http://egonomy.iri-resesarch.org', + 'db_engine':'postgresql_psycopg2', + 'db_name':'platform', + 'db_user': 'iriuser', + 'db_password': '', + 'db_host': 'sql.iri.centrepompidou.fr', + 'db_port': 5432, + 'haystack_url' : 'http://localhost:9200', + 'haystack_index' : 'egonomy', + 'log_file' : env.remote_path['web'] + '/log/log.txt', + 'secret_key' : ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)]), + }, +} + +env.check_folder_access = True diff -r f9d0d6a840cf -r a2e6d314da2f sbin/sync/fabfile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sbin/sync/fabfile.py Wed Aug 31 17:37:28 2016 +0200 @@ -0,0 +1,85 @@ +import imp +import os.path +import StringIO + +import config # @UnusedImport +from fablib import (export_version, do_sync_web, create_config, + clean_export_folder, do_sync_comp, sync_install_build, do_create_virtualenv, + clean_rsync_folder, rsync_export, get_src_version, sync_build, + do_relaunch_server, install_build, do_create_virtualenv_requirement) +from fabric.api import task, env +from fabric.colors import green + +def build_source(version): + print(green("build source with version %s" % version)) + export_path = export_version(iconolab=version) + export_path_full = os.path.join(export_path,'src') + build_src(export_path_full) + (_,version_str) = get_src_version(env.key, src_path) + return os.path.join(src_path,"dist","%s-%s.tar.gz" % (env.key,version_str)) + +@task +def relaunch_server(do_collectstatic=True, do_migrate=True, do_check_folder_access=False): + print("Relaunch server") + do_relaunch_server(do_collectstatic, do_migrate, env.get('check_folder_access',do_check_folder_access)) + +@task +def sync_site(version): + print(green("sync site with version %s" % version)) + build_path = build_source(version) + res_trans = sync_build(build_path) + # untar build + run('tar zxf %s' % res_trans[0]) + # rebuild virtualenv + requirements_path = os.path.join(res_trans[0][0:-7], env.repos[env.key]['requirements']) + do_create_virtualenv_requirement(requirements_path, env.remote_path['virtualenv'], env.repos[env.key]['python_version']) + # add setting path to virtualenv + ext_path = "import sys; sys.__plen = len(sys.path)\n" + for l in env.remote_path.get('pythonpath', []): + ext_path += l + "\n" + ext_path += "import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)" + put(StringIO.StringIO(ext_path), os.path.join(env.remote_path['virtualenv'], 'lib/python%s/site-packages/_virtualenv_path_extensions.pth'%env.repos[env.key]['python_version'])) + # install build + install_build(res_trans[0], env.remote_path['virtualenv']) + # remove build + untared folder + run('rm -fr "%s" "%s" ' % (res_trans[0], res_trans[0][0:-7])) + clean_export_folder(export_path) + relaunch_server() + +@task +def create_virtualenv(version): + print(green("create virtualenv with version %s" % version)) + export_path = export_version(web=version) + export_path_web = os.path.join(export_path,'web') + venv_remote_export_path = "" + try: + virtualenv_path = os.path.join(export_path_web, "virtualenv") + + venv_remote_export_path = os.path.join(env.remote_path['venv_export'], env.export_prefix, version,"virtualenv") + rsync_export(virtualenv_path, venv_remote_export_path, env.rsync_filters['venv']) + do_create_virtualenv(venv_remote_export_path, env.remote_path['virtualenv']) + finally: + clean_export_folder(export_path) + if venv_remote_export_path: + clean_rsync_folder(venv_remote_export_path) + +if len(env.repos) > 1: + @task + def sync_platform(version): + print(green("sync platform with version web %s" % (version))) + export_path = export_version(web=version) + + export_path_web = os.path.join(export_path,'web') + do_sync_web(version, export_path_web) + + comp_versions = get_comp_versions_dict(export_path_web) + + for key in [k for k in env.repos if k != 'web']: + export_path_key = export_version(**{key: comp_versions[key]}) + export_path_comp = os.path.join(export_path_key, key) + do_sync_comp(key, export_path_comp) + clean_export_folder(export_path_key) + + create_config(export_path_web) + clean_export_folder(export_path) + relaunch_server() diff -r f9d0d6a840cf -r a2e6d314da2f src/dist/iconolab-0.0.1.dev.tar.gz Binary file src/dist/iconolab-0.0.1.dev.tar.gz has changed diff -r f9d0d6a840cf -r a2e6d314da2f src/requirements/base.txt.in --- a/src/requirements/base.txt.in Wed Aug 31 17:20:07 2016 +0200 +++ b/src/requirements/base.txt.in Wed Aug 31 17:37:28 2016 +0200 @@ -1,13 +1,2 @@ -Django >= 1.9 -django-comments-xtd -django-contrib-comments -django-haystack -django-notifications-hq -elasticsearch -jsonfield -Pillow -psycopg2 -pytz -requests -six -sorl-thumbnail >= 12.4a1 +# must run "pip install -r base.txt.in" in src/requirements +-e .. diff -r f9d0d6a840cf -r a2e6d314da2f src/setup.py --- a/src/setup.py Wed Aug 31 17:20:07 2016 +0200 +++ b/src/setup.py Wed Aug 31 17:37:28 2016 +0200 @@ -87,6 +87,9 @@ m.write("include LICENSE\n") m.write("include README.md\n") m.write("include MANIFEST.in\n") + m.write("include requirements/base.txt\n") + m.write("include requirements/dev.txt\n") + m.write("include requirements/prod.txt\n") for entry in data_files: file_list = entry[1] for filename in file_list: @@ -116,6 +119,21 @@ 'Programming Language :: Python', 'Topic :: Utilities', ], + install_requires=[ + "Django >= 1.9", + "django-comments-xtd", + "django-contrib-comments", + "django-haystack", + "django-notifications-hq", + "elasticsearch", + "jsonfield", + "Pillow", + "psycopg2", + "pytz", + "requests", + "six", + "sorl-thumbnail >= 12.4a1", + ], )