--- a/.hgignore Thu May 10 09:51:59 2012 +0200
+++ b/.hgignore Thu May 10 12:14:34 2012 +0200
@@ -1,56 +1,32 @@
syntax: regexp
^web/index$
-syntax: regexp
^web/log$
-syntax: regexp
^virtualenv/web/env/
-syntax: regexp
.*\.pyc$
-
-syntax: regexp
^virtualenv/web/project-boot\.py$
-syntax: regexp
^web/ldtplatform/config\.py$
^web/ldtplatform/\.htaccess$
^web/\.htaccess$
-
-syntax: regexp
^virtualenv/web/distribute-0\.6\.14\.tar\.gz$
-syntax: regexp
^src/ldt/Ldt\.egg-info$
-syntax: regexp
^src/ldt/dist$
-
-syntax: regexp
^web/static/media/ldt$
-syntax: regexp
^web/static/site/admin$
-syntax: regexp
^web/static/site/ldt$
-syntax: regexp
^virtualenv/setup/project-boot\.py$
-syntax: regexp
^virtualenv/setup/env/
+^src/ldt/build$
+^src/ldt/ldt\.egg-info$
+^src/ldt/distribute-0\.6\.14\.tar\.gz$
+^src/ldt/distribute-0\.6\.14-py2\.6\.egg$
+^src/ldt/MANIFEST\.in$
+^src/ldt/MANIFEST$
+^\.pydevproject$
+^web/static/media/cache$
+^\.settings/org\.eclipse\.core\.resources\.prefs$
+^web/static/media/thumbnails$
+^virtualenv/sync/env
syntax: regexp
-^src/ldt/build$
-syntax: regexp
-^src/ldt/ldt\.egg-info$
-syntax: regexp
-^src/ldt/distribute-0\.6\.14\.tar\.gz$
-syntax: regexp
-^src/ldt/distribute-0\.6\.14-py2\.6\.egg$
-syntax: regexp
-^src/ldt/MANIFEST\.in$
-
-syntax: regexp
-^src/ldt/MANIFEST$
-syntax: regexp
-^\.pydevproject$
-syntax: regexp
-^web/static/media/cache$
-syntax: regexp
-^\.settings/org\.eclipse\.core\.resources\.prefs$
-syntax: regexp
-^web/static/media/thumbnails$
\ No newline at end of file
+^sbin/sync/config\.py$
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbin/sync/config.py.tmpl Thu May 10 12:14:34 2012 +0200
@@ -0,0 +1,29 @@
+from fabric.api import env
+
+env.hosts = ['iri@web.iri.centrepompidou.fr']
+env.base_export_path = "~/tmp"
+env.export_prefix = "platform"
+env.remote_web_path = "/iridata/www/ldt/"
+env.platform_web_module = "ldtplatform"
+env.remote_ldt_base_path = "/tmp"
+env.remote_virtualenv_path = "/iridata/virtualenv/platform"
+env.remote_venv_export_path = "/tmp"
+env.remote_baseline_venv = "/iridata/virtualenv/baseline2.7"
+env.web_rsync_filters = [
+ "+ core",
+ "P .htpasswd",
+ "P .htaccess",
+ "P ldtplatform/.htaccess",
+ "P ldtplatform/config.py",
+ "P ldtplatform/modwsgi.wsgi",
+ "P robots.txt",
+ "P env/***",
+ "P log/***",
+ "P index/***",
+ "P static/media/***",
+ "P crossdomain.xml",
+]
+env.venv_rsync_filters = [
+ "+ core",
+]
+env.web_relaunch_cmd = "supervisorctl restart platform"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbin/sync/fabfile.py Thu May 10 12:14:34 2012 +0200
@@ -0,0 +1,191 @@
+from fabric.api import task, run, local, env, cd, put, prefix, sudo
+from fabric.colors import green
+from fabric.contrib.project import rsync_project
+from mercurial import commands, ui, hg, cmdutil
+import imp
+import os, os.path
+import shutil
+import sys
+
+import config
+
+def get_export_path(version):
+ base_path = os.path.join(env.base_export_path,env.export_prefix).lstrip("/")
+ return os.path.expanduser(base_path) + "_%s" % (str(version))
+
+def clean_export_folder(path):
+ print("Removing %s" % path)
+ if os.path.isdir(path):
+ shutil.rmtree(path, ignore_errors=True)
+
+def do_export_version(path, version):
+ print("Export version %s"%str(version))
+
+ #hgui = ui.ui()
+ #repo = hg.repository(hgui, cmdutil.findrepo(os.getcwd()))
+ #commands.archive(hgui, repo, path, rev=str(version))
+
+ local("hg archive -r \'%s\' \"%s\"" % (str(version),path))
+ print("Export version %s done"%str(version))
+
+def rsync_export(path, remotepath, filters):
+ print("Rsync %s to %s",(path,remotepath))
+
+ if filters:
+ filter_option_str = " ".join(["--filter \"%s\"" % (f) for f in filters])
+ else:
+ filter_option_str =""
+
+ run("mkdir -p \"%s\"" % remotepath)
+ rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)
+ print("Rsync %s to %s done",(path,remotepath))
+
+def clean_rsync_folder(remotepath):
+ print("clean rsync folder %s" % remotepath)
+ run("rm -fr \"%s\"" % remotepath)
+
+def build_src(path):
+ print("Build source dist at %s" % path)
+ f = None
+ try:
+ f, pathname, description = imp.find_module("setup", [path])
+ print(" 2 Build source dist at %s" % path)
+ setup_mod = imp.load_module("setup", f, pathname, description)
+ print(" 3 Build source dist at %s" % path)
+ finally:
+ if f:
+ f.close()
+
+ setup_mod.launch_setup("setup.py", ['sdist'])
+
+ print("Build source dist at %s done" % path)
+
+
+def get_src_version(path):
+ print("get src version at %s" % path)
+ f = None
+ try:
+ f, pathname, description = imp.find_module("ldt", [path])
+ ldt_mod = imp.load_module("ldt", f, pathname, description)
+ finally:
+ if f:
+ f.close()
+ version = ldt_mod.VERSION
+ version_str = ldt_mod.get_version()
+
+ return (version, version_str)
+
+
+def sync_build(path):
+ print("Sync build %s" % path)
+ with cd(env.remote_ldt_base_path):
+ filename = os.path.basename(path)
+ res_trans = put(path, os.path.join(env.remote_ldt_base_path, filename))
+ print("Sync build %s to %s" % (path,repr(res_trans)))
+ return res_trans
+
+def remove_build(path):
+ print("remove build build %s" % path)
+ run("rm \"%s\"" % path)
+
+
+def install_build(remotepath, remotevirtualenvpath):
+ print("Install build %s in %s" % (remotepath, remotevirtualenvpath))
+ activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
+
+ with prefix("source %s" % activate_path):
+ run("pip install \"%s\"" % remotepath)
+
+def collectstatic(remotepath, remotevirtualenvpath):
+ print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))
+ activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
+ with prefix("source %s" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(os.path.join(remotepath, env.platform_web_module)):
+ run("python manage.py collectstatic --noinput")
+
+
+
+def export_version(version):
+ print("export version %s" % str(version))
+
+ export_path = get_export_path(version)
+
+ clean_export_folder(export_path)
+ do_export_version(export_path,version)
+
+ return export_path
+
+def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):
+ print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))
+ if "remote_baseline_venv" in env and env.remote_baseline_venv:
+ prefix_str = os.path.join(env.remote_baseline_venv, "bin/activate")
+ else:
+ prefix_str = "echo"
+ run("mkdir -p \"%s\"" % remotevirtualenvpath)
+ with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):
+ run("python create_python_env.py")
+ run("python project-boot.py --unzip-setuptools --no-site-packages --clear --type-install=local \"%s\"" % remotevirtualenvpath)
+
+def do_sync_ldt(version, export_path):
+ print("do_sync_ldt with version %s and path %s" % (version,export_path))
+ src_path = export_path + "/src/ldt"
+ build_src(src_path)
+ (_,version_str) = get_src_version(src_path)
+ build_path = os.path.join(src_path,"dist","ldt-%s.tar.gz" % version_str)
+ res_trans = None
+ try:
+ res_trans = sync_build(build_path)
+ install_build(res_trans[0], env.remote_virtualenv_path)
+ finally:
+ if res_trans:
+ remove_build(res_trans[0])
+
+def do_sync_web(version, export_path):
+ print("do_sync_web with version %s and path %s" % (version,export_path))
+ web_path = os.path.join(export_path,"web/")
+ rsync_export(web_path, env.remote_web_path, env.web_rsync_filters)
+
+def relaunch_server():
+ print("Relaunch server")
+ collectstatic(env.remote_web_path, env.remote_virtualenv_path)
+ sudo(env.web_relaunch_cmd)
+
+@task
+def sync_web(version):
+ print(green("sync web with version %s" % version))
+ export_path = export_version(version)
+ do_sync_web(version, export_path)
+ clean_export_folder(export_path)
+ relaunch_server()
+
+@task
+def sync_ldt(version):
+ print(green("sync ldt with version %s" % version))
+ export_path = export_version(version)
+ do_sync_ldt(version, export_path)
+ clean_export_folder(export_path)
+ relaunch_server()
+
+@task
+def sync_platform(version):
+ print(green("sync platform with version %s" % version))
+ export_path = export_version(version)
+ do_sync_ldt(version, export_path)
+ do_sync_web(version, export_path)
+ clean_export_folder(export_path)
+ relaunch_server()
+
+@task
+def create_virtualenv(version):
+ print(green("create virtualenv with version %s" % version))
+ export_path = export_version(version)
+ venv_remote_export_path = ""
+ try:
+ virtualenv_path = os.path.join(export_path, "virtualenv")
+
+ venv_remote_export_path = os.path.join(env.remote_venv_export_path, env.export_prefix, version,"virtualenv")
+ rsync_export(virtualenv_path, venv_remote_export_path, env.venv_rsync_filters)
+ do_create_virtualenv(venv_remote_export_path, env.remote_virtualenv_path)
+ finally:
+ clean_export_folder(export_path)
+ if venv_remote_export_path:
+ clean_rsync_folder(venv_remote_export_path)
--- a/sbin/sync/sync_ldt_platform Thu May 10 09:51:59 2012 +0200
+++ b/sbin/sync/sync_ldt_platform Thu May 10 12:14:34 2012 +0200
@@ -1,4 +1,3 @@
-
#!/usr/bin/env bash
set -e
if [ -d ~/tmp/platform_V$1 ]; then
--- a/src/ldt/setup.py Thu May 10 09:51:59 2012 +0200
+++ b/src/ldt/setup.py Thu May 10 12:14:34 2012 +0200
@@ -4,7 +4,6 @@
from distutils.command.install import INSTALL_SCHEMES
import sys
-print("hello 1")
class osx_install_data(install_data):
# On MacOS, the platform-specific lib dir is /System/Library/Framework/Python/.../
@@ -18,7 +17,6 @@
# install_data class uses ('install_data', 'install_dir') instead.
self.set_undefined_options('install', ('install_lib', 'install_dir'))
install_data.finalize_options(self)
-print("hello 2")
def fullsplit(path, result=None):
"""
@@ -34,7 +32,7 @@
return result
return fullsplit(head, [tail] + result)
-print("hello 3")
+
def launch_setup(script_name, script_args):
if sys.platform == "darwin":
cmdclasses = {'install_data': osx_install_data}
@@ -120,10 +118,8 @@
if __name__ == "__main__":
-
- print("hello 4")
+
script_name = os.path.basename(sys.argv[0])
script_args = sys.argv[1:]
- print("hello 5")
- launch_setup(script_name, script_args)
- print("hello 6")
\ No newline at end of file
+
+ launch_setup(script_name, script_args)
\ No newline at end of file
--- a/virtualenv/res/lib/lib_create_env.py Thu May 10 09:51:59 2012 +0200
+++ b/virtualenv/res/lib/lib_create_env.py Thu May 10 12:14:34 2012 +0200
@@ -32,6 +32,10 @@
'SORL_THUMBNAIL' : { 'setup': 'sorl-thumbnail', 'url':'http://pypi.python.org/packages/source/s/sorl-thumbnail/sorl-thumbnail-11.12.tar.gz', 'local':"sorl-thumbnail-v10.12.1.tar.gz"},
'LIBJPEG': {'setup': None, 'url':'jpegsrc.v8d.tar.gz', 'local':'jpegsrc.v8d.tar.gz'},
'ZLIB': {'setup': None, 'url':'zlib-1.2.6.tar.gz', 'local':'zlib-1.2.6.tar.gz'},
+ 'PYCRYPTO': {'setup': 'pycrypto', 'url':'https://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/pycrypto-2.5.tar.gz', 'local':'pycrypto-2.5.tar.gz'},
+ 'SSH': {'setup': 'ssh', 'url':'http://pypi.python.org/packages/source/s/ssh/ssh-1.7.13.tar.gz#md5=26800ef2c1ee3f185f48fd05258302f4', 'local':'ssh-1.7.13.tar.gz'},
+ 'FABRIC': {'setup': 'fabric', 'url':'https://github.com/fabric/fabric/tarball/1.4.1', 'local':'fabric-1.4.1.tar.gz'},
+ 'MERCURIAL': {'setup': 'mercurial', 'url':'http://mercurial.selenic.com/release/mercurial-2.1.2.tar.gz', 'local':'mercurial-2.1.2.tar.gz'},
}
if system_str == 'Windows':
Binary file virtualenv/res/src/fabric-1.4.1.tar.gz has changed
Binary file virtualenv/res/src/pycrypto-2.5.tar.gz has changed
Binary file virtualenv/res/src/ssh-1.7.13.tar.gz has changed
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/virtualenv/sync/create_python_env.py Thu May 10 12:14:34 2012 +0200
@@ -0,0 +1,64 @@
+"""
+Call this like ``python create_python_env.py``; it will
+refresh the project-boot.py script
+
+-prerequisite:
+
+- virtualenv
+
+- python project-boot.py --unzip-setuptools --no-site-packages --clear --type-install=local <path_to_venv>
+
+"""
+
+import os
+import subprocess
+import re
+import sys
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+base_dir = here
+script_name = os.path.join(base_dir, 'project-boot.py')
+
+import virtualenv
+
+# things to install
+# - psycopg2 -> pip
+# - PIL -> pip
+# - pyxml -> pip
+# - 4Suite-xml - easy_install ftp://ftp.4suite.org/pub/4Suite/4Suite-XML-1.0.2.tar.bz2
+# - pylucene - script
+
+src_base = os.path.abspath(os.path.join(here,"..","res","src")).replace("\\","/")
+lib_path = os.path.abspath(os.path.join(here,"..","res","lib")).replace("\\","/")
+patch_path = os.path.abspath(os.path.join(here,"res","patch")).replace("\\","/")
+
+
+EXTRA_TEXT = "import sys\n"
+EXTRA_TEXT += "sys.path.append('%s')\n" % (lib_path)
+EXTRA_TEXT += "sys.path.append('%s')\n" % (os.path.abspath(os.path.join(here,"res")).replace("\\","/"))
+EXTRA_TEXT += "from res_create_env import generate_install_methods\n"
+EXTRA_TEXT += "adjust_options, extend_parser, after_install = generate_install_methods(path_locations, '%s', Logger, call_subprocess)\n" % (src_base)
+
+
+def main():
+ python_version = ".".join(map(str,sys.version_info[0:2]))
+ text = virtualenv.create_bootstrap_script(EXTRA_TEXT, python_version=python_version)
+ if os.path.exists(script_name):
+ f = open(script_name)
+ cur_text = f.read()
+ f.close()
+ else:
+ cur_text = ''
+ print 'Updating %s' % script_name
+ if cur_text == 'text':
+ print 'No update'
+ else:
+ print 'Script changed; updating...'
+ f = open(script_name, 'w')
+ f.write(text)
+ f.close()
+
+if __name__ == '__main__':
+ main()
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/virtualenv/sync/res/res_create_env.py Thu May 10 12:14:34 2012 +0200
@@ -0,0 +1,16 @@
+import platform
+
+from lib_create_env import lib_generate_install_methods, install_pylucene, install_psycopg2
+
+system_str = platform.system()
+
+
+INSTALLS = [#(key,method, option_str, dict_extra_env)
+ ('DISTRIBUTE', 'pip', None, None),
+ ('PYCRYPTO', 'pip', None, None),
+ ('SSH', 'pip', None, None),
+ ('FABRIC', 'pip', None, None),
+]
+
+def generate_install_methods(path_locations, src_base, Logger, call_subprocess):
+ return lib_generate_install_methods(path_locations, src_base, Logger, call_subprocess, INSTALLS)