--- a/.hgtags Tue May 07 11:15:15 2013 +0200
+++ b/.hgtags Mon May 13 15:55:28 2013 +0200
@@ -18,6 +18,7 @@
7152adc1daaf7f1b8fdd2223d7799fef90fe6c86 V02.08.01
a73f1e6e1eed53bfdc8c17a0e014c579937bc888 V02.08.02
89e5ee57bd89ad5858848ad5c69950164d074a0c V02.08.03
+ccca12eaa406a5138ce67a9870c58cf4ff9560cc V02.08.04
eb20d3d467bedcd848a40f632cacc461c86521af V02.08.05
188306b32f74b5d0b7bb643d7c169d62c67c2e1d V02.08.06
0f2fe357ceea06884cf5d75413c66b976268fe54 V02.08.07
--- a/sbin/sync/config.py.tmpl Tue May 07 11:15:15 2013 +0200
+++ b/sbin/sync/config.py.tmpl Mon May 13 15:55:28 2013 +0200
@@ -1,11 +1,17 @@
from fabric.api import env
+from random import choice
env.hosts = ['iri@web.iri.centrepompidou.fr']
env.web_group = 'www-data'
env.folders = ['log', 'static/media']
-env.repos = {'ldt': "/Users/ymh/dev/workspace/platform", 'web': "/Users/ymh/dev/workspace/platform_web", 'hashcut': "/Users/ymh/dev/workspace/hashcut" }
+env.repos = {
+ 'ldt': {'repo':"/Users/ymh/dev/workspace/platform", 'src_root':'src/ldt'},
+ 'web': {'repo':"/Users/ymh/dev/workspace/platform_web", 'src_root':'src'},
+ 'hashcut': {'repo':"/Users/ymh/dev/workspace/hashcut", 'src_root':'src'},
+ 'django-chunked-uploads' : {'repo':"/Users/ymh/dev/workspace/django-chunked-uploads", 'src_root':'', 'module': 'chunked-uploads'},
+}
env.base_export_path = "~/tmp"
env.export_prefix = "platform"
@@ -13,7 +19,7 @@
'web':"/iridata/www/ldt/",
'src':"/Users/ymh/dev/tmp/testfab/src",
'virtualenv':"/iridata/virtualenv/ldt",
- 'ldt_base':"/tmp",
+ 'build_export':"/tmp",
'venv_export':"/iridata/users/iri/tmp",
}
@@ -66,7 +72,8 @@
'email_host_user' : "",
'email_host_user' : "",
'email_port' : 0,
- 'forbidden_stream_url' : "rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/forbidden_stream.mp4?old_path="
+ 'forbidden_stream_url' : "rtmp://media.iri.centrepompidou.fr/ddc_player/mp4:video/forbidden_stream.mp4?old_path=",
+ 'secret_key' : ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)]),
},
}
--- a/sbin/sync/fabfile.py Tue May 07 11:15:15 2013 +0200
+++ b/sbin/sync/fabfile.py Mon May 13 15:55:28 2013 +0200
@@ -1,9 +1,10 @@
-from fabric.api import task, env, sudo
+from fablib import (export_version, do_sync_web, create_config,
+ clean_export_folder, do_sync_comp, sync_install_build, do_create_virtualenv,
+ clean_rsync_folder, rsync_export, get_comp_versions_dict, SyncComp,
+ do_relaunch_server)
+from fabric.api import task, env
from fabric.colors import green
-from ldt_fablib import (check_folder_access, syncdb, collectstatic,
- export_version, do_sync_web, create_config, clean_export_folder, do_sync_comp,
- sync_install_build, do_create_virtualenv, clean_rsync_folder, rsync_export,
- get_comp_versions_dict, SyncComp, do_relaunch_server)
+import config #@UnusedImport
import imp
import os.path
--- a/sbin/sync/ldt_fablib/__init__.py Tue May 07 11:15:15 2013 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,293 +0,0 @@
-from fabric.api import run, local, env, cd, put, prefix, sudo, lcd
-from fabric.context_managers import settings
-from fabric.contrib.files import exists, upload_template
-from fabric.contrib.project import rsync_project
-from mercurial import commands, ui, hg, cmdutil
-from fabric.tasks import Task
-from fabric.colors import green
-import config
-import imp
-import os
-import os.path
-import re
-import shutil
-import urlparse
-
-
-
-__all__ = ["check_folder_access", "syncdb", "collectstatic",
- "export_version", "do_sync_web", "create_config", "clean_export_folder", "relaunch_server",
- "do_sync_ldt", "sync_install_build", "do_create_virtualenv", "clean_rsync_folder", "rsync_export",
- "SyncComp"]
-
-def get_export_path(version):
- base_path = os.path.join(env.base_export_path,env.export_prefix).rstrip("/")
- return os.path.expanduser(base_path) + "_%s" % (str(version))
-
-def clean_export_folder(path):
- print("Removing %s" % path)
- if os.path.isdir(path):
- shutil.rmtree(path, ignore_errors=True)
-
-def do_export_version(path, **export_keys):
- print("Export version %s : %s " % (path,repr(export_keys)))
-
- for export_key, version in export_keys.items():
- export_path = os.path.join(path,export_key)
-
- repo_url = env.repos[export_key]
- url_part = urlparse.urlparse(repo_url)
- if url_part.scheme or url_part.netloc:
- # this is a remote repo. Let's clone first
- clone_path = os.path.join(path,'clone',export_keys)
- os.makedirs(clone_path)
- local("hg clone \"%s\" \"%s\"" % (repo_url,clone_path))
- else:
- clone_path = repo_url
-
- with lcd(clone_path):
- local("hg archive -r \'%s\' \"%s\"" % (str(version),export_path))
-
- print("Export version %s done"%repr(export_keys))
-
-
-def get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, settings_key):
- activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
- with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
- tempfilepath = run("mktemp -t ldtplatform.XXXXXX")
- with settings(warn_only=True):
- run("echo \"import os\" > %s" % (tempfilepath))
- map(lambda str_echo: run("echo \"%s\" >> %s" % (str_echo, tempfilepath)),
- ["os.environ.setdefault('DJANGO_SETTINGS_MODULE', '%s.settings')" % (platform_web_module),
- "from django.conf import settings",
- "print settings.%s" % (settings_key)])
- res = run("python < %s" % (tempfilepath))
- run("rm -f \"%s\"" % (tempfilepath))
- return res
-
-
-
-def rsync_export(path, remotepath, filters):
- print("Rsync %s to %s",(path,remotepath))
-
- filter_option_str = "--progress --stats"
- if filters:
- filter_option_str += " " + " ".join(["--filter \"%s\"" % (f) for f in filters])
-
- run("mkdir -p \"%s\"" % remotepath)
- rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)
- print("Rsync %s to %s done",(path,remotepath))
-
-def clean_rsync_folder(remotepath):
- print("clean rsync folder %s" % remotepath)
- run("rm -fr \"%s\"" % remotepath)
-
-def build_src(path):
- print("Build source dist at %s" % path)
- f = None
- try:
- f, pathname, description = imp.find_module("setup", [path])
- print(" 2 Build source dist at %s" % path)
- setup_mod = imp.load_module("setup", f, pathname, description)
- print(" 3 Build source dist at %s" % path)
- finally:
- if f:
- f.close()
-
- setup_mod.launch_setup("setup.py", ['sdist'])
-
- print("Build source dist at %s done" % path)
-
-
-def get_src_version(mod_name, path):
- print("get src version for %s at %s" % (mod_name,path))
- f = None
- try:
- f, pathname, description = imp.find_module(mod_name, [path])
- ldt_mod = imp.load_module(mod_name, f, pathname, description)
- finally:
- if f:
- f.close()
- version = ldt_mod.VERSION
- version_str = ldt_mod.get_version()
-
- return (version, version_str)
-
-
-def sync_build(path):
- print("Sync build %s" % path)
- with cd(env.remote_path['ldt_base']):
- filename = os.path.basename(path)
- res_trans = put(path, os.path.join(env.remote_path['ldt_base'], filename))
- print("Sync build %s to %s" % (path,repr(res_trans)))
- return res_trans
-
-def remove_build(path):
- print("remove build build %s" % path)
- run("rm \"%s\"" % path)
-
-
-def install_build(remotepath, remotevirtualenvpath, module_to_uninstall= None):
- print("Install build %s in %s" % (remotepath, remotevirtualenvpath))
- activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
-
- with prefix("source %s" % activate_path):
- if module_to_uninstall:
- with settings(warn_only=True):
- run("pip uninstall -y %s" % module_to_uninstall)
- run("pip install \"%s\"" % remotepath)
-
-def collectstatic(remotepath, remotevirtualenvpath, platform_web_module):
- print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))
- remotestaticsitepath = get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, "STATIC_ROOT")
- activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
- with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
- #remocve old files optio -c of collect static fail !
- run("rm -fr \"%s\"" % (remotestaticsitepath))
- run("python manage.py collectstatic --noinput")
-
-def syncdb(remotepath, remotevirtualenvpath):
- activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
- with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):
- run("python manage.py syncdb --migrate --noinput")
-
-def create_config(export_path):
- print("Create config from %s" % (export_path,))
- remotepath = env.remote_path['src']
- remote_config_path = os.path.join(remotepath, env.platform_web_module, "config.py")
- template_path = os.path.join(export_path, "src", env.platform_web_module, "config.py.tmpl")
-
- context = {
- 'base_dir': os.path.join(remotepath, env.platform_web_module).rstrip("/")+"/",
- 'asctime': '%(asctime)s',
- 'levelname': '%(levelname)s',
- 'message': '%(message)s',
- 'module': '%(module)s',
- }
- context.update(env.config['web'])
-
- if not exists(remote_config_path, verbose=True):
- upload_template(template_path, remote_config_path, context=context)
-
-def export_version(**kwargs):
- print("export version %s" % (repr(kwargs)))
-
- export_path = kwargs.get('path', None)
-
- if not export_path:
- export_path = get_export_path("_".join(["%s_%s" % (k,v) for k,v in kwargs.items()]))
-
- clean_export_folder(export_path)
-
- do_export_version(export_path,**kwargs)
-
- return export_path
-
-def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):
- print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))
- activate_path = os.path.join(remotevirtualenvpath, "bin/activate")
- if "remote_baseline_venv" in env and env.remote_baseline_venv:
- prefix_str = "source \"%s\"" % os.path.join(env.remote_baseline_venv, "bin/activate")
- else:
- prefix_str = "echo"
- with settings(warn_only=True):
- run("rm -fr \"%s\"" % remotevirtualenvpath)
- run("mkdir -p \"%s\"" % remotevirtualenvpath)
- with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):
- run("python create_python_env.py")
- run("python project-boot.py \"%s\"" % remotevirtualenvpath)
- with prefix("source \"%s\"" % activate_path):
- run("pip install -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))
-
-def do_sync_comp(key, export_path):
- print("do_sync_comp with path %s" % (export_path))
-
- src_path = os.path.join(export_path,"src")
- # find setup.py
- for root, _, files in os.walk(src_path):
- if "setup.py" in files:
- src_path = root
-
- build_src(src_path)
- (_,version_str) = get_src_version(key, src_path)
- build_path = os.path.join(src_path,"dist","%s-%s.tar.gz" % (key,version_str))
- sync_install_build(build_path, key)
-
-
-def sync_install_build(build_path, module_to_uninstall=None):
- res_trans = None
- try:
- res_trans = sync_build(build_path)
- install_build(res_trans[0], env.remote_path['virtualenv'], module_to_uninstall)
- finally:
- if res_trans:
- remove_build(res_trans[0])
-
-
-def do_sync_web(version, export_path):
- print("do_sync_web with version %s and path %s" % (version,export_path))
- #sync web
- web_path = os.path.join(export_path,"web/")
- rsync_export(web_path, env.remote_path['web'], env.rsync_filters['web'])
- #sync src
- src_path = os.path.join(export_path,"src/")
- rsync_export(src_path, env.remote_path['src'], env.rsync_filters['src'])
-
-
-def check_folder_access():
- print("Check folder access")
- # get remote user
- for folder_path in env.folders:
- if not os.path.isabs(folder_path):
- folder_path = env.remote_path['web'].rstrip("/")+ "/" + folder_path
- with settings(warn_only=True):
- if not exists(folder_path):
- run("mkdir -p \"%s\"" % folder_path)
- run("chown -R -c :%s \"%s\"" % (env.web_group, folder_path))
- run("chmod -R -c g+w \"%s\"" % folder_path)
-
-def get_comp_versions_dict(export_path_web):
- comp_versions = {}
- with open(os.path.join(export_path_web, 'src', 'requirement.txt')) as f:
- for line in f:
- m = re.match('^(\w+)\s+\(\s*\=\=\s*([\.\d\w]+)\s*\)', line)
- if m:
- key, version_req = m.groups()
- if "." in version_req:
- version_req = "V" + ".".join(["%02d" % (int(s)) if s.isdigit() else s for s in version_req.split(".")])
- comp_versions[key] = version_req
-
- return comp_versions
-
-def do_relaunch_server(do_collectstatic, do_syncdb):
- check_folder_access()
- if do_syncdb:
- syncdb(env.remote_path['src'], env.remote_path['virtualenv'])
- if do_collectstatic:
- collectstatic(env.remote_path['src'], env.remote_path['virtualenv'], env.platform_web_module)
- sudo(env.web_relaunch_cmd, shell=False)
-
-
-class SyncComp(Task):
-
- def __init__(self, key):
- self.key = key
-
- def __get_name(self):
- return "sync_" + self.key
-
- name = property(__get_name)
-
- def run(self, version):
- print(green("sync ldt with version %s" % version))
- export_path_web = export_version(web=version)
- export_path_web_full = os.path.join(export_path_web,'web')
- comp_versions = get_comp_versions_dict(export_path_web_full)
-
- export_path = export_version(**{self.key:comp_versions[self.key]})
- export_path_full = os.path.join(export_path,self.key)
- do_sync_comp(self.key, export_path_full)
- clean_export_folder(export_path)
- clean_export_folder(export_path_web)
-
- do_relaunch_server(do_collectstatic=True, do_syncdb=True)
--- a/src/ldtplatform/config.py.tmpl Tue May 07 11:15:15 2013 +0200
+++ b/src/ldtplatform/config.py.tmpl Mon May 13 15:55:28 2013 +0200
@@ -36,6 +36,9 @@
# DO NOT forget to do command line ./manage.py collectstatic to gather static media into the web/static folder
STATIC_ROOT = BASE_STATIC_ROOT + "site/"
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '%(secret_key)s'
+
# PATH to the ffmpeg executable, used to know automatically the media file duration
FFMPEG_PATH = "%(ffmpeg_path)s"
@@ -76,6 +79,11 @@
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
+ 'filters': {
+ 'require_debug_false': {
+ '()': 'django.utils.log.RequireDebugFalse'
+ }
+ },
'formatters' : {
'simple' : {
'format': "%(asctime)s - %(levelname)s : %(message)s",
@@ -87,6 +95,7 @@
'handlers': {
'mail_admins': {
'level': 'ERROR',
+ 'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
},
'stream_to_console': {
@@ -161,3 +170,5 @@
#'INDEX_NAME': 'ldt',
},
}
+
+LDT_INDEXATION_INSERT_BATCH_SIZE = 5000
\ No newline at end of file
--- a/src/ldtplatform/settings.py Tue May 07 11:15:15 2013 +0200
+++ b/src/ldtplatform/settings.py Mon May 13 15:55:28 2013 +0200
@@ -148,6 +148,7 @@
'sorl.thumbnail',
'tastypie',
'hashcut',
+ 'chunked_uploads',
)
AUTH_PROFILE_MODULE = 'user.UserProfile'
@@ -241,25 +242,29 @@
'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
},
}
+HAYSTACK_SIGNAL_PROCESSOR = 'ldt.indexation.signals.LdtSignalProcessor'
from config import *
-if not "PLATFORM_BASE_URL" in locals():
- PLATFORM_BASE_URL = BASE_URL + 'ldtplatform/'
+if not "SRC_BASE_URL" in locals():
+ SRC_BASE_URL = BASE_URL + __name__.split('.')[0] + '/'
+
+if not "SRC_BASE_URL" in locals():
+ SRC_BASE_URL = BASE_URL + 'ldtplatform/'
if not "LOGIN_URL" in locals():
- LOGIN_URL = PLATFORM_BASE_URL + 'accounts/login/'
+ LOGIN_URL = SRC_BASE_URL + 'accounts/login/'
if not "LOGOUT_URL" in locals():
- LOGOUT_URL = PLATFORM_BASE_URL + 'accounts/disconnect/'
+ LOGOUT_URL = SRC_BASE_URL + 'accounts/disconnect/'
if not "LOGIN_REDIRECT_URL" in locals():
- LOGIN_REDIRECT_URL = PLATFORM_BASE_URL + 'ldt/'
+ LOGIN_REDIRECT_URL = SRC_BASE_URL + 'ldt/'
if not "LOGOUT_REDIRECT_URL" in locals():
- LOGOUT_REDIRECT_URL = PLATFORM_BASE_URL + 'accounts/login'
+ LOGOUT_REDIRECT_URL = SRC_BASE_URL + 'accounts/login'
if not "PROFILE_REDIRECT_URL" in locals():
- PROFILE_REDIRECT_URL = PLATFORM_BASE_URL + 'auth_accounts/create/profile'
+ PROFILE_REDIRECT_URL = SRC_BASE_URL + 'auth_accounts/create/profile'
if not "LOGIN_ERROR_URL" in locals():
- LOGIN_ERROR_URL = PLATFORM_BASE_URL + 'accounts/login'
+ LOGIN_ERROR_URL = SRC_BASE_URL + 'accounts/login'
# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
--- a/virtualenv/res/lib/lib_create_env.py Tue May 07 11:15:15 2013 +0200
+++ b/virtualenv/res/lib/lib_create_env.py Mon May 13 15:55:28 2013 +0200
@@ -17,8 +17,8 @@
URLS = {
#'': {'setup': '', 'url':'', 'local':''},
'DISTRIBUTE': {'setup': 'distribute', 'url':'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.31.tar.gz', 'local':"distribute-0.6.31.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'DJANGO': {'setup': 'django', 'url': 'http://www.djangoproject.com/download/1.4.2/tarball/', 'local':"Django-1.4.2.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'DJANGO-EXTENSIONS': { 'setup': 'django-extensions', 'url':'https://github.com/django-extensions/django-extensions/tarball/0.8', 'local':"django-extensions-0.8.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'DJANGO': {'setup': 'django', 'url': 'http://www.djangoproject.com/download/1.4.5/tarball/', 'local':"Django-1.4.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'DJANGO-EXTENSIONS': { 'setup': 'django-extensions', 'url':'https://github.com/django-extensions/django-extensions/archive/1.0.2.tar.gz', 'local':"django-extensions-1.0.2.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'DJANGO-REGISTRATION': { 'setup': 'django-registration', 'url':'https://bitbucket.org/ubernostrum/django-registration/get/v0.8.tar.gz', 'local':"django-registration-0.8.tar.gz", 'install': {'method': 'easy_install', 'option_str': '-Z', 'dict_extra_env': None}},
'DJANGO-TAGGING': { 'setup': 'django-tagging', 'url':'http://django-tagging.googlecode.com/files/django-tagging-0.3.1.tar.gz', 'local':"django-tagging-0.3.1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'OAUTH2': { 'setup': 'python-oauth2', 'url':"https://github.com/simplegeo/python-oauth2/tarball/hudson-python-oauth2-211", 'local':"oauth2-1.5.211.tar.gz", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},
@@ -29,16 +29,17 @@
'SOCIAL_AUTH': {'setup':'social_auth', 'url':'https://github.com/omab/django-social-auth/tarball/v0.3.10', 'local':"omab-django-social-auth-v0.3.10-modified.tar.gz", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},
'SOUTH': { 'setup': 'South', 'url':'http://www.aeracode.org/releases/south/south-0.7.5.tar.gz', 'local':"south-0.7.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'DJANGO_GUARDIAN' : { 'setup': 'django-guardian', 'url':'https://github.com/lukaszb/django-guardian/archive/9e18609c69cfc4f21d9ffe6bfa47bc14e2c8cf23.zip', 'local':"django-guardian-9e18609c69cfc4f21d9ffe6bfa47bc14e2c8cf23.zip", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'SORL_THUMBNAIL' : { 'setup': 'sorl-thumbnail', 'url':'http://pypi.python.org/packages/source/s/sorl-thumbnail/sorl-thumbnail-11.12.tar.gz', 'local':"sorl-thumbnail-v10.12.1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'LIBJPEG': {'setup': None, 'url':'jpegsrc.v8d.tar.gz', 'local':'jpegsrc.v8d.tar.gz', 'install': {'method': 'install_libjpeg', 'option_str': None, 'dict_extra_env': None}},
+ 'SORL_THUMBNAIL' : { 'setup': 'sorl-thumbnail', 'url':'https://github.com/sorl/sorl-thumbnail/archive/master.tar.gz', 'local':"sorl-thumbnail-v11.12.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'LIBJPEG': {'setup': None, 'url':'jpegsrc.v9.tar.gz', 'local':'jpegsrc.v9.tar.gz', 'install': {'method': 'install_libjpeg', 'option_str': None, 'dict_extra_env': None}},
'ZLIB': {'setup': None, 'url':'zlib-1.2.7.tar.gz', 'local':'zlib-1.2.7.tar.gz', 'install': {'method': 'install_zlib', 'option_str': None, 'dict_extra_env': None}},
'PYCRYPTO': {'setup': 'pycrypto', 'url':'https://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/pycrypto-2.6.tar.gz', 'local':'pycrypto-2.6.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'PARAMIKO': {'setup': 'paramiko', 'url':'https://github.com/paramiko/paramiko/archive/v1.9.0.tar.gz', 'local':'paramiko-1.9.0.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'FABRIC': {'setup': 'fabric', 'url':'https://github.com/fabric/fabric/tarball/1.5.1', 'local':'fabric-1.5.1.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'MERCURIAL': {'setup': 'mercurial', 'url':'http://mercurial.selenic.com/release/mercurial-2.2.3.tar.gz', 'local':'mercurial-2.2.3.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'HAYSTACK': {'setup': 'django-haystack', 'url': 'https://github.com/toastdriven/django-haystack/tarball/master', 'local': 'django-haystack-v2.0.0.tar.gz', 'install':{'method':'pip', 'option_str': None, 'dict_extra_env': None}},
- 'REQUESTS': {'setup': 'requests', 'url':'https://github.com/kennethreitz/requests/archive/v1.0.2.tar.gz', 'local':'requests-v1.0.2.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
- 'PYELASTICSEARCH': {'setup': 'pyelasticsearch', 'url':'https://github.com/toastdriven/pyelasticsearch/tarball/master', 'local':'pyelasticsearch.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'HAYSTACK': {'setup': 'django-haystack', 'url': 'https://github.com/toastdriven/django-haystack/archive/v2.0.0.tar.gz', 'local': 'django-haystack-2.0.0.tar.gz', 'install':{'method':'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'REQUESTS': {'setup': 'requests', 'url':'https://github.com/kennethreitz/requests/archive/v1.1.0.tar.gz', 'local':'requests-1.1.0.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'SIMPLEJSON': {'setup': 'simplejson','url':'https://github.com/simplejson/simplejson/archive/v3.0.7.tar.gz', 'local': 'simplejson-3.0.7.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'PYELASTICSEARCH': {'setup': 'pyelasticsearch', 'url':'https://github.com/rhec/pyelasticsearch/archive/0.3.tar.gz', 'local':'pyelasticsearch-0.3.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'WHOOSH': {'setup': 'whoosh', 'url':'https://bitbucket.org/mchaput/whoosh/get/tip.tar.gz', 'local':'whoosh-2.5.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'SETUPTOOLS_HG' : {'setup':'setuptools_hg', 'url':'http://pypi.python.org/packages/source/s/setuptools_hg/setuptools_hg-0.4.tar.gz', 'local': 'setuptools_hg-0.4.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'WSGIREF' : {'setup':'wsgiref', 'url':'http://pypi.python.org/packages/source/w/wsgiref/wsgiref-0.1.2.zip', 'local': 'wsgiref-0.1.2.zip', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
@@ -49,6 +50,7 @@
'PYYAML' : {'setup':'pyyaml', 'url':'http://pypi.python.org/packages/source/P/PyYAML/PyYAML-3.10.tar.gz', 'local': 'PyYAML-3.10.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'PYTHON-DIGEST' : {'setup':'python-digest', 'url':'http://pypi.python.org/packages/source/p/python-digest/python-digest-1.7.tar.gz', 'local': 'python-digest-1.7.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'DJANGO-TASTYPIE' : {'setup':'django-tastypie', 'url':'django-tastypie-0.9.11-modified.tar.gz', 'local': 'django-tastypie-0.9.11-modified.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'DJANGO-CHUNCKED-UPLOADS' : {'setup': 'django-chuncked-uploads', 'url':'https://github.com/IRI-Research/django-chunked-uploads/archive/v0.4.tar.gz', 'local':'django-chunked-uploads-0.4.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
}
if system_str == 'Windows':
@@ -60,7 +62,7 @@
mysqlres = "MySQL-python-1.2.3.win-amd64-py2.7.exe"
URLS.update({
- 'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.4.5.win32-py2.7-pg9.1.3-release.zip', 'local':"psycopg2-2.4.5.win32-py2.7-pg9.1.3-release.zip", 'install': {'method': 'install_psycopg2', 'option_str': None, 'dict_extra_env': None}},
+ 'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.4.6.win32-py2.7-pg9.2.2-release.exe', 'local':"psycopg2-2.4.6.win32-py2.7-pg9.2.2-release.exe", 'install': {'method': 'install_psycopg2', 'option_str': None, 'dict_extra_env': None}},
'PIL': {'setup': 'pil', 'url': 'http://effbot.org/media/downloads/PIL-1.1.7.win32-py2.7.exe', 'local':"PIL-1.1.7.win32-py2.7.exe", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},
'LXML': {'setup': 'lxml', 'url': 'http://pypi.python.org/packages/2.7/l/lxml/lxml-2.3-py2.7-win32.egg', 'local':"lxml-2.3-py2.7-win32.egg", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},
'MYSQL': { 'setup': 'mysql-python', 'url': mysqlres, 'local': mysqlres, 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},
@@ -76,8 +78,8 @@
mysql_method = 'pip'
URLS.update({
- 'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.5.tar.gz', 'local':"psycopg2-2.4.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'PIL': {'setup': 'pil', 'url': 'http://effbot.org/downloads/Imaging-1.1.7.tar.gz', 'local':"Imaging-1.1.7.tar.gz", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},
+ 'PSYCOPG2': {'setup': 'psycopg2','url': 'http://initd.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.6.tar.gz', 'local':"psycopg2-2.4.6.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'PIL': {'setup': 'pil', 'url': 'https://github.com/python-imaging/Pillow/archive/1.7.8.tar.gz', 'local':"Pillow-1.7.8.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'LXML': {'setup': 'lxml', 'url':"lxml-2.3.4.tar.bz2", 'local':"lxml-2.3.4.tar.bz2", 'install': {'method': lxml_method, 'option_str': None, 'dict_extra_env': lxml_options}},
#'MYSQL': { 'setup': 'mysql-python', 'url': 'http://sourceforge.net/projects/mysql-python/files/mysql-python/1.2.3/MySQL-python-1.2.3.tar.gz/download', 'local':"MySQL-python-1.2.3.tar.gz", 'install': {'method': mysql_method, 'option_str': None, 'dict_extra_env': None}},
})
@@ -197,6 +199,7 @@
def install_lib(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):
lib_src = os.path.join(src_dir,lib_name+".tar.gz")
+ logger.notify("Copy %s to %s " % (res_env.URLS[lib_key][res_source_key],lib_src))
shutil.copy(res_env.URLS[lib_key][res_source_key], lib_src)
tf = tarfile.open(lib_src,'r:gz')
lib_base_path = os.path.join(src_dir, lib_name)
@@ -206,30 +209,27 @@
lib_src_path = os.path.join(lib_base_path, os.listdir(lib_base_path)[0])
- logger.notify(libname + " configure")
- call_subprocess(['configure', '--prefix='+os.path.abspath(home_dir)] + configure_options,
+ logger.notify(lib_name + " configure in " + lib_src_path)
+ call_subprocess(['./configure', '--prefix='+os.path.abspath(home_dir)] + configure_options,
cwd=os.path.abspath(lib_src_path),
filter_stdout=filter_python_develop,
show_stdout=True)
- logger.notify(libname + " make")
+ logger.notify(lib_name + " make in " + lib_src_path)
call_subprocess(['make'],
cwd=os.path.abspath(lib_src_path),
filter_stdout=filter_python_develop,
show_stdout=True)
- logger.notify(libname + "make install")
+ logger.notify(lib_name + "make install in " + lib_src_path)
call_subprocess(['make', 'install'],
cwd=os.path.abspath(lib_src_path),
filter_stdout=filter_python_develop,
show_stdout=True)
- return install_lib
+ return install_lib
install_libjpeg = gen_install_comp_lib("libjpeg", "LIBJPEG", ['--enable-shared'])
install_zlib = gen_install_comp_lib("zlib", "ZLIB", [])
-
-
-
def lib_generate_install_methods(path_locations, src_base, Logger, call_subprocess, normal_installs, options_to_add=None, urls= None):
@@ -327,6 +327,7 @@
extra_env['TMPDIR'] = os.path.abspath(tmp_dir)
#isinstance(lst, (list, tuple))
if key not in ignore_packages:
+ logger.notify("install %s with method %s" % (key, repr(method)))
if callable(method):
method(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop)
elif method in globals() and callable(globals()[method]) and method not in ['pip', 'easy_install']:
Binary file virtualenv/res/src/Django-1.4.2.tar.gz has changed
Binary file virtualenv/res/src/Django-1.4.5.tar.gz has changed
Binary file virtualenv/res/src/Imaging-1.1.7.tar.gz has changed
Binary file virtualenv/res/src/Pillow-1.7.8.tar.gz has changed
Binary file virtualenv/res/src/django-chunked-uploads-0.4.tar.gz has changed
Binary file virtualenv/res/src/django-extensions-0.8.tar.gz has changed
Binary file virtualenv/res/src/django-extensions-1.0.2.tar.gz has changed
Binary file virtualenv/res/src/django-haystack-2.0.0.tar.gz has changed
Binary file virtualenv/res/src/django-haystack-v2.0.0.tar.gz has changed
Binary file virtualenv/res/src/psycopg2-2.4.5.tar.gz has changed
Binary file virtualenv/res/src/psycopg2-2.4.5.win32-py2.7-pg9.1.3-release.zip has changed
Binary file virtualenv/res/src/psycopg2-2.4.6.tar.gz has changed
Binary file virtualenv/res/src/psycopg2-2.4.6.win32-py2.7-pg9.2.2-release.exe has changed
Binary file virtualenv/res/src/pyelasticsearch-0.3.tar.gz has changed
Binary file virtualenv/res/src/pyelasticsearch.tar.gz has changed
Binary file virtualenv/res/src/requests-1.1.0.tar.gz has changed
Binary file virtualenv/res/src/requests-v1.0.2.tar.gz has changed
Binary file virtualenv/res/src/simplejson-3.0.7.tar.gz has changed
Binary file virtualenv/res/src/six-1.2.0.tar.gz has changed
Binary file virtualenv/res/src/sorl-thumbnail-v10.12.1.tar.gz has changed
Binary file virtualenv/res/src/sorl-thumbnail-v11.12.tar.gz has changed
Binary file virtualenv/res/src/whoosh-2.5.tar.gz has changed
--- a/virtualenv/web/res/res_create_env.py Tue May 07 11:15:15 2013 +0200
+++ b/virtualenv/web/res/res_create_env.py Mon May 13 15:55:28 2013 +0200
@@ -27,6 +27,7 @@
'SORL_THUMBNAIL',
'HAYSTACK',
'REQUESTS',
+ 'SIMPLEJSON',
'PYELASTICSEARCH',
'WHOOSH',
'MIMEPARSE',
@@ -35,6 +36,7 @@
'PYYAML',
'PYTHON-DIGEST',
'DJANGO-TASTYPIE',
+ 'DJANGO-CHUNCKED-UPLOADS',
]
if system_str == "Linux":