"""
Call this like ``python create_python_env.py``; it will
refresh the project-boot.py script
-prerequisite:
- virtualenv
- distribute
- psycopg2 requires the PostgreSQL libpq libraries and the pg_config utility
- python project-boot.py --distribute --no-site-packages --index-url=http://pypi.websushi.org/ --clear --type-install=local <path_to_venv>
- For Linux :
python project-boot.py --unzip-setuptools --no-site-packages --index-url=http://pypi.websushi.org/ --clear --type-install=local <path_to_venv>
"""
import os
import subprocess
import re
import sys
here = os.path.dirname(os.path.abspath(__file__))
base_dir = here
script_name = os.path.join(base_dir, 'project-boot.py')
import virtualenv
# things to install
# - psycopg2 -> pip
# - PIL -> pip
# - pyxml -> pip
# - 4Suite-xml - easy_install ftp://ftp.4suite.org/pub/4Suite/4Suite-XML-1.0.2.tar.bz2
# - pylucene - script
src_base = os.path.join(here,"res","src")
lib_path = os.path.abspath(os.path.join(here,"res","lib"))
patch_path = os.path.abspath(os.path.join(here,"res","patch"))
EXTRA_TEXT = "URLS = { \n"
EXTRA_TEXT += " 'DISTRIBUTE' : { 'setup': 'distribute', 'url': 'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.13.tar.gz', 'local': '"+ os.path.abspath(os.path.join(src_base,"distribute-0.6.13.tar.gz"))+"'},\n"
EXTRA_TEXT += " 'PSYCOPG2' : { 'setup': 'psycopg2','url': 'http://initd.org/pub/software/psycopg/psycopg2-2.2.1.tar.gz', 'local': '"+ os.path.abspath(os.path.join(src_base,"psycopg2-2.2.1.tar.gz"))+"'},\n"
EXTRA_TEXT += " 'MYSQL' : { 'setup': 'mysql-python', 'url': 'http://sourceforge.net/projects/mysql-python/files/mysql-python-test/1.2.3c1/MySQL-python-1.2.3c1.tar.gz/download', 'local' : '"+ os.path.abspath(os.path.join(src_base,"MySQL-python-1.2.3c1.tar.gz"))+"'},\n"
EXTRA_TEXT += " 'FOURSUITE_XML' : { 'setup': '4Suite-XML', 'url': 'ftp://ftp.4suite.org/pub/4Suite/4Suite-XML-1.0.2.tar.bz2', 'local': '"+ os.path.abspath(os.path.join(src_base,"4Suite-XML-1.0.2.tar.bz2"))+"'},\n"
EXTRA_TEXT += " 'PYLUCENE' : { 'setup': 'http://apache.crihan.fr/dist/lucene/pylucene/pylucene-3.0.1-1-src.tar.gz', 'url': 'http://apache.crihan.fr/dist/lucene/pylucene/pylucene-3.0.1-1-src.tar.gz', 'local': '"+ os.path.abspath(os.path.join(src_base,"pylucene-3.0.1-1-src.tar.gz"))+"'},\n"
EXTRA_TEXT += " 'PIL' : { 'setup': 'pil', 'url': 'http://effbot.org/downloads/Imaging-1.1.7.tar.gz', 'local': '"+ os.path.abspath(os.path.join(src_base,"Imaging-1.1.7.tar.gz"))+"'},\n"
EXTRA_TEXT += " 'PYXML' : { 'setup': 'http://sourceforge.net/projects/pyxml/files/pyxml/0.8.4/PyXML-0.8.4.tar.gz/download', 'url': 'http://sourceforge.net/projects/pyxml/files/pyxml/0.8.4/PyXML-0.8.4.tar.gz/download', 'local': '"+ os.path.abspath(os.path.join(src_base,"PyXML-0.8.4.tar.gz"))+"', 'patch': '"+os.path.join(patch_path,"pyxml.patch")+"'},\n"
EXTRA_TEXT += " 'DJANGO' : { 'setup': 'django', 'url': 'http://www.djangoproject.com/download/1.2.1/tarball/', 'local': '"+ os.path.abspath(os.path.join(src_base,"Django-1.2.1.tar.gz"))+"'},\n"
EXTRA_TEXT += " 'DJANGO-EXTENSIONS' : { 'setup': 'django-extensions', 'url':'http://django-command-extensions.googlecode.com/files/django-extensions-0.4.1.tar.gz', 'local':'"+ os.path.abspath(os.path.join(src_base,"django-extensions-0.4.1.tar.gz"))+"' },\n"
EXTRA_TEXT += " 'DJANGO-REGISTRATION' : { 'setup': 'django-registration', 'url':'http://bitbucket.org/ubernostrum/django-registration/get/tip.tar.gz', 'local':'"+ os.path.abspath(os.path.join(src_base,"django-registration.tar.gz"))+"' },\n"
EXTRA_TEXT += "}\n"
EXTRA_TEXT += "import sys\n"
EXTRA_TEXT += "sys.path.append('"+lib_path+"')\n"
EXTRA_TEXT += """
import shutil
import tarfile
import urllib
import platform
import patch
INDEX_URL = 'http://pypi.python.org/simple/'
def extend_parser(parser):
parser.add_option(
'--index-url',
metavar='INDEX_URL',
dest='index_url',
default='',
help='base URL of Python Package Index')
parser.add_option(
'--type-install',
metavar='type_install',
dest='type_install',
default='local',
help='type install : local, url, setup')
parser.add_option(
'--ignore-packages',
metavar='ignore_packages',
dest='ignore_packages',
default=None,
help='list of comma separated keys for package to ignore')
def adjust_options(options, args):
pass
def after_install(options, home_dir):
home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)
base_dir = os.path.dirname(home_dir)
src_dir = join(home_dir, 'src')
tmp_dir = join(home_dir, 'tmp')
ensure_dir(src_dir)
ensure_dir(tmp_dir)
system_str = platform.system()
res_source_key = options.type_install
ignore_packages = []
if options.ignore_packages :
ignore_packages = options.ignore_packages.split(",")
logger.indent += 2
try:
if 'PYLUCENE' not in ignore_packages:
#get pylucene
logger.notify("Get Pylucene from %s " % URLS['PYLUCENE'][res_source_key])
pylucene_src = os.path.join(src_dir,"pylucene.tar.gz")
urllib.urlretrieve(URLS['PYLUCENE'][res_source_key], pylucene_src)
tf = tarfile.open(pylucene_src,'r:gz')
pylucene_base_path = os.path.join(src_dir,"pylucene")
logger.notify("Extract Pylucene to %s " % pylucene_base_path)
tf.extractall(pylucene_base_path)
tf.close()
pylucene_src_path = os.path.join(pylucene_base_path, os.listdir(pylucene_base_path)[0])
jcc_src_path = os.path.abspath(os.path.join(pylucene_src_path,"jcc"))
#install jcc
#patch for linux
if system_str == 'Linux' :
olddir = os.getcwd()
patch_dest_path = os.path.join(lib_dir,'site-packages','setuptools-0.6c11-py'+'%s.%s' % (sys.version_info[0], sys.version_info[1])+'.egg')
logger.notify("Patch jcc : %s " % (patch_dest_path))
os.chdir(patch_dest_path)
p = patch.fromfile(os.path.join(jcc_src_path,"jcc","patches","patch.43.0.6c11"))
p.apply()
os.chdir(olddir)
logger.notify("Install jcc")
call_subprocess([os.path.abspath(os.path.join(home_dir, 'bin', 'python')), 'setup.py', 'install'],
cwd=jcc_src_path,
filter_stdout=filter_python_develop,
show_stdout=True)
#install pylucene
logger.notify("Install pylucene")
#modify makefile
makefile_path = os.path.join(pylucene_src_path,"Makefile")
logger.notify("Modify makefile %s " % makefile_path)
shutil.move( makefile_path, makefile_path+"~" )
destination= open( makefile_path, "w" )
source= open( makefile_path+"~", "r" )
destination.write("PREFIX_PYTHON="+os.path.abspath(home_dir)+"\\n")
destination.write("ANT=ant\\n")
destination.write("PYTHON=$(PREFIX_PYTHON)/bin/python\\n")
if system_str == "Darwin":
if sys.version_info >= (2,6):
destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386\\n")
else:
destination.write("JCC=$(PYTHON) -m jcc --shared --arch x86_64 --arch i386\\n")
destination.write("NUM_FILES=2\\n")
elif system_str == "Windows":
destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386\\n")
destination.write("NUM_FILES=2\\n")
else:
destination.write("JCC=$(PYTHON) -m jcc --shared\\n")
destination.write("NUM_FILES=2\\n")
for line in source:
destination.write( line )
source.close()
destination.close()
os.remove(makefile_path+"~" )
logger.notify("pylucene make")
call_subprocess(['make'],
cwd=os.path.abspath(pylucene_src_path),
filter_stdout=filter_python_develop,
show_stdout=True)
logger.notify("pylucene make install")
call_subprocess(['make', 'install'],
cwd=os.path.abspath(pylucene_src_path),
filter_stdout=filter_python_develop,
show_stdout=True)
if system_str == 'Linux' and 'DISTRIBUTE' not in ignore_packages:
normal_install('DISTRIBUTE', 'pip', None, res_source_key, home_dir, tmp_dir)
if 'PYXML' not in ignore_packages:
logger.notify("PyXML install : %s " % URLS['PYXML'][res_source_key])
if sys.version_info >= (2,6):
logger.notify("PyXML -> python version >= 2.6 : patching")
pyxml_src = os.path.join(src_dir,"pyxml.tar.gz")
urllib.urlretrieve(URLS['PYXML'][res_source_key], pyxml_src)
logger.notify("PyXML -> python version >= 2.6 : extract archive")
tf = tarfile.open(pyxml_src,'r:gz')
pyxml_base_path = os.path.join(src_dir,"pyxml")
tf.extractall(pyxml_base_path)
tf.close()
#patch
pyxml_version = os.listdir(pyxml_base_path)[0]
pyxml_path = os.path.join(pyxml_base_path, pyxml_version)
olddir = os.getcwd()
os.chdir(pyxml_path)
logger.notify("PyXML -> python version >= 2.6 : do patch %s : %s " % (pyxml_path, URLS['PYXML']['patch']))
p = patch.fromfile(URLS['PYXML']['patch'])
p.apply()
os.chdir(olddir)
logger.notify("PyXML -> python version >= 2.6 : install")
call_subprocess([os.path.abspath(os.path.join(home_dir, 'bin', 'pip')), 'install', '-E', os.path.abspath(home_dir), '--build='+os.path.abspath(pyxml_base_path), '--no-download', pyxml_version],
cwd=os.path.abspath(tmp_dir),
filter_stdout=filter_python_develop,
show_stdout=True)
else:
call_subprocess([os.path.abspath(os.path.join(home_dir, 'bin', 'pip')), 'install', '-E', os.path.abspath(home_dir), URLS['PYXML'][res_source_key]],
cwd=os.path.abspath(tmp_dir),
filter_stdout=filter_python_develop,
show_stdout=True)
NORMAL_INSTALL = [ #(key,method)
('PSYCOPG2', 'pip', None),
('MYSQL', 'pip', None),
('PIL', 'pip', None),
('FOURSUITE_XML','easy_install', None),
('DJANGO','pip', None),
('DJANGO-EXTENSIONS', 'pip', None),
('DJANGO-REGISTRATION', 'easy_install', '-Z')
]
for key, method, option_str in NORMAL_INSTALL:
if key not in ignore_packages:
normal_install(key, method, option_str, res_source_key, home_dir, tmp_dir)
logger.notify("Clear source dir")
shutil.rmtree(src_dir)
finally:
logger.indent -= 2
script_dir = join(base_dir, 'bin')
logger.notify('Run "%s Package" to install new packages that provide builds'
% join(script_dir, 'easy_install'))
def normal_install(key, method, option_str, res_source_key, home_dir, tmp_dir):
logger.notify("Install %s from %s with %s" % (key,URLS[key][res_source_key],method))
if method == 'pip':
args = [os.path.abspath(os.path.join(home_dir, 'bin', 'pip')), 'install', '-E', os.path.abspath(home_dir), URLS[key][res_source_key]]
if option_str :
args.insert(4,option_str)
call_subprocess(args,
cwd=os.path.abspath(tmp_dir),
filter_stdout=filter_python_develop,
show_stdout=True)
else:
args = [os.path.abspath(os.path.join(home_dir, 'bin', 'easy_install')), URLS[key][res_source_key]]
if option_str :
args.insert(1,option_str)
call_subprocess(args,
cwd=os.path.abspath(tmp_dir),
filter_stdout=filter_python_develop,
show_stdout=True)
def ensure_dir(dir):
if not os.path.exists(dir):
logger.notify('Creating directory %s' % dir)
os.makedirs(dir)
def filter_python_develop(line):
if not line.strip():
return Logger.DEBUG
for prefix in ['Searching for', 'Reading ', 'Best match: ', 'Processing ',
'Moving ', 'Adding ', 'running ', 'writing ', 'Creating ',
'creating ', 'Copying ']:
if line.startswith(prefix):
return Logger.DEBUG
return Logger.NOTIFY
"""
def main():
python_version = ".".join(map(str,sys.version_info[0:2]))
text = virtualenv.create_bootstrap_script(EXTRA_TEXT, python_version=python_version)
if os.path.exists(script_name):
f = open(script_name)
cur_text = f.read()
f.close()
else:
cur_text = ''
print 'Updating %s' % script_name
if cur_text == 'text':
print 'No update'
else:
print 'Script changed; updating...'
f = open(script_name, 'w')
f.write(text)
f.close()
if __name__ == '__main__':
main()