virtualenv/res/lib/lib_create_env.py
author cavaliet
Fri, 12 Aug 2011 14:44:04 +0200
branchgroup_management
changeset 130 1cc949de2d1f
parent 109 b823ac24bc88
child 139 f7c6c59dff95
permissions -rw-r--r--
Useless code removed. Iri group finally removed from the model. User's templates cleaned. Little css changed for home.

import sys
import os
import os.path
import shutil
import tarfile
import zipfile
import urllib
import platform
import patch

join = os.path.join
system_str = platform.system()


URLS = {
    #'': {'setup': '', 'url':'', 'local':''},
    'DISTRIBUTE': {'setup': 'distribute', 'url':'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.14.tar.gz', 'local':"distribute-0.6.14.tar.gz"},
    'DJANGO': {'setup': 'django', 'url': 'http://www.djangoproject.com/download/1.3/tarball/', 'local':"Django-1.3.tar.gz"},
    'DJANGO-EXTENSIONS': { 'setup': 'django-extensions', 'url':'https://github.com/django-extensions/django-extensions/tarball/0.6', 'local':"django-extensions-0.6.tar.gz"},
    'DJANGO-REGISTRATION': { 'setup': 'django-registration', 'url':'http://bitbucket.org/ubernostrum/django-registration/get/tip.tar.gz', 'local':"django-registration.tar.gz"},
    'DJANGO-TAGGING': { 'setup': 'django-tagging', 'url':'http://django-tagging.googlecode.com/files/django-tagging-0.3.1.tar.gz', 'local':"django-tagging-0.3.1.tar.gz"},
    'DJANGO-PISTON': { 'setup': 'django-piston', 'url':"django-piston-0.2.2-modified.tar.gz", 'local':"django-piston-0.2.2-modified.tar.gz"},
    'OAUTH2': { 'setup': 'python-oauth2', 'url':"http://pypi.python.org/packages/source/o/oauth2/oauth2-1.5.168.tar.gz", 'local':"oauth2-1.5.167.tar.gz"},
    'HTTPLIB2': { 'setup': 'python-httplib2', 'url':'http://httplib2.googlecode.com/files/httplib2-0.6.0.tar.gz', 'local':"httplib2-0.6.0.tar.gz"},
    'DJANGO-OAUTH-PLUS': { 'setup': 'django-oauth-plus', 'url':'http://bitbucket.org/david/django-oauth-plus/get/f314f018e473.gz', 'local':"django-oauth-plus.tar.gz"},
    'MYSQL': { 'setup': 'mysql-python', 'url': 'http://sourceforge.net/projects/mysql-python/files/mysql-python/1.2.3/MySQL-python-1.2.3.tar.gz/download', 'local':"MySQL-python-1.2.3.tar.gz"},
    'SETUPTOOLS-HG': { 'setup':'mercurial_hg', 'url':'http://pypi.python.org/packages/source/s/setuptools_hg/setuptools_hg-0.2.tar.gz', 'local':"setuptools_hg-0.2.tar.gz"},
    'MERCURIAL': {'setup':'mercurial', 'url':'http://pypi.python.org/packages/source/d/mercurial/mercurial-1.7.5.tar.gz', 'local':"mercurial-1.7.5.tar.gz"},
    'OPENID': {'setup':'openid', 'url':'http://pypi.python.org/packages/source/p/python-openid/python-openid-2.2.5.tar.gz', 'local':"python-openid-2.2.5.tar.gz"},
    'DJANGO_OPENID_CONSUMER': {'setup':'django_openid_consumer', 'url':'http://pypi.python.org/packages/source/d/django-openid-consumer/django-openid-consumer-0.1.1.tar.gz', 'local':"django-openid-consumer-0.1.1.tar.gz"},
    'SOCIAL_AUTH': {'setup':'social_auth', 'url':'https://github.com/omab/django-social-auth/tarball/v0.3.10', 'local':"omab-django-social-auth-v0.3.10-modified.tar.gz"}
}

if system_str == 'Windows':
    URLS.update({
        'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.0.14.win32-py2.6-pg8.4.3-release.zip', 'local':"psycopg2-2.0.14.win32-py2.6-pg8.4.3-release.zip"},
        'JCC': {'setup': 'jcc', 'url': 'http://pylucene-win32-binary.googlecode.com/files/JCC-2.6-py2.6-win32.egg', 'local':"JCC-2.6-py2.6-win32.egg"},
        'PYLUCENE': {'setup': 'pylucene', 'url': 'http://pylucene-win32-binary.googlecode.com/files/lucene-3.0.2-py2.6-win32.egg', 'local':"lucene-3.0.2-py2.6-win32.egg"},
        'PIL': {'setup': 'pil', 'url': 'http://effbot.org/media/downloads/PIL-1.1.7.win32-py2.6.exe', 'local':"PIL-1.1.7.win32-py2.6.exe"},
        'LXML': {'setup': 'lxml', 'url': 'http://pypi.python.org/packages/2.6/l/lxml/lxml-2.2.2-py2.6-win32.egg', 'local':"lxml-2.2.2-py2.6-win32.egg"}
    })
else:
    URLS.update({
        'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.tar.gz', 'local':"psycopg2-2.4.tar.gz"},
        'PYLUCENE': {'setup': 'pylucene', 'url': 'http://apache.crihan.fr/dist//lucene/pylucene/pylucene-3.1.0-1-src.tar.gz', 'local':"pylucene-3.1.0-1-src.tar.gz"},
        'PIL': {'setup': 'pil', 'url': 'http://effbot.org/downloads/Imaging-1.1.7.tar.gz', 'local':"Imaging-1.1.7.tar.gz"},
        'LXML': {'setup': 'lxml', 'url':"lxml-2.3.tar.bz2", 'local':"lxml-2.3.tar.bz2"}
    })



class ResourcesEnv(object):

    def __init__(self, src_base, urls, normal_installs):
        self.src_base = src_base
        self.URLS = {}
        self.__init_url(urls)
        self.NORMAL_INSTALL = normal_installs

    def get_src_base_path(self, fpath):
        return os.path.abspath(os.path.join(self.src_base, fpath)).replace("\\","/")
    
    def __add_package_def(self, key, setup, url, local):
        self.URLS[key] = {'setup':setup, 'url':url, 'local':self.get_src_base_path(local)}

    def __init_url(self, urls):
        for key, url_dict in urls.items():
            url = url_dict['url']
            if not url.startswith("http://"):
                url = self.get_src_base_path(url)
            self.__add_package_def(key, url_dict["setup"], url, url_dict["local"])

def ensure_dir(dir, logger):
    if not os.path.exists(dir):
        logger.notify('Creating directory %s' % dir)
        os.makedirs(dir)

def extend_parser(parser):    
    parser.add_option(
        '--index-url',
        metavar='INDEX_URL',
        dest='index_url',
        default='http://pypi.python.org/simple/',
        help='base URL of Python Package Index')
    parser.add_option(
        '--type-install',
        metavar='type_install',
        dest='type_install',
        default='local',
        help='type install : local, url, setup')
    parser.add_option(
        '--ignore-packages',
        metavar='ignore_packages',
        dest='ignore_packages',
        default=None,
        help='list of comma separated keys for package to ignore')

def adjust_options(options, args):
    pass


def install_pylucene(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):
    
    logger.notify("Get Pylucene from %s " % res_env.URLS['PYLUCENE'][res_source_key])
    pylucene_src = os.path.join(src_dir,"pylucene.tar.gz")
    if res_source_key == 'local':
        shutil.copy(res_env.URLS['PYLUCENE'][res_source_key], pylucene_src)
    else:
        urllib.urlretrieve(res_env.URLS['PYLUCENE'][res_source_key], pylucene_src)
    tf = tarfile.open(pylucene_src,'r:gz')
    pylucene_base_path = os.path.join(src_dir,"pylucene") 
    logger.notify("Extract Pylucene to %s " % pylucene_base_path)
    tf.extractall(pylucene_base_path)
    tf.close()
    
    pylucene_src_path = os.path.join(pylucene_base_path, os.listdir(pylucene_base_path)[0])
    jcc_src_path = os.path.abspath(os.path.join(pylucene_src_path,"jcc"))
    
    #install jcc

    #patch for linux
    if system_str == 'Linux' :
        olddir = os.getcwd()
        setuptools_path = os.path.join(lib_dir, 'site-packages', 'setuptools')
        if os.path.exists(setuptools_path) and os.path.isdir(setuptools_path):
            patch_dest_path = os.path.join(lib_dir, 'site-packages')
        else:
            patch_dest_path = os.path.join(lib_dir,'site-packages','setuptools-0.6c11-py%s.%s.egg' % (sys.version_info[0], sys.version_info[1]))
            if os.path.isfile(patch_dest_path):
                # must unzip egg
                # rename file and etract all
                shutil.move(patch_dest_path, patch_dest_path + ".zip")
                zf = zipfile.ZipFile(patch_dest_path + ".zip",'r')
                zf.extractall(patch_dest_path)
                os.remove(patch_dest_path + ".zip")
        logger.notify("Patch jcc : %s " % (patch_dest_path))
        os.chdir(patch_dest_path)
        p = patch.fromfile(os.path.join(jcc_src_path,"jcc","patches","patch.43.0.6c11"))
        p.apply()
        os.chdir(olddir)

    logger.notify("Install jcc")
    call_subprocess([os.path.abspath(os.path.join(home_dir, 'bin', 'python')), 'setup.py', 'install'],
                    cwd=jcc_src_path,
                    filter_stdout=filter_python_develop,
                    show_stdout=True)
    #install pylucene
    
    logger.notify("Install pylucene")
    #modify makefile
    makefile_path = os.path.join(pylucene_src_path,"Makefile")
    logger.notify("Modify makefile %s " % makefile_path)
    shutil.move( makefile_path, makefile_path+"~" )

    destination= open( makefile_path, "w" )
    source= open( makefile_path+"~", "r" )
    destination.write("PREFIX_PYTHON="+os.path.abspath(home_dir)+"\n")
    destination.write("ANT=ant\n")
    destination.write("PYTHON=$(PREFIX_PYTHON)/bin/python\n")
    
    if system_str == "Darwin":
        if sys.version_info >= (2,6):
            destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386\n")
        else:
            destination.write("JCC=$(PYTHON) -m jcc --shared --arch x86_64 --arch i386\n")
        destination.write("NUM_FILES=2\n")
    elif system_str == "Windows":
        destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386\n")
        destination.write("NUM_FILES=2\n")
    else:
        if sys.version_info >= (2,6) and sys.version_info <= (2,7):
            destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared\n")
        else:
            destination.write("JCC=$(PYTHON) -m jcc --shared\n")
        destination.write("NUM_FILES=2\n")
    for line in source:
        destination.write( line )
    source.close()
    destination.close()
    os.remove(makefile_path+"~" )

    logger.notify("pylucene make")
    call_subprocess(['make'],
                    cwd=os.path.abspath(pylucene_src_path),
                    filter_stdout=filter_python_develop,
                    show_stdout=True)

    logger.notify("pylucene make install")
    call_subprocess(['make', 'install'],
                    cwd=os.path.abspath(pylucene_src_path),
                    filter_stdout=filter_python_develop,
                    show_stdout=True)
    

def install_psycopg2(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):
    psycopg2_src = os.path.join(src_dir,"psycopg2.zip")
    shutil.copy(res_env.URLS['PSYCOPG2'][res_source_key], psycopg2_src)
    #extract psycopg2
    zf = zipfile.ZipFile(psycopg2_src)
    psycopg2_base_path = os.path.join(src_dir,"psycopg2")
    zf.extractall(psycopg2_base_path)
    zf.close()
    
    psycopg2_src_path = os.path.join(psycopg2_base_path, os.listdir(psycopg2_base_path)[0])
    shutil.copytree(os.path.join(psycopg2_src_path, 'psycopg2'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'psycopg2')))
    shutil.copy(os.path.join(psycopg2_src_path, 'psycopg2-2.0.14-py2.6.egg-info'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'site-packages')))



def lib_generate_install_methods(path_locations, src_base, Logger, call_subprocess, normal_installs, urls=None):
    
    all_urls = URLS.copy()
    if urls is not None:
        all_urls.update(urls)
        
    res_env = ResourcesEnv(src_base, all_urls, normal_installs)

    def filter_python_develop(line):
        if not line.strip():
            return Logger.DEBUG
        for prefix in ['Searching for', 'Reading ', 'Best match: ', 'Processing ',
                       'Moving ', 'Adding ', 'running ', 'writing ', 'Creating ',
                       'creating ', 'Copying ']:
            if line.startswith(prefix):
                return Logger.DEBUG
        return Logger.NOTIFY
    
    
    def normal_install(key, method, option_str, extra_env, res_source_key, home_dir, tmp_dir, res_env, logger, call_subprocess):
        logger.notify("Install %s from %s with %s" % (key,res_env.URLS[key][res_source_key],method))
        if method == 'pip':
            if sys.platform == 'win32':
                args = [os.path.abspath(os.path.join(home_dir, 'Scripts', 'pip')), 'install', '-E', os.path.abspath(home_dir), res_env.URLS[key][res_source_key]]
            else:
                args = [os.path.abspath(os.path.join(home_dir, 'bin', 'pip')), 'install', '-E', os.path.abspath(home_dir), res_env.URLS[key][res_source_key]]
            if option_str :
                args.insert(4,option_str)
            call_subprocess(args,
                    cwd=os.path.abspath(tmp_dir),
                    filter_stdout=filter_python_develop,
                    show_stdout=True,
                    extra_env=extra_env)
        else:
            if sys.platform == 'win32':
                args = [os.path.abspath(os.path.join(home_dir, 'Scripts', 'easy_install')), res_env.URLS[key][res_source_key]]
            else:
                args = [os.path.abspath(os.path.join(home_dir, 'bin', 'easy_install')), res_env.URLS[key][res_source_key]]
            if option_str :
                args.insert(1,option_str)
            call_subprocess(args,
                    cwd=os.path.abspath(tmp_dir),
                    filter_stdout=filter_python_develop,
                    show_stdout=True,
                    extra_env=extra_env)            
 
    
    def after_install(options, home_dir):
        
        global logger
        
        verbosity = options.verbose - options.quiet
        logger = Logger([(Logger.level_for_integer(2-verbosity), sys.stdout)])

        
        home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)
        base_dir = os.path.dirname(home_dir)
        src_dir = os.path.join(home_dir, 'src')
        tmp_dir = os.path.join(home_dir, 'tmp')
        ensure_dir(src_dir, logger)
        ensure_dir(tmp_dir, logger)
        system_str = platform.system()
        
        res_source_key = options.type_install
        
        ignore_packages = []
        
        if options.ignore_packages :
            ignore_packages = options.ignore_packages.split(",")
        
        logger.indent += 2
        try:    
            for key, method, option_str, extra_env in res_env.NORMAL_INSTALL:
                if key not in ignore_packages:
                    if callable(method):
                        method(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop)
                    else:
                        normal_install(key, method, option_str, extra_env, res_source_key, home_dir, tmp_dir, res_env, logger, call_subprocess)
                            
            logger.notify("Clear source dir")
            shutil.rmtree(src_dir)
    
        finally:
            logger.indent -= 2
        script_dir = join(base_dir, bin_dir)
        logger.notify('Run "%s Package" to install new packages that provide builds'
                      % join(script_dir, 'easy_install'))
    

    return adjust_options, extend_parser, after_install