17     'DISTRIBUTE': {'setup': 'distribute', 'url':'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.14.tar.gz', 'local':"distribute-0.6.14.tar.gz"},

    19     'DISTRIBUTE': {'setup': 'distribute', 'url':'http://pypi.python.org/packages/source/d/distribute/distribute-0.6.24.tar.gz', 'local':"distribute-0.6.24.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    18     'DJANGO': {'setup': 'django', 'url': 'http://www.djangoproject.com/download/1.4/tarball/', 'local':"Django-1.4.tar.gz"},

    20     'DJANGO': {'setup': 'django', 'url': 'http://www.djangoproject.com/download/1.4.1/tarball/', 'local':"Django-1.4.1.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    19     'DJANGO-EXTENSIONS': { 'setup': 'django-extensions', 'url':'https://github.com/django-extensions/django-extensions/tarball/0.6', 'local':"django-extensions-0.6.tar.gz"},

    21     'DJANGO-EXTENSIONS': { 'setup': 'django-extensions', 'url':'https://github.com/django-extensions/django-extensions/tarball/0.8', 'local':"django-extensions-0.8.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    20     'SOUTH': { 'setup': 'South', 'url':'http://www.aeracode.org/releases/south/south-0.7.3.tar.gz', 'local':"south-0.7.3.tar.gz"},

    22     'HTTPLIB2': { 'setup': 'python-httplib2', 'url':'http://code.google.com/p/httplib2/downloads/detail?name=httplib2-0.7.4.tar.gz&can=2&q=', 'local':"httplib2-0.7.4.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    21     'HTTPLIB2': { 'setup': 'python-httplib2', 'url':'http://httplib2.googlecode.com/files/httplib2-0.6.0.tar.gz', 'local':"httplib2-0.6.0.tar.gz"},

    23     'SOUTH': { 'setup': 'South', 'url':'http://www.aeracode.org/releases/south/south-0.7.5.tar.gz', 'local':"south-0.7.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    22     'HAYSTACK': { 'setup': 'haystack', 'url':'https://github.com/toastdriven/django-haystack/zipball/v1.2.7', 'local': "django-haystack-v1.2.7.tar.gz"},

    24     'PYCRYPTO': {'setup': 'pycrypto', 'url':'https://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/pycrypto-2.6.tar.gz', 'local':'pycrypto-2.6.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    23     'WHOOSH' : { 'setup': 'Whoosh', 'url': 'https://bitbucket.org/mchaput/whoosh/get/tip.tar.bz2', 'local': 'whoosh-2.3.2.tar.bz2'},

    25     'SSH': {'setup': 'ssh', 'url':'http://pypi.python.org/packages/source/s/ssh/ssh-1.7.14.tar.gz#md5=4cdd0549ef4699bd67b96264d3b21427', 'local':'ssh-1.7.14.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    24     'WIKITOOLS' : { 'setup': 'wikitools', 'url': 'wikitools.tar.bz2', 'local': 'wikitools.tar.bz2'},

    26     'FABRIC': {'setup': 'fabric', 'url':'https://github.com/fabric/fabric/tarball/1.4.2', 'local':'fabric-1.4.2.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    25     'RDFLIB' : { 'setup': 'rdflib', 'url': 'http://rdflib.googlecode.com/files/rdflib-3.2.0.tar.gz', 'local': 'rdflib-3.2.0.tar.gz'},

    27     'MERCURIAL': {'setup': 'mercurial', 'url':'http://mercurial.selenic.com/release/mercurial-2.2.3.tar.gz', 'local':'mercurial-2.2.3.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    26     'SPARQLWRAPPER' : { 'setup': 'SPARQLWrapper', 'url': 'http://downloads.sourceforge.net/project/sparql-wrapper/sparql-wrapper-python/1.5.0/SPARQLWrapper-1.5.0.tar.gz', 'local' : 'SPARQLWrapper-1.5.0.tar.gz'},

    28     'HAYSTACK': {'setup': 'django-haystack', 'url': 'https://github.com/toastdriven/django-haystack/tarball/master', 'local': 'django-haystack-v2.0.0.tar.gz', 'install':{'method':'pip', 'option_str': None, 'dict_extra_env': None}},

    29     'WHOOSH': {'setup': 'whoosh', 'url':'https://bitbucket.org/mchaput/whoosh/get/tip.tar.gz', 'local':'whoosh-2.5.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},

    30     'WIKITOOLS' : { 'setup': 'wikitools', 'url': 'wikitools.tar.bz2', 'local': 'wikitools.tar.bz2', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    31     'ISODATE' : {'setup': 'isodate', 'url': 'http://pypi.python.org/packages/source/i/isodate/isodate-0.4.8.tar.gz', 'local': 'isodate-0.4.8.tar.gz', 'install' : {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    32     'RDFLIB' : { 'setup': 'rdflib', 'url': 'http://rdflib.googlecode.com/files/rdflib-3.2.0.tar.gz', 'local': 'rdflib-3.2.0.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    33     'SPARQLWRAPPER' : { 'setup': 'SPARQLWrapper', 'url': 'http://downloads.sourceforge.net/project/sparql-wrapper/sparql-wrapper-python/1.5.0/SPARQLWrapper-1.5.0.tar.gz', 'local' : 'SPARQLWrapper-1.5.0.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},    

    31         'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.0.14.win32-py2.6-pg8.4.3-release.zip', 'local':"psycopg2-2.0.14.win32-py2.6-pg8.4.3-release.zip"},

    45         'PSYCOPG2': {'setup': 'psycopg2','url': 'psycopg2-2.0.14.win32-py2.6-pg8.4.3-release.zip', 'local':"psycopg2-2.0.14.win32-py2.6-pg8.4.3-release.zip", 'install': {'method': 'install_psycopg2', 'option_str': None, 'dict_extra_env': None}},

    32         'JCC': {'setup': 'jcc', 'url': 'http://pylucene-win32-binary.googlecode.com/files/JCC-2.6-py2.6-win32.egg', 'local':"JCC-2.6-py2.6-win32.egg"},

    46         'PIL': {'setup': 'pil', 'url': 'http://effbot.org/media/downloads/PIL-1.1.7.win32-py2.6.exe', 'local':"PIL-1.1.7.win32-py2.6.exe", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},

    33         'PYLUCENE': {'setup': 'pylucene', 'url': 'http://pylucene-win32-binary.googlecode.com/files/lucene-3.0.2-py2.6-win32.egg', 'local':"lucene-3.0.2-py2.6-win32.egg"},

    47         'LXML': {'setup': 'lxml', 'url': 'http://pypi.python.org/packages/2.6/l/lxml/lxml-2.2.2-py2.6-win32.egg', 'local':"lxml-2.2.2-py2.6-win32.egg", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},

    34         'PIL': {'setup': 'pil', 'url': 'http://effbot.org/media/downloads/PIL-1.1.7.win32-py2.6.exe', 'local':"PIL-1.1.7.win32-py2.6.exe"},

    48         'PYYAML' : { 'setup': 'PyYAML', 'url': 'http://pyyaml.org/download/pyyaml/PyYAML-3.09.win32-py2.6.exe', 'local': 'PyYAML-3.09.win32-py2.6.exe', 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},

    35         'LXML': {'setup': 'lxml', 'url': 'http://pypi.python.org/packages/2.6/l/lxml/lxml-2.2.2-py2.6-win32.egg', 'local':"lxml-2.2.2-py2.6-win32.egg"},

    36         'PYYAML' : { 'setup': 'PyYAML', 'url': 'http://pyyaml.org/download/pyyaml/PyYAML-3.09.win32-py2.6.exe', 'local': 'PyYAML-3.09.win32-py2.6.exe'},

    41         'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.tar.gz', 'local':"psycopg2-2.4.tar.gz"},

    62         'PSYCOPG2': {'setup': 'psycopg2','url': 'http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.5.tar.gz', 'local':"psycopg2-2.4.5.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    42         'PYLUCENE': {'setup': 'pylucene', 'url': 'http://apache.crihan.fr/dist//lucene/pylucene/pylucene-3.1.0-1-src.tar.gz', 'local':"pylucene-3.1.0-1-src.tar.gz"},

    63         'PIL': {'setup': 'pil', 'url': 'http://effbot.org/downloads/Imaging-1.1.7.tar.gz', 'local':"Imaging-1.1.7.tar.gz", 'install': {'method': 'easy_install', 'option_str': None, 'dict_extra_env': None}},

    43         'PIL': {'setup': 'pil', 'url': 'http://effbot.org/downloads/Imaging-1.1.7.tar.gz', 'local':"Imaging-1.1.7.tar.gz"},

    64         'LXML': {'setup': 'lxml', 'url':"lxml-2.3.4.tar.bz2", 'local':"lxml-2.3.4.tar.bz2", 'install': {'method': lxml_method, 'option_str': None, 'dict_extra_env': lxml_options}},

    44         'LXML': {'setup': 'lxml', 'url':"lxml-2.3.tar.bz2", 'local':"lxml-2.3.tar.bz2"},

    65         'PYYAML' : { 'setup': 'PyYAML', 'url': 'http://pyyaml.org/download/pyyaml/PyYAML-3.09.tar.gz', 'local': 'PyYAML-3.09.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    45         'PYYAML' : { 'setup': 'PyYAML', 'url': 'http://pyyaml.org/download/pyyaml/PyYAML-3.09.tar.gz', 'local': 'PyYAML-3.09.tar.gz'},

    47 

    67     

    48 

    61     def __add_package_def(self, key, setup, url, local):

    80     def __add_package_def(self, key, dict):

    62         self.URLS[key] = {'setup':setup, 'url':url, 'local':self.get_src_base_path(local)}

    81         self.URLS[key] = dict

    63 

    82         

    66             url = url_dict['url']

    85             url_dict_copy = url_dict.copy()

    67             if not url.startswith("http://"):

    86             if not url_dict['url'].startswith("http://"):

    68                 url = self.get_src_base_path(url)

    87                 url_dict_copy['url'] = self.get_src_base_path(url_dict['url'])

    69             self.__add_package_def(key, url_dict["setup"], url, url_dict["local"])

    88             url_dict_copy['local'] = self.get_src_base_path(url_dict['local'])            

    89             

    90             self.__add_package_def(key, url_dict_copy )

    87         default='local',

   108         help='type install : local, url, setup - default : local')

    88         help='type install : local, url, setup')

    94         help='list of comma separated keys for package to ignore')

   114         help='list of comma separated keys for package to ignore')    

    95 

    96 def adjust_options(options, args):

    97     pass

    98 

    99 

   100 def install_pylucene(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   101     

   102     logger.notify("Get Pylucene from %s " % res_env.URLS['PYLUCENE'][res_source_key])

   103     pylucene_src = os.path.join(src_dir,"pylucene.tar.gz")

   104     if res_source_key == 'local':

   105         shutil.copy(res_env.URLS['PYLUCENE'][res_source_key], pylucene_src)

   106     else:

   107         urllib.urlretrieve(res_env.URLS['PYLUCENE'][res_source_key], pylucene_src)

   108     tf = tarfile.open(pylucene_src,'r:gz')

   109     pylucene_base_path = os.path.join(src_dir,"pylucene") 

   110     logger.notify("Extract Pylucene to %s " % pylucene_base_path)

   111     tf.extractall(pylucene_base_path)

   112     tf.close()

   113     

   114     pylucene_src_path = os.path.join(pylucene_base_path, os.listdir(pylucene_base_path)[0])

   115     jcc_src_path = os.path.abspath(os.path.join(pylucene_src_path,"jcc"))

   116     

   117     #install jcc

   118 

   119     #patch for linux

   120     if system_str == 'Linux' :

   121         olddir = os.getcwd()

   122         setuptools_path = os.path.join(lib_dir, 'site-packages', 'setuptools')

   123         if os.path.exists(setuptools_path) and os.path.isdir(setuptools_path):

   124             patch_dest_path = os.path.join(lib_dir, 'site-packages')

   125         else:

   126             patch_dest_path = os.path.join(lib_dir,'site-packages','setuptools-0.6c11-py%s.%s.egg' % (sys.version_info[0], sys.version_info[1]))

   127             if os.path.isfile(patch_dest_path):

   128                 # must unzip egg

   129                 # rename file and etract all

   130                 shutil.move(patch_dest_path, patch_dest_path + ".zip")

   131                 zf = zipfile.ZipFile(patch_dest_path + ".zip",'r')

   132                 zf.extractall(patch_dest_path)

   133                 os.remove(patch_dest_path + ".zip")

   134         logger.notify("Patch jcc : %s " % (patch_dest_path))

   135         os.chdir(patch_dest_path)

   136         p = patch.fromfile(os.path.join(jcc_src_path,"jcc","patches","patch.43.0.6c11"))

   137         p.apply()

   138         os.chdir(olddir)

   139 

   140     logger.notify("Install jcc")

   141     call_subprocess([os.path.abspath(os.path.join(home_dir, 'bin', 'python')), 'setup.py', 'install'],

   142                     cwd=jcc_src_path,

   143                     filter_stdout=filter_python_develop,

   144                     show_stdout=True)

   145     #install pylucene

   146     

   147     logger.notify("Install pylucene")

   148     #modify makefile

   149     makefile_path = os.path.join(pylucene_src_path,"Makefile")

   150     logger.notify("Modify makefile %s " % makefile_path)

   151     shutil.move( makefile_path, makefile_path+"~" )

   152 

   153     destination= open( makefile_path, "w" )

   154     source= open( makefile_path+"~", "r" )

   155     destination.write("PREFIX_PYTHON="+os.path.abspath(home_dir)+"\n")

   156     destination.write("ANT=ant\n")

   157     destination.write("PYTHON=$(PREFIX_PYTHON)/bin/python\n")

   158     

   159     if system_str == "Darwin":

   160         if sys.version_info >= (2,6):

   161             destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386\n")

   162         else:

   163             destination.write("JCC=$(PYTHON) -m jcc --shared --arch x86_64 --arch i386\n")

   164         destination.write("NUM_FILES=2\n")

   165     elif system_str == "Windows":

   166         destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared --arch x86_64 --arch i386\n")

   167         destination.write("NUM_FILES=2\n")

   168     else:

   169         if sys.version_info >= (2,6) and sys.version_info <= (2,7):

   170             destination.write("JCC=$(PYTHON) -m jcc.__main__ --shared\n")

   171         else:

   172             destination.write("JCC=$(PYTHON) -m jcc --shared\n")

   173         destination.write("NUM_FILES=2\n")

   174     for line in source:

   175         destination.write( line )

   176     source.close()

   177     destination.close()

   178     os.remove(makefile_path+"~" )

   179 

   180     logger.notify("pylucene make")

   181     call_subprocess(['make'],

   182                     cwd=os.path.abspath(pylucene_src_path),

   183                     filter_stdout=filter_python_develop,

   184                     show_stdout=True)

   185 

   186     logger.notify("pylucene make install")

   187     call_subprocess(['make', 'install'],

   188                     cwd=os.path.abspath(pylucene_src_path),

   189                     filter_stdout=filter_python_develop,

   190                     show_stdout=True)

   191     

   204     shutil.copy(os.path.join(psycopg2_src_path, 'psycopg2-2.0.14-py2.6.egg-info'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages')))

   127     shutil.copy(os.path.join(psycopg2_src_path, 'psycopg2-2.0.14-py2.6.egg-info'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'site-packages')))

   128     

   129 

   130 def install_mysql(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   131     

   132     args = [os.path.abspath(os.path.join(home_dir, 'bin', 'pip')), 'install', res_env.URLS['MYSQL'][res_source_key]]                

   133     if option_str :

   134         args.insert(4,option_str)

   135     call_subprocess(args,

   136             cwd=os.path.abspath(tmp_dir),

   137             filter_stdout=filter_python_develop,

   138             show_stdout=True,

   139             extra_env=extra_env)

   140 

   141     mysqlconfig_output = []

   142     

   143     call_subprocess(['mysql_config', '--libmysqld-libs'],

   144         cwd=os.path.abspath(tmp_dir),

   145         filter_stdout=lambda line: mysqlconfig_output.append(line),

   146         show_stdout=True)

   147         

   148     mysqlconfig_output = "".join(mysqlconfig_output)

   149     m = re.search("\-L[\'\"]?([\w\/]+)[\'\"]?", mysqlconfig_output)

   150     if m:

   151         repdylibpath = m.group(1)

   152     else:

   153         repdylibpath = '/usr/local/mysql/lib'

   154         

   155     dyliblist = glob.glob(repdylibpath+"/libmysqlclient.*.dylib")

   156     def key_func(s):

   157         m = re.match(repdylibpath+"/libmysqlclient\.([\d]+)\.dylib", s)

   158         if m:

   159             return int(m.group(1))

   160         else:

   161             return sys.maxint

   162     dyliblist.sort(key=key_func)

   163     

   164     if dyliblist:

   165         dylibpath = dyliblist[0]

   166     else:

   167         dylibpath = '/usr/local/mysql/lib/libmysqlclient.18.dylib'

   168         

   169     dylibname = os.path.basename(dylibpath)    

   170     sopath = os.path.join(os.path.abspath(lib_dir), 'site-packages', '_mysql.so')

   171     

   172     call_subprocess(['install_name_tool', '-change', dylibname, dylibpath, sopath],

   173         cwd=os.path.abspath(tmp_dir),

   174         filter_stdout=filter_python_develop,

   175         show_stdout=True)

   176 

   177 

   178 def gen_install_comp_lib(lib_name, lib_key, configure_options=[]):

   179     

   180     def install_lib(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   181         lib_src = os.path.join(src_dir,lib_name+".tar.gz")

   182         shutil.copy(res_env.URLS[lib_key][res_source_key], lib_src)

   183         tf = tarfile.open(lib_src,'r:gz')

   184         lib_base_path = os.path.join(src_dir, lib_name) 

   185         logger.notify("Extract %s to %s " % (lib_name,lib_base_path))

   186         tf.extractall(lib_base_path)

   187         tf.close()

   188         

   189         lib_src_path = os.path.join(lib_base_path, os.listdir(lib_base_path)[0])

   190     

   191         logger.notify(libname + " configure")

   192         call_subprocess(['configure', '--prefix='+os.path.abspath(home_dir)] + configure_options,

   193                         cwd=os.path.abspath(lib_src_path),

   194                         filter_stdout=filter_python_develop,

   195                         show_stdout=True)

   196         

   197         logger.notify(libname + " make")

   198         call_subprocess(['make'],

   199                         cwd=os.path.abspath(lib_src_path),

   200                         filter_stdout=filter_python_develop,

   201                         show_stdout=True)

   202     

   203         logger.notify(libname + "make install")

   204         call_subprocess(['make', 'install'],

   205                         cwd=os.path.abspath(lib_src_path),

   206                         filter_stdout=filter_python_develop,

   207                         show_stdout=True)

   208         return install_lib

   209 

   210 install_libjpeg = gen_install_comp_lib("libjpeg", "LIBJPEG", ['--enable-shared'])

   211 install_zlib = gen_install_comp_lib("zlib", "ZLIB", [])

   219     

   226 

   220 #TO DO does not work in virtual env

   227 def lib_generate_install_methods(path_locations, src_base, Logger, call_subprocess, normal_installs, options_to_add=None, urls= None):

   221 def install_libyaml(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   222     logger.notify("Get libyaml from %s " % res_env.URLS['LIBYAML'][res_source_key])

   223     libyaml_src = os.path.join(src_dir,"libyaml.tar.gz")

   224     if res_source_key == 'local':

   225         shutil.copy(res_env.URLS['LIBYAML'][res_source_key], libyaml_src)

   226     else:

   227         urllib.urlretrieve(res_env.URLS['LIBYAML'][res_source_key], libyaml_src)

   228     tf = tarfile.open(pylucene_src,'r:gz')

   229     libyaml_base_path = os.path.join(src_dir,"libyaml") 

   230     logger.notify("Extract libyaml to %s " % libyaml_base_path)

   231     tf.extractall(libyaml_base_path)

   232     tf.close()

   233     

   234     libyaml_src_path = os.path.join(libyaml_base_path, os.listdir(libyaml_base_path)[0])

   235     

   236     logger.notify("libyaml configure")

   237     call_subprocess(['configure'],

   238                     cwd=os.path.abspath(libyaml_src_path),

   239                     filter_stdout=filter_python_develop,

   240                     show_stdout=True)

   241 

   242     logger.notify("libyaml make")

   243     call_subprocess(['make'],

   244                     cwd=os.path.abspath(libyaml_src_path),

   245                     filter_stdout=filter_python_develop,

   246                     show_stdout=True)

   247 

   248     logger.notify("libyaml make install")

   249     call_subprocess(['make', 'install'],

   250                     cwd=os.path.abspath(libyaml_src_path),

   251                     filter_stdout=filter_python_develop,

   252                     show_stdout=True)

   253     

   254 

   255 

   256 

   257 def lib_generate_install_methods(path_locations, src_base, Logger, call_subprocess, normal_installs, urls=None):

   319         

   289                 

   320         res_source_key = options.type_install

   290         res_source_key = getattr(options, 'type_install') if hasattr(options, 'type_install') else 'local' #.get('type_install', 'local')

   291         if res_source_key is None:

   292             res_source_key = local

   329             for key, method, option_str, extra_env in res_env.NORMAL_INSTALL:

   306             for key in res_env.NORMAL_INSTALL:

   307                 if key not in res_env.URLS:

   308                     logger.notify("%s not found in def : passing" % (key,))

   309                 install_options = res_env.URLS[key].get('install', None)

   310                 if install_options is None:

   311                     install_options = default_install_options

   312                 method = install_options.get('method', default_install_options['method'])

   313                 option_str = install_options.get('option_str', default_install_options['option_str'])

   314                 extra_env = install_options.get('dict_extra_env', default_install_options['dict_extra_env'])

   315                 #isinstance(lst, (list, tuple))