13 

    13 

    14 join = os.path.join

    14 join = os.path.join

    15 system_str = platform.system()

    15 system_str = platform.system()

    16 

    16 

    17 URLS = {

    17 URLS = {

    19     'ANYJSON': {'setup': 'anyjson','url': 'https://bitbucket.org/runeh/anyjson/get/0.3.3.tar.gz', 'local':"anyjson-0.3.3.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    19     'ANYJSON': {'setup': 'anyjson','url': 'https://bitbucket.org/runeh/anyjson/get/0.3.3.tar.gz', 'local':"anyjson-0.3.3.tar.gz", 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},

    36 }

    40 }

    37 

    41 

    39     lxml_options = {'STATIC_DEPS': 'true', 'LIBXML2_VERSION': '2.9.1', 'LIBXSLT_VERSION': '1.1.28', 'LIBICONV_VERSION': '1.14'}

    43     lxml_options = {'STATIC_DEPS': 'true', 'LIBXML2_VERSION': '2.9.1', 'LIBXSLT_VERSION': '1.1.28', 'LIBICONV_VERSION': '1.14'}

    40 else:

    44 else:

    41     lxml_options = None

    45     lxml_options = None

    42 

    46 

    43 URLS.update({

    47 URLS.update({

    45 })

    49 })

    46 

    50 

    47 

    51 

    48 class ResourcesEnv(object):

    52 class ResourcesEnv(object):

    49 

    53 

    56 

    60 

    57     def get_src_base_path(self, fpath):

    61     def get_src_base_path(self, fpath):

    58         return os.path.abspath(os.path.join(self.src_base, fpath)).replace("\\","/")

    62         return os.path.abspath(os.path.join(self.src_base, fpath)).replace("\\","/")

    59 

    63 

    60     def get_run_res_base_path(self, fpath):

    64     def get_run_res_base_path(self, fpath):

    63     def __add_package_def(self, key, dict):

    67     def __add_package_def(self, key, dict):

    64         self.URLS[key] = dict

    68         self.URLS[key] = dict

    66     def __init_url(self, urls):

    70     def __init_url(self, urls):

    67         for key, url_dict in urls.items():

    71         for key, url_dict in urls.items():

    68             url_dict_copy = url_dict.copy()

    72             url_dict_copy = url_dict.copy()

    69             if url_dict.get('install', {}).get('method','pip') == 'pip-req':

    73             if url_dict.get('install', {}).get('method','pip') == 'pip-req':

    70                 get_base_path = self.get_run_res_base_path

    74                 get_base_path = self.get_run_res_base_path

    71             else:

    75             else:

    73             if not url_dict['url'].startswith("http://"):

    77             if not url_dict['url'].startswith("http://"):

    74                 url_dict_copy['url'] = get_base_path(url_dict['url'])

    78                 url_dict_copy['url'] = get_base_path(url_dict['url'])

    75             url_dict_copy['local'] = get_base_path(url_dict['local'])

    79             url_dict_copy['local'] = get_base_path(url_dict['local'])

    76 

    80 

    77             self.__add_package_def(key, url_dict_copy )

    81             self.__add_package_def(key, url_dict_copy )

    80     logger.notify('Check directory %s' % dir)

    84     logger.notify('Check directory %s' % dir)

    81     if not os.path.exists(dir):

    85     if not os.path.exists(dir):

    82         logger.notify('Creating directory %s' % dir)

    86         logger.notify('Creating directory %s' % dir)

    83         os.makedirs(dir)

    87         os.makedirs(dir)

    84 

    88 

    86     parser.add_option(

    90     parser.add_option(

    87         '--index-url',

    91         '--index-url',

    88         metavar='INDEX_URL',

    92         metavar='INDEX_URL',

    89         dest='index_url',

    93         dest='index_url',

    90         default='http://pypi.python.org/simple/',

    94         default='http://pypi.python.org/simple/',

    97     parser.add_option(

   101     parser.add_option(

    98         '--ignore-packages',

   102         '--ignore-packages',

    99         metavar='ignore_packages',

   103         metavar='ignore_packages',

   100         dest='ignore_packages',

   104         dest='ignore_packages',

   101         default=None,

   105         default=None,

   103 

   107 

   104 def install_psycopg2(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   108 def install_psycopg2(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   105     psycopg2_src = os.path.join(src_dir,"psycopg2.zip")

   109     psycopg2_src = os.path.join(src_dir,"psycopg2.zip")

   106     shutil.copy(res_env.URLS['PSYCOPG2'][res_source_key], psycopg2_src)

   110     shutil.copy(res_env.URLS['PSYCOPG2'][res_source_key], psycopg2_src)

   107     #extract psycopg2

   111     #extract psycopg2

   108     zf = zipfile.ZipFile(psycopg2_src)

   112     zf = zipfile.ZipFile(psycopg2_src)

   109     psycopg2_base_path = os.path.join(src_dir,"psycopg2")

   113     psycopg2_base_path = os.path.join(src_dir,"psycopg2")

   110     zf.extractall(psycopg2_base_path)

   114     zf.extractall(psycopg2_base_path)

   111     zf.close()

   115     zf.close()

   113     psycopg2_src_path = os.path.join(psycopg2_base_path, os.listdir(psycopg2_base_path)[0])

   117     psycopg2_src_path = os.path.join(psycopg2_base_path, os.listdir(psycopg2_base_path)[0])

   114     shutil.copytree(os.path.join(psycopg2_src_path, 'psycopg2'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'psycopg2')))

   118     shutil.copytree(os.path.join(psycopg2_src_path, 'psycopg2'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'psycopg2')))

   115     shutil.copy(os.path.join(psycopg2_src_path, 'psycopg2-2.4.5-py2.7.egg-info'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'site-packages')))

   119     shutil.copy(os.path.join(psycopg2_src_path, 'psycopg2-2.4.5-py2.7.egg-info'), os.path.abspath(os.path.join(home_dir, 'Lib/site-packages', 'site-packages')))

   117 

   121 

   118 def install_mysql(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   122 def install_mysql(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   121     if option_str :

   125     if option_str :

   122         args.insert(4,option_str)

   126         args.insert(4,option_str)

   123     call_subprocess(args,

   127     call_subprocess(args,

   124             cwd=os.path.abspath(tmp_dir),

   128             cwd=os.path.abspath(tmp_dir),

   125             filter_stdout=filter_python_develop,

   129             filter_stdout=filter_python_develop,

   126             show_stdout=True,

   130             show_stdout=True,

   127             extra_env=extra_env)

   131             extra_env=extra_env)

   128 

   132 

   129     mysqlconfig_output = []

   133     mysqlconfig_output = []

   131     call_subprocess(['mysql_config', '--libmysqld-libs'],

   135     call_subprocess(['mysql_config', '--libmysqld-libs'],

   132         cwd=os.path.abspath(tmp_dir),

   136         cwd=os.path.abspath(tmp_dir),

   133         filter_stdout=lambda line: mysqlconfig_output.append(line),

   137         filter_stdout=lambda line: mysqlconfig_output.append(line),

   134         show_stdout=True)

   138         show_stdout=True)

   136     mysqlconfig_output = "".join(mysqlconfig_output)

   140     mysqlconfig_output = "".join(mysqlconfig_output)

   137     m = re.search("\-L[\'\"]?([\w\/]+)[\'\"]?", mysqlconfig_output)

   141     m = re.search("\-L[\'\"]?([\w\/]+)[\'\"]?", mysqlconfig_output)

   138     if m:

   142     if m:

   139         repdylibpath = m.group(1)

   143         repdylibpath = m.group(1)

   140     else:

   144     else:

   141         repdylibpath = '/usr/local/mysql/lib'

   145         repdylibpath = '/usr/local/mysql/lib'

   143     dyliblist = glob.glob(repdylibpath+"/libmysqlclient.*.dylib")

   147     dyliblist = glob.glob(repdylibpath+"/libmysqlclient.*.dylib")

   144     def key_func(s):

   148     def key_func(s):

   145         m = re.match(repdylibpath+"/libmysqlclient\.([\d]+)\.dylib", s)

   149         m = re.match(repdylibpath+"/libmysqlclient\.([\d]+)\.dylib", s)

   146         if m:

   150         if m:

   147             return int(m.group(1))

   151             return int(m.group(1))

   148         else:

   152         else:

   149             return sys.maxint

   153             return sys.maxint

   150     dyliblist.sort(key=key_func)

   154     dyliblist.sort(key=key_func)

   152     if dyliblist:

   156     if dyliblist:

   153         dylibpath = dyliblist[0]

   157         dylibpath = dyliblist[0]

   154     else:

   158     else:

   155         dylibpath = '/usr/local/mysql/lib/libmysqlclient.18.dylib'

   159         dylibpath = '/usr/local/mysql/lib/libmysqlclient.18.dylib'

   158     sopath = os.path.join(os.path.abspath(lib_dir), 'site-packages', '_mysql.so')

   162     sopath = os.path.join(os.path.abspath(lib_dir), 'site-packages', '_mysql.so')

   160     call_subprocess(['install_name_tool', '-change', dylibname, dylibpath, sopath],

   164     call_subprocess(['install_name_tool', '-change', dylibname, dylibpath, sopath],

   161         cwd=os.path.abspath(tmp_dir),

   165         cwd=os.path.abspath(tmp_dir),

   162         filter_stdout=filter_python_develop,

   166         filter_stdout=filter_python_develop,

   163         show_stdout=True)

   167         show_stdout=True)

   164 

   168 

   165 

   169 

   166 def gen_install_comp_lib(lib_name, lib_key, configure_options=[]):

   170 def gen_install_comp_lib(lib_name, lib_key, configure_options=[]):

   168     def install_lib(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   172     def install_lib(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop):

   169         lib_src = os.path.join(src_dir,lib_name+".tar.gz")

   173         lib_src = os.path.join(src_dir,lib_name+".tar.gz")

   170         logger.notify("Copy %s to %s " % (res_env.URLS[lib_key][res_source_key],lib_src))

   174         logger.notify("Copy %s to %s " % (res_env.URLS[lib_key][res_source_key],lib_src))

   171         shutil.copy(res_env.URLS[lib_key][res_source_key], lib_src)

   175         shutil.copy(res_env.URLS[lib_key][res_source_key], lib_src)

   172         tf = tarfile.open(lib_src,'r:gz')

   176         tf = tarfile.open(lib_src,'r:gz')

   174         logger.notify("Extract %s to %s " % (lib_name,lib_base_path))

   178         logger.notify("Extract %s to %s " % (lib_name,lib_base_path))

   175         tf.extractall(lib_base_path)

   179         tf.extractall(lib_base_path)

   176         tf.close()

   180         tf.close()

   178         lib_src_path = os.path.join(lib_base_path, os.listdir(lib_base_path)[0])

   182         lib_src_path = os.path.join(lib_base_path, os.listdir(lib_base_path)[0])

   180         logger.notify(lib_name + " configure in " + lib_src_path)

   184         logger.notify(lib_name + " configure in " + lib_src_path)

   181         call_subprocess(['./configure', '--prefix='+os.path.abspath(home_dir)] + configure_options,

   185         call_subprocess(['./configure', '--prefix='+os.path.abspath(home_dir)] + configure_options,

   182                         cwd=os.path.abspath(lib_src_path),

   186                         cwd=os.path.abspath(lib_src_path),

   183                         filter_stdout=filter_python_develop,

   187                         filter_stdout=filter_python_develop,

   184                         show_stdout=True)

   188                         show_stdout=True)

   186         logger.notify(lib_name + " make in " + lib_src_path)

   190         logger.notify(lib_name + " make in " + lib_src_path)

   187         call_subprocess(['make'],

   191         call_subprocess(['make'],

   188                         cwd=os.path.abspath(lib_src_path),

   192                         cwd=os.path.abspath(lib_src_path),

   189                         filter_stdout=filter_python_develop,

   193                         filter_stdout=filter_python_develop,

   190                         show_stdout=True)

   194                         show_stdout=True)

   192         logger.notify(lib_name + "make install in " + lib_src_path)

   196         logger.notify(lib_name + "make install in " + lib_src_path)

   193         call_subprocess(['make', 'install'],

   197         call_subprocess(['make', 'install'],

   194                         cwd=os.path.abspath(lib_src_path),

   198                         cwd=os.path.abspath(lib_src_path),

   195                         filter_stdout=filter_python_develop,

   199                         filter_stdout=filter_python_develop,

   196                         show_stdout=True)

   200                         show_stdout=True)

   197     return install_lib

   201     return install_lib

   198 

   202 

   199 install_libjpeg = gen_install_comp_lib("libjpeg", "LIBJPEG", ['--enable-shared'])

   203 install_libjpeg = gen_install_comp_lib("libjpeg", "LIBJPEG", ['--enable-shared'])

   200 install_zlib = gen_install_comp_lib("zlib", "ZLIB", [])

   204 install_zlib = gen_install_comp_lib("zlib", "ZLIB", [])

   202 

   206 

   203 def lib_generate_install_methods(path_locations, src_base, run_base, Logger, call_subprocess, normal_installs, options_to_add=None, urls= None):

   207 def lib_generate_install_methods(path_locations, src_base, run_base, Logger, call_subprocess, normal_installs, options_to_add=None, urls= None):

   205     all_urls = URLS.copy()

   209     all_urls = URLS.copy()

   206     if urls is not None:

   210     if urls is not None:

   207         all_urls.update(urls)

   211         all_urls.update(urls)

   209     res_env = ResourcesEnv(src_base, run_base, all_urls, normal_installs)

   213     res_env = ResourcesEnv(src_base, run_base, all_urls, normal_installs)

   210 

   214 

   211     def filter_python_develop(line):

   215     def filter_python_develop(line):

   212         if not line.strip():

   216         if not line.strip():

   213             return Logger.DEBUG

   217             return Logger.DEBUG

   215                        'Moving ', 'Adding ', 'running ', 'writing ', 'Creating ',

   219                        'Moving ', 'Adding ', 'running ', 'writing ', 'Creating ',

   216                        'creating ', 'Copying ']:

   220                        'creating ', 'Copying ']:

   217             if line.startswith(prefix):

   221             if line.startswith(prefix):

   218                 return Logger.DEBUG

   222                 return Logger.DEBUG

   219         return Logger.NOTIFY

   223         return Logger.NOTIFY

   222     def normal_install(key, res_path, method, option_str, extra_env, res_source_key, home_dir, tmp_dir, res_env, logger, call_subprocess):

   226     def normal_install(key, res_path, method, option_str, extra_env, res_source_key, home_dir, tmp_dir, res_env, logger, call_subprocess):

   223         logger.notify("Install %s from %s with %s" % (key,res_path,method))

   227         logger.notify("Install %s from %s with %s" % (key,res_path,method))

   224         if method == 'pip':

   228         if method == 'pip':

   225             if sys.platform == 'win32':

   229             if sys.platform == 'win32':

   226                 args = [os.path.abspath(os.path.join(home_dir, 'Scripts', 'pip')), 'install', res_path]

   230                 args = [os.path.abspath(os.path.join(home_dir, 'Scripts', 'pip')), 'install', res_path]

   270                 args.insert(1,option_str)

   274                 args.insert(1,option_str)

   271             call_subprocess(args,

   275             call_subprocess(args,

   272                     cwd=os.path.abspath(tmp_dir),

   276                     cwd=os.path.abspath(tmp_dir),

   273                     filter_stdout=filter_python_develop,

   277                     filter_stdout=filter_python_develop,

   274                     show_stdout=True,

   278                     show_stdout=True,

   278     def after_install(options, home_dir):

   282     def after_install(options, home_dir):

   280         global logger

   284         global logger

   282         verbosity = options.verbose - options.quiet

   286         verbosity = options.verbose - options.quiet

   283         print(repr(Logger))

   287         print(repr(Logger))

   284         logger = Logger([(Logger.level_for_integer(2-verbosity), sys.stdout)])

   288         logger = Logger([(Logger.level_for_integer(2-verbosity), sys.stdout)])

   285 

   289 

   287         home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)

   291         home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)

   288         base_dir = os.path.dirname(home_dir)

   292         base_dir = os.path.dirname(home_dir)

   289         src_dir = os.path.join(home_dir, 'src')

   293         src_dir = os.path.join(home_dir, 'src')

   290         tmp_dir = os.path.join(home_dir, 'tmp')

   294         tmp_dir = os.path.join(home_dir, 'tmp')

   291         ensure_dir(src_dir, logger)

   295         ensure_dir(src_dir, logger)

   292         ensure_dir(tmp_dir, logger)

   296         ensure_dir(tmp_dir, logger)

   293         system_str = platform.system()

   297         system_str = platform.system()

   295         res_source_key = getattr(options, 'type_install') if hasattr(options, 'type_install') else 'local' #.get('type_install', 'local')

   299         res_source_key = getattr(options, 'type_install') if hasattr(options, 'type_install') else 'local' #.get('type_install', 'local')

   296         if res_source_key is None:

   300         if res_source_key is None:

   297             res_source_key = 'local'

   301             res_source_key = 'local'

   299         ignore_packages = []

   303         ignore_packages = []

   301         if system_str == 'Windows':

   305         if system_str == 'Windows':

   302             default_install_options = {'method': 'easy_install', 'option_str': None, 'dict_extra_env': {}}

   306             default_install_options = {'method': 'easy_install', 'option_str': None, 'dict_extra_env': {}}

   303         else:

   307         else:

   304             default_install_options = {'method': 'pip', 'option_str': None, 'dict_extra_env': {}}

   308             default_install_options = {'method': 'pip', 'option_str': None, 'dict_extra_env': {}}

   306         if options.ignore_packages :

   310         if options.ignore_packages :

   307             ignore_packages = options.ignore_packages.split(",")

   311             ignore_packages = options.ignore_packages.split(",")

   309         logger.indent += 2

   313         logger.indent += 2

   311             for key in res_env.NORMAL_INSTALL:

   315             for key in res_env.NORMAL_INSTALL:

   312                 install_options = None

   316                 install_options = None

   313                 if isinstance(key, dict):

   317                 if isinstance(key, dict):

   314                     install_options = key.get('install', default_install_options)

   318                     install_options = key.get('install', default_install_options)

   315                     install_options['method'] = 'pip-req'

   319                     install_options['method'] = 'pip-req'

   324                 method = install_options.get('method', default_install_options['method'])

   328                 method = install_options.get('method', default_install_options['method'])

   325                 option_str = install_options.get('option_str', default_install_options['option_str'])

   329                 option_str = install_options.get('option_str', default_install_options['option_str'])

   326                 extra_env = install_options.get('dict_extra_env', default_install_options['dict_extra_env'])

   330                 extra_env = install_options.get('dict_extra_env', default_install_options['dict_extra_env'])

   327                 if not extra_env:

   331                 if not extra_env:

   328                     extra_env = {}

   332                     extra_env = {}

   330                 if 'TMPDIR' not in extra_env:

   334                 if 'TMPDIR' not in extra_env:

   332                 #isinstance(lst, (list, tuple))

   336                 #isinstance(lst, (list, tuple))

   333                 if key not in ignore_packages:

   337                 if key not in ignore_packages:

   334                     logger.notify("install %s with method %s" % (key, repr(method)))

   338                     logger.notify("install %s with method %s" % (key, repr(method)))

   335                     if callable(method):

   339                     if callable(method):

   336                         method(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop)

   340                         method(option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop)

   338                         globals()[method](option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop)

   342                         globals()[method](option_str, extra_env, res_source_key, home_dir, lib_dir, tmp_dir, src_dir, res_env, logger, call_subprocess, filter_python_develop)

   339                     else:

   343                     else:

   340                         normal_install(key, res_path, method, option_str, extra_env, res_source_key, home_dir, tmp_dir, res_env, logger, call_subprocess)

   344                         normal_install(key, res_path, method, option_str, extra_env, res_source_key, home_dir, tmp_dir, res_env, logger, call_subprocess)

   342             logger.notify("Clear source dir")

   346             logger.notify("Clear source dir")

   343             shutil.rmtree(src_dir)

   347             shutil.rmtree(src_dir)

   345         finally:

   349         finally:

   346             logger.indent -= 2

   350             logger.indent -= 2

   347         script_dir = join(base_dir, bin_dir)

   351         script_dir = join(base_dir, bin_dir)

   348         logger.notify('Run "%s Package" to install new packages that provide builds'

   352         logger.notify('Run "%s Package" to install new packages that provide builds'

   349                       % join(script_dir, 'easy_install'))

   353                       % join(script_dir, 'easy_install'))

   351     def adjust_options(options, args):

   355     def adjust_options(options, args):

   352         if not options_to_add:

   356         if not options_to_add:

   353             return

   357             return

   354         for opt in options_to_add:

   358         for opt in options_to_add:

   355             test_opt = opt.split('=',1)[0]

   359             test_opt = opt.split('=',1)[0]