1 # -*- coding: utf-8 -*-

     2 '''

     3 Created on Feb 20, 2013

     4 

     5 @author: ymh

     6 '''

     7 # from fabric.api import run, local, env, cd, put, prefix, sudo, lcd

     8 # from fabric.colors import green

     9 # from fabric.context_managers import settings

    10 # from fabric.contrib.files import exists, upload_template

    11 # from fabric.contrib.project import rsync_project

    12 # from fabric.tasks import Task

    13 from fabric import Connection

    14 import imp

    15 import os.path

    16 import re

    17 import shutil

    18 import sys

    19 import urllib.parse

    20 import requirements

    21 

    22 

    23 # __all__ = ["check_folder_access", "migrate", "collectstatic", "do_relaunch_server",

    24 #            "export_version", "do_sync_web", "create_config", "clean_export_folder",

    25 #            "sync_install_build", "do_create_virtualenv", "clean_rsync_folder", "rsync_export",

    26 #            "do_sync_comp", "get_comp_versions_dict", "SyncComp", "get_src_version", "sync_build",

    27 #            "install_build", "do_create_virtualenv_requirement", "build_src"]

    28 

    29 def get_export_path(env, version):

    30     base_path = os.path.join(env.base_export_path,env.export_prefix).rstrip("/")

    31     return os.path.expanduser(base_path) + "_%s" % (str(version))

    32 

    33 def clean_export_folder(path):

    34     print("Removing %s" % path)

    35     if os.path.isdir(path):

    36         shutil.rmtree(path, ignore_errors=True)

    37 

    38 def do_export_version(c, path, **export_keys):

    39     print("Export version %s : %s" % (path,repr(export_keys)))

    40 

    41     env = c.env

    42 

    43     for export_key, version in export_keys.items():

    44         export_path = os.path.join(path,export_key)

    45 

    46         repo_url = env.repos[export_key]['repo']

    47         url_part = urllib.parse.urlparse(repo_url)

    48         if url_part.scheme or url_part.netloc:

    49             # this is a remote repo. Let's clone first

    50             clone_path = os.path.join(path,'clone',export_keys[export_key])

    51             os.makedirs(clone_path)

    52 

    53             output = c.run('git ls-remote \"%s\"' % repo_url, warn=True)

    54             print("OUTPUT %r" % output)

    55             scm = "hg" if output.failed else "git"

    56             if scm == "hg":

    57                 output = c.run("hg clone \"%s\" \"%s\"" % (repo_url,clone_path))

    58             else:

    59                 c.run("git clone \"%s\" \"%s\"" % (repo_url,clone_path))

    60         else:

    61             clone_path = repo_url

    62 

    63         with c.cd(clone_path):

    64             # detetct .git or .hg subfolder

    65             if os.path.exists(os.path.join(clone_path,".git")):

    66                 os.makedirs(export_path)

    67                 cmd_str = "git archive \'%s\' | tar -x -C \"%s\""

    68             else:

    69                 cmd_str = "hg archive -r \'%s\' \"%s\""

    70             c.run(cmd_str % (str(version),export_path))

    71 

    72     print("Export version %s done"%repr(export_keys))

    73 

    74 def launch_setup_command(c, command_array, path):

    75     f = None

    76     sys.path.append(path)

    77     current_path = os.getcwd()

    78     try:

    79         os.chdir(path)

    80         try:

    81             f, pathname, description = imp.find_module("setup", [path])

    82             print("launch_setup_command at %s : found setup" % path)

    83             setup_mod = imp.load_module("setup", f, pathname, description)

    84             print("launch_setup_command at %s : setup loaded" % path)

    85         except:

    86             e = sys.exc_info()[0]

    87             print("Error launching commands %s : %s" % (path, str(e)))

    88             raise

    89         finally:

    90             if f:

    91                 f.close()

    92 

    93         return setup_mod.launch_setup("setup.py", command_array)

    94     finally:

    95         os.chdir(current_path)

    96 

    97 

    98 def get_src_dependencies(c, pkg_name, path):

    99     print("Get source dependencies at %s" % path)

   100     launch_setup_command(c, ['egg_info'], path)

   101     egg_requirement_file = os.path.join(path, "%s.egg-info" % pkg_name, "requires.txt")

   102     res = []

   103     with open(egg_requirement_file) as f:

   104         for req in requirements.parse(f):

   105             if req.name in c.env['repos']:

   106                 r_version = req.specs[0][1] if req.specs else 'tip'

   107                 res.append((req.name, r_version))

   108     print("Build source dist at %s done : %r" % (path, res))

   109     return res

   110 

   111 

   112 def get_remote_env(c, remotepath, remotevirtualenvpath, application_module, settings_key, settings_module=''):

   113     if not settings_module:

   114         settings_module = '%s.%s' % (application_module, 'settings')

   115     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   116 

   117     env = c.env

   118     with Connection(env['hosts'][0]) as rconnection:

   119         with rconnection.prefix("echo $SHELL && . \"%s\"" % os.path.join(remotevirtualenvpath, "bin/activate")), rconnection.prefix("export PYTHONPATH=\"%s\"" % remotepath):

   120             return rconnection.run("DJANGO_SETTINGS_MODULE=%s python -c 'import django.conf;print(django.conf.settings.%s)'" % (settings_module, settings_key)).stdout

   121 

   122 

   123 # def rsync_export(path, remotepath, filters):

   124 #     print("Rsync %s to %s",(path,remotepath))

   125 

   126 #     filter_option_str = "--progress --stats"

   127 #     if filters:

   128 #         filter_option_str += " " + " ".join(["--filter \"%s\"" % (f) for f in filters])

   129 

   130 #     run("mkdir -p \"%s\"" % remotepath)

   131 #     rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)

   132 #     print("Rsync %s to %s done",(path,remotepath))

   133 

   134 # def clean_rsync_folder(remotepath):

   135 #     print("clean rsync folder %s" % remotepath)

   136 #     run("rm -fr \"%s\"" % remotepath)

   137 

   138 def build_src(c, path):

   139     print("Build source dist at %s" % path)

   140     launch_setup_command(c, ['sdist'], path)

   141     print("Build source dist at %s done" % path)

   142 

   143 

   144 def get_src_version(c, key, path):

   145 

   146     print("get src version for %s at %s" % (key,path))

   147 

   148     env = c.env

   149 

   150     mod_name = env.repos[key].get('module', key) or key

   151 

   152     f = None

   153     sys.path.append(path)

   154     current_path = os.getcwd()

   155     os.chdir(path)

   156     try:

   157         f, pathname, description = imp.find_module(mod_name, [path])

   158         src_mod = imp.load_module(mod_name, f, pathname, description)

   159     except:

   160         src_mod = None

   161         print("Could not import module, trying to parse")

   162     finally:

   163         os.chdir(current_path)

   164         if f:

   165             f.close()

   166     version = None

   167     if src_mod is None:

   168         with open(os.path.join(path,mod_name,"__init__.py"),'r') as init_file:

   169             for line in init_file:

   170                 m = re.search('VERSION\s+=\s+\((.+)\)', line, re.I)

   171                 if m:

   172                     version = tuple([re.sub('[\s\"\']','', item) for item in m.group(1).split(',')])

   173                     break

   174     elif hasattr(src_mod, "VERSION"):

   175         version = src_mod.VERSION

   176     elif hasattr(src_mod, "__version__"):

   177         version = src_mod.__version__

   178 

   179     print("VERSION : %s" % repr(version))

   180 

   181     if version is None:

   182         version = ""

   183 

   184     if not isinstance(version, str):

   185         if src_mod and hasattr(src_mod, "get_version"):

   186             version_str = src_mod.get_version()

   187         elif isinstance(version, tuple):

   188             #convert num

   189             version_str = get_version([int(s) if s.isdigit() else s for s in version])

   190         else:

   191             version_str = str(version)

   192     else:

   193         version_str = version

   194 

   195     print("VERSION str : %s" % repr(version_str))

   196     return (version, version_str)

   197 

   198 

   199 def sync_build(c, path):

   200     print("Sync build %s" % path)

   201     env = c.env

   202     with Connection(env['hosts'][0]) as host_connection:

   203         with host_connection.cd(env.remote_path['build_export']):

   204             filename = os.path.basename(path)

   205             res_trans = host_connection.put(path, os.path.join(env.remote_path['build_export'], filename))

   206             print("Sync build %s to %s" % (path,res_trans.remote))

   207             return res_trans

   208 

   209 

   210 def collectstatic(c, remotepath, remotevirtualenvpath, platform_web_module, module_settings="", admin_cmd="python manage.py"):

   211     print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))

   212     remotestaticsitepath = get_remote_env(c, remotepath, remotevirtualenvpath, platform_web_module, "STATIC_ROOT", c.env.settings)

   213     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   214     with Connection(c.env['hosts'][0]) as rconnection:

   215         with rconnection.prefix("source \"%s\"" % activate_path), rconnection.prefix("export PYTHONPATH=\"%s\"" % remotepath), rconnection.cd(remotepath):

   216             #remove old files optio -c of collect static fail !

   217             rconnection.run("rm -fr \"%s\"/*" % (remotestaticsitepath))

   218             rconnection.run("%s collectstatic --noinput %s" % (admin_cmd, "--settings="+module_settings if module_settings else ""))

   219 

   220 

   221 def migrate(c, remotepath, remotevirtualenvpath, module_settings="", admin_cmd="python manage.py"):

   222     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   223     with Connection(c.env['hosts'][0]) as rconnection:

   224         with rconnection.prefix("source \"%s\"" % activate_path), rconnection.prefix("export PYTHONPATH=\"%s\"" % remotepath), rconnection.cd(remotepath):

   225             rconnection.run("%s migrate --noinput %s" % (admin_cmd, "--settings="+module_settings if module_settings else ""))

   226 

   227 

   228 def export_version(c, **kwargs):

   229     print("export version %s" % (repr(kwargs)))

   230 

   231     export_path = kwargs.get('path', None)

   232 

   233     if not export_path:

   234         export_path = get_export_path(c.env, "_".join(["%s_%s" % (k,v) for k,v in kwargs.items()]))

   235 

   236     clean_export_folder(export_path)

   237 

   238     do_export_version(c, export_path,**kwargs)

   239 

   240     return export_path

   241 

   242 def do_create_virtualenv(c, remote_venv_export_path, remotevirtualenvpath):

   243     print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))

   244     env = c.env

   245     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   246     if env.get('remote_baseline_venv'):

   247         prefix_str = "source \"%s\"" % os.path.join(env.get('remote_baseline_venv'), "bin/activate")

   248     else:

   249         prefix_str = "echo"

   250     with Connection(env['hosts'][0]) as rconnection:

   251         rconnection.run("rm -fr \"%s\"" % remotevirtualenvpath, warn=True)

   252         run("mkdir -p \"%s\"" % remotevirtualenvpath)

   253         with rconnection.prefix(prefix_str), rconnection.cd(os.path.join(remote_venv_export_path,"virtualenv","web")):

   254             rconnection.run("python create_python_env.py")

   255             rconnection.run("python project-boot.py \"%s\"" % remotevirtualenvpath)

   256         with rconnection.prefix("source \"%s\"" % activate_path):

   257             rconnection.run("pip install --no-cache-dir -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))

   258 

   259 def do_create_virtualenv_requirement(c, remote_venv_requirement_path, remotevirtualenvpath, python_version = "2"):

   260     print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_requirement_path, remotevirtualenvpath))

   261     env = c.env

   262     with Connection(env['hosts'][0]) as rconnection:

   263         rconnection.run("rm -fr \"%s\"" % remotevirtualenvpath, warn=True)

   264         rconnection.run("mkdir -p \"%s\"" % remotevirtualenvpath)

   265         # rconnection.run("virtualenv -p `which python%s` %s" % (python_version, remotevirtualenvpath))

   266         rconnection.run("python%s -m venv %s" % (python_version, remotevirtualenvpath))

   267         with rconnection.prefix("echo $SHELL && . \"%s\"" % os.path.join(remotevirtualenvpath, "bin/activate")):

   268             rconnection.run("pip install -r \"%s\"" % remote_venv_requirement_path)

   269 

   270 

   271 def do_relaunch_server(c, do_collectstatic, do_migrate):

   272     env = c.env

   273 

   274     if do_migrate:

   275         migrate(c, env.remote_path['src'], env.remote_path['virtualenv'], env.get('settings', ''), env.get('admin_cmd', 'python manage.py'))

   276     if do_collectstatic:

   277         collectstatic(c, env.remote_path['src'], env.remote_path['virtualenv'], env.platform_web_module, env.get('settings', ''), env.get('admin_cmd', 'python manage.py'))

   278 

   279     with Connection(env['hosts'][0]) as rconnection:

   280         rconnection.sudo(env.web_relaunch_cmd, shell=False)

   281