1 # -*- coding: utf-8 -*-

     2 '''

     3 Created on Feb 20, 2013

     4 

     5 @author: ymh

     6 '''

     7 from fabric.api import run, local, env, cd, put, prefix, sudo, lcd

     8 from fabric.context_managers import settings

     9 from fabric.contrib.files import exists, upload_template

    10 from fabric.contrib.project import rsync_project

    11 from mercurial import commands, ui, hg, cmdutil

    12 from fabric.tasks import Task

    13 from fabric.colors import green

    14 import imp

    15 import os.path

    16 import re

    17 import shutil

    18 import urlparse

    19 

    20 

    21 __all__ = ["check_folder_access", "syncdb", "collectstatic", 

    22     "export_version", "do_sync_web", "create_config", "clean_export_folder", "relaunch_server", 

    23     "do_sync_ldt", "sync_install_build", "do_create_virtualenv", "clean_rsync_folder", "rsync_export",

    24     "SyncComp"] 

    25 

    26 def get_export_path(version):

    27     base_path = os.path.join(env.base_export_path,env.export_prefix).rstrip("/")

    28     return os.path.expanduser(base_path) + "_%s" % (str(version))

    29 

    30 def clean_export_folder(path):

    31     print("Removing %s" % path)

    32     if os.path.isdir(path):

    33         shutil.rmtree(path, ignore_errors=True)

    34 

    35 def do_export_version(path, **export_keys):

    36     print("Export version %s : %s " % (path,repr(export_keys)))

    37 

    38     for export_key, version in export_keys.items():

    39         export_path = os.path.join(path,export_key)

    40         

    41         repo_url = env.repos[export_key]

    42         url_part = urlparse.urlparse(repo_url)

    43         if url_part.scheme or url_part.netloc:

    44             # this is a remote repo. Let's clone first

    45             clone_path = os.path.join(path,'clone',export_keys)

    46             os.makedirs(clone_path)

    47             local("hg clone \"%s\" \"%s\"" % (repo_url,clone_path))

    48         else:

    49             clone_path = repo_url

    50         

    51         with lcd(clone_path):

    52             local("hg archive -r \'%s\' \"%s\"" % (str(version),export_path))

    53     

    54     print("Export version %s done"%repr(export_keys))

    55 

    56     

    57 def get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, settings_key):

    58     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

    59     with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):

    60         tempfilepath = run("mktemp -t ldtplatform.XXXXXX")

    61         with settings(warn_only=True):

    62             run("echo \"import os\" > %s" % (tempfilepath))

    63             map(lambda str_echo: run("echo \"%s\" >> %s" % (str_echo, tempfilepath)),

    64                 ["os.environ.setdefault('DJANGO_SETTINGS_MODULE', '%s.settings')" % (platform_web_module),

    65                  "from django.conf import settings",

    66                  "print settings.%s" % (settings_key)])

    67             res = run("python < %s" % (tempfilepath))

    68         run("rm -f \"%s\"" % (tempfilepath))

    69     return res

    70     

    71     

    72     

    73 def rsync_export(path, remotepath, filters):

    74     print("Rsync %s to %s",(path,remotepath))

    75     

    76     filter_option_str = "--progress --stats"

    77     if filters:

    78         filter_option_str += " " + " ".join(["--filter \"%s\"" % (f) for f in filters])

    79     

    80     run("mkdir -p \"%s\"" % remotepath)

    81     rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)

    82     print("Rsync %s to %s done",(path,remotepath))

    83     

    84 def clean_rsync_folder(remotepath):

    85     print("clean rsync folder %s" % remotepath)

    86     run("rm -fr \"%s\"" % remotepath)

    87     

    88 def build_src(path):

    89     print("Build source dist at %s" % path)

    90     f = None

    91     try:

    92         f, pathname, description = imp.find_module("setup", [path])

    93         print(" 2 Build source dist at %s" % path)

    94         setup_mod = imp.load_module("setup", f, pathname, description)

    95         print(" 3 Build source dist at %s" % path)

    96     finally:

    97         if f:

    98             f.close()

    99         

   100     setup_mod.launch_setup("setup.py", ['sdist'])

   101     

   102     print("Build source dist at %s done" % path)

   103     

   104 

   105 def get_src_version(mod_name, path):

   106     print("get src version for %s at %s" % (mod_name,path))

   107     f = None

   108     try:

   109         f, pathname, description = imp.find_module(mod_name, [path])

   110         ldt_mod = imp.load_module(mod_name, f, pathname, description)

   111     finally:

   112         if f:

   113             f.close()

   114     version = ldt_mod.VERSION

   115     version_str = ldt_mod.get_version()

   116     

   117     return (version, version_str) 

   118     

   119 

   120 def sync_build(path):

   121     print("Sync build %s" % path)

   122     with cd(env.remote_path['ldt_base']):

   123         filename = os.path.basename(path)

   124         res_trans = put(path, os.path.join(env.remote_path['ldt_base'], filename))

   125         print("Sync build %s to %s" % (path,repr(res_trans)))

   126         return res_trans

   127 

   128 def remove_build(path):

   129     print("remove build build %s" % path)

   130     run("rm \"%s\"" % path)

   131         

   132 

   133 def install_build(remotepath, remotevirtualenvpath, module_to_uninstall= None):

   134     print("Install build %s in %s" % (remotepath, remotevirtualenvpath))

   135     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   136     

   137     with prefix("source %s" % activate_path):

   138         if module_to_uninstall:

   139             with settings(warn_only=True):

   140                 run("pip uninstall -y %s" % module_to_uninstall)

   141         run("pip install \"%s\"" % remotepath)

   142 

   143 def collectstatic(remotepath, remotevirtualenvpath, platform_web_module):

   144     print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))

   145     remotestaticsitepath = get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, "STATIC_ROOT")

   146     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   147     with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):

   148         #remocve old files optio -c of collect static fail !

   149         run("rm -fr \"%s\"" % (remotestaticsitepath))

   150         run("python manage.py collectstatic --noinput")

   151 

   152 def syncdb(remotepath, remotevirtualenvpath):

   153     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   154     with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):

   155         run("python manage.py syncdb --migrate --noinput")

   156         

   157 def create_config(export_path):    

   158     print("Create config from %s" % (export_path,))

   159     remotepath = env.remote_path['src']

   160     remote_config_path = os.path.join(remotepath, env.platform_web_module, "config.py")

   161     template_path = os.path.join(export_path, "src", env.platform_web_module, "config.py.tmpl")

   162     

   163     context = {

   164         'base_dir': os.path.join(remotepath, env.platform_web_module).rstrip("/")+"/",

   165         'asctime': '%(asctime)s',

   166         'levelname': '%(levelname)s',

   167         'message': '%(message)s',

   168         'module': '%(module)s',

   169     }

   170     context.update(env.config['web'])

   171     

   172     if not exists(remote_config_path, verbose=True):

   173         upload_template(template_path, remote_config_path, context=context)

   174 

   175 def export_version(**kwargs):

   176     print("export version %s" % (repr(kwargs)))

   177     

   178     export_path = kwargs.get('path', None)

   179     

   180     if not export_path:    

   181         export_path = get_export_path("_".join(["%s_%s" % (k,v) for k,v in kwargs.items()]))

   182     

   183     clean_export_folder(export_path)

   184     

   185     do_export_version(export_path,**kwargs)

   186     

   187     return export_path

   188 

   189 def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):

   190     print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))

   191     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   192     if "remote_baseline_venv" in env and env.remote_baseline_venv:

   193         prefix_str = "source \"%s\"" % os.path.join(env.remote_baseline_venv, "bin/activate")

   194     else:

   195         prefix_str = "echo"

   196     with settings(warn_only=True):

   197         run("rm -fr \"%s\"" % remotevirtualenvpath)

   198     run("mkdir -p \"%s\"" % remotevirtualenvpath)

   199     with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):

   200         run("python create_python_env.py")

   201         run("python project-boot.py \"%s\"" % remotevirtualenvpath)

   202     with prefix("source \"%s\"" % activate_path):

   203         run("pip install -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))

   204 

   205 def do_sync_comp(key, export_path):

   206     print("do_sync_comp with  path %s" % (export_path))

   207     

   208     src_path = os.path.join(export_path,"src")

   209     # find setup.py

   210     for root, _, files in os.walk(src_path):

   211         if "setup.py" in files:

   212             src_path = root

   213     

   214     build_src(src_path)

   215     (_,version_str) = get_src_version(key, src_path)

   216     build_path = os.path.join(src_path,"dist","%s-%s.tar.gz" % (key,version_str))

   217     sync_install_build(build_path, key)

   218         

   219 

   220 def sync_install_build(build_path, module_to_uninstall=None):

   221     res_trans = None

   222     try:

   223         res_trans = sync_build(build_path)

   224         install_build(res_trans[0], env.remote_path['virtualenv'], module_to_uninstall)        

   225     finally:

   226         if res_trans:

   227             remove_build(res_trans[0])

   228 

   229 

   230 def do_sync_web(version, export_path):

   231     print("do_sync_web with version %s and path %s" % (version,export_path))

   232     #sync web

   233     web_path = os.path.join(export_path,"web/") 

   234     rsync_export(web_path, env.remote_path['web'], env.rsync_filters['web'])

   235     #sync src

   236     src_path = os.path.join(export_path,"src/") 

   237     rsync_export(src_path, env.remote_path['src'], env.rsync_filters['src'])

   238         

   239     

   240 def check_folder_access():

   241     print("Check folder access")

   242     # get remote user

   243     for folder_path in env.folders:

   244         if not os.path.isabs(folder_path):

   245             folder_path = env.remote_path['web'].rstrip("/")+ "/" + folder_path

   246             with settings(warn_only=True):

   247                 if not exists(folder_path):

   248                     run("mkdir -p \"%s\"" % folder_path)

   249                 run("chown -R -c :%s \"%s\"" % (env.web_group, folder_path))

   250                 run("chmod -R -c g+w \"%s\"" % folder_path)

   251 

   252 def get_comp_versions_dict(export_path_web):

   253     comp_versions = {}

   254     requirement_file_path = os.path.join(export_path_web, 'src', 'requirement.txt')

   255     if not os.path.exists(requirement_file_path):

   256         return comp_versions

   257     with open(requirement_file_path) as f:

   258         for line in f:

   259             m = re.match('^(\w+)\s+\(\s*\=\=\s*([\.\d\w]+)\s*\)', line)

   260             if m:

   261                 key, version_req = m.groups()

   262                 if "." in version_req:

   263                     version_req = "V" + ".".join(["%02d" % (int(s)) if s.isdigit() else s for s in version_req.split(".")])

   264                 comp_versions[key] = version_req

   265     

   266     return comp_versions

   267 

   268 def do_relaunch_server(do_collectstatic, do_syncdb):

   269     check_folder_access()

   270     if do_syncdb:

   271         syncdb(env.remote_path['src'], env.remote_path['virtualenv'])

   272     if do_collectstatic:

   273         collectstatic(env.remote_path['src'], env.remote_path['virtualenv'], env.platform_web_module)

   274     sudo(env.web_relaunch_cmd, shell=False)

   275 

   276 

   277 class SyncComp(Task):

   278     

   279     def __init__(self, key):

   280         self.key = key

   281 

   282     def __get_name(self):

   283         return "sync_" + self.key

   284     

   285     name = property(__get_name)

   286     

   287     def run(self, version):

   288         print(green("sync %s with version %s" % (self.key, version)))

   289         export_path_web = export_version(web=version)    

   290         export_path_web_full = os.path.join(export_path_web,'web')

   291         comp_versions = get_comp_versions_dict(export_path_web_full)

   292 

   293         export_path = export_version(**{self.key:comp_versions[self.key]})

   294         export_path_full = os.path.join(export_path,self.key)

   295         do_sync_comp(self.key, export_path_full)

   296         clean_export_folder(export_path)

   297         clean_export_folder(export_path_web)

   298         

   299         do_relaunch_server(do_collectstatic=True, do_syncdb=True)