1 from fabric.api import task, run, local, env, cd, put, prefix, sudo

     2 from fabric.colors import green

     3 from fabric.contrib.project import rsync_project

     4 from fabric.contrib.files import exists, upload_template

     5 from fabric.context_managers import settings 

     6 from mercurial import commands, ui, hg, cmdutil

     7 import imp

     8 import os, os.path

     9 import shutil

    10 import sys

    11 

    12 import config

    13 

    14 def get_export_path(version):

    15     base_path = os.path.join(env.base_export_path,env.export_prefix).rstrip("/")

    16     return os.path.expanduser(base_path) + "_%s" % (str(version))

    17 

    18 def clean_export_folder(path):

    19     print("Removing %s" % path)

    20     if os.path.isdir(path):

    21         shutil.rmtree(path, ignore_errors=True)

    22 

    23 def do_export_version(path, version):

    24     print("Export version %s"%str(version))

    25         

    26     local("hg archive -r \'%s\' \"%s\"" % (str(version),path))

    27     print("Export version %s done"%str(version))

    28 

    29     

    30 def get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, settings_key):

    31     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

    32     res = ""

    33     with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):

    34         tempfilepath = run("mktemp -t ldtplatform.XXXXXX")

    35         with settings(warn_only=True):

    36             run("echo \"import os\" > %s" % (tempfilepath))

    37             map(lambda str: run("echo \"%s\" >> %s" % (str, tempfilepath)),

    38                 ["os.environ.setdefault('DJANGO_SETTINGS_MODULE', '%s.settings')" % (platform_web_module),

    39                  "from django.conf import settings",

    40                  "print settings.%s" % (settings_key)])

    41             res = run("python < %s" % (tempfilepath))

    42         run("rm -f \"%s\"" % (tempfilepath))

    43     return res

    44     

    45     

    46     

    47 def rsync_export(path, remotepath, filters):

    48     print("Rsync %s to %s",(path,remotepath))

    49     

    50     filter_option_str = "--progress --stats"

    51     if filters:

    52         filter_option_str += " " + " ".join(["--filter \"%s\"" % (f) for f in filters])

    53     

    54     run("mkdir -p \"%s\"" % remotepath)

    55     rsync_project(remotepath, local_dir=path, extra_opts=filter_option_str, delete=True)

    56     print("Rsync %s to %s done",(path,remotepath))

    57     

    58 def clean_rsync_folder(remotepath):

    59     print("clean rsync folder %s" % remotepath)

    60     run("rm -fr \"%s\"" % remotepath)

    61     

    62 def build_src(path):

    63     print("Build source dist at %s" % path)

    64     f = None

    65     try:

    66         f, pathname, description = imp.find_module("setup", [path])

    67         print(" 2 Build source dist at %s" % path)

    68         setup_mod = imp.load_module("setup", f, pathname, description)

    69         print(" 3 Build source dist at %s" % path)

    70     finally:

    71         if f:

    72             f.close()

    73         

    74     setup_mod.launch_setup("setup.py", ['sdist'])

    75     

    76     print("Build source dist at %s done" % path)

    77     

    78 

    79 def get_src_version(path):

    80     print("get src version at %s" % path)

    81     f = None

    82     try:

    83         f, pathname, description = imp.find_module("ldt", [path])

    84         ldt_mod = imp.load_module("ldt", f, pathname, description)

    85     finally:

    86         if f:

    87             f.close()

    88     version = ldt_mod.VERSION

    89     version_str = ldt_mod.get_version()

    90     

    91     return (version, version_str) 

    92     

    93 

    94 def sync_build(path):

    95     print("Sync build %s" % path)

    96     with cd(env.remote_ldt_base_path):

    97         filename = os.path.basename(path)

    98         res_trans = put(path, os.path.join(env.remote_ldt_base_path, filename))

    99         print("Sync build %s to %s" % (path,repr(res_trans)))

   100         return res_trans

   101 

   102 def remove_build(path):

   103     print("remove build build %s" % path)

   104     run("rm \"%s\"" % path)

   105         

   106 

   107 def install_build(remotepath, remotevirtualenvpath):

   108     print("Install build %s in %s" % (remotepath, remotevirtualenvpath))

   109     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   110     

   111     with prefix("source %s" % activate_path):

   112         run("pip install \"%s\"" % remotepath)

   113 

   114 def collectstatic(remotepath, remotevirtualenvpath, platform_web_module):

   115     print("Collect static in %s with %s" % (remotepath, remotevirtualenvpath))

   116     remotestaticsitepath = get_remote_env(remotepath, remotevirtualenvpath, platform_web_module, "STATIC_ROOT")

   117     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   118     with prefix("source \"%s\"" % activate_path), prefix("export PYTHONPATH=\"%s\"" % remotepath), cd(remotepath):

   119         #remocve old files optio -c of collect static fail !

   120         run("rm -fr \"%s\"" % (remotestaticsitepath))

   121         run("python manage.py collectstatic --noinput")

   122         

   123 def create_config(export_path):    

   124     print("Create config from %s" % (export_path,))

   125     remotepath = env.remote_web_path

   126     remote_config_path = os.path.join(remotepath, env.platform_web_module, "config.py")

   127     template_path = os.path.join(export_path, "web", env.platform_web_module, "config.py.tmpl")

   128     

   129     context = {

   130      'base_dir': os.path.join(remotepath, env.platform_web_module).rstrip("/")+"/",

   131      'base_url': env.base_url,

   132      'web_url': env.web_url,

   133      'stream_url': env.stream_url,

   134      'stream_src_prefix': env.stream_src_prefix,

   135      'ffmpeg_path': env.ffmpeg_path,

   136      'db_engine': env.db_engine,

   137      'db_name': env.db_name,

   138      'db_user': env.db_user,

   139      'db_password': env.db_password,

   140      'db_host': env.db_host,

   141      'db_port': env.db_port,

   142      'log_file': env.log_file,

   143      'google_analytics_code': env.google_analytics_code,

   144      'email_use_tls': env.email_use_tls,

   145      'email_host': env.email_host,

   146      'email_host_user': env.email_host_user,

   147      'email_host_user': env.email_host_user,

   148      'email_port': env.email_port,

   149      'forbidden_stream_url': env.forbidden_stream_url,

   150     }

   151     

   152     if not exists(remote_config_path, verbose=True):

   153         upload_template(template_path, remote_config_path, context=context)

   154 

   155 def export_version(version):

   156     print("export version %s" % str(version))

   157     

   158     export_path = get_export_path(version)

   159     

   160     clean_export_folder(export_path)

   161     do_export_version(export_path,version)

   162     

   163     return export_path

   164 

   165 def do_create_virtualenv(remote_venv_export_path, remotevirtualenvpath):

   166     print("Create virtualenv export_path : %s - remote venvpath : %s" % (remote_venv_export_path, remotevirtualenvpath))

   167     activate_path = os.path.join(remotevirtualenvpath, "bin/activate")

   168     if "remote_baseline_venv" in env and env.remote_baseline_venv:

   169         prefix_str = "source \"%s\"" % os.path.join(env.remote_baseline_venv, "bin/activate")

   170     else:

   171         prefix_str = "echo"

   172     with settings(warn_only=True):

   173         run("rm -fr \"%s\"" % remotevirtualenvpath)

   174     run("mkdir -p \"%s\"" % remotevirtualenvpath)

   175     with prefix(prefix_str), cd(os.path.join(remote_venv_export_path,"virtualenv","web")):

   176         run("python create_python_env.py")

   177         run("python project-boot.py \"%s\"" % remotevirtualenvpath)

   178     with prefix("source \"%s\"" % activate_path):

   179         run("pip install -r \"%s\"" % os.path.join(remote_venv_export_path,"virtualenv","web","res","srvr_requirements.txt"))

   180 

   181 def do_sync_ldt(version, export_path):

   182     print("do_sync_ldt with version %s and path %s" % (version,export_path))

   183     src_path = export_path + "/src/ldt"

   184     build_src(src_path)

   185     (_,version_str) = get_src_version(src_path)

   186     build_path = os.path.join(src_path,"dist","ldt-%s.tar.gz" % version_str)

   187     sync_install_build(build_path)

   188         

   189 

   190 def sync_install_build(build_path):

   191     res_trans = None

   192     try:

   193         res_trans = sync_build(build_path)

   194         install_build(res_trans[0], env.remote_virtualenv_path)        

   195     finally:

   196         if res_trans:

   197             remove_build(res_trans[0])

   198 

   199 

   200 def do_sync_web(version, export_path):

   201     print("do_sync_web with version %s and path %s" % (version,export_path))

   202     web_path = os.path.join(export_path,"web/") 

   203     rsync_export(web_path, env.remote_web_path, env.web_rsync_filters)    

   204     

   205 def check_folder_access():

   206     print("Check folder access")

   207     # get remote user

   208     for folder_path in env.folders:

   209         if not os.path.isabs(folder_path):

   210             folder_path = env.remote_web_path.rstrip("/")+ "/" + folder_path

   211             with settings(warn_only=True):

   212                 if not exists(folder_path):

   213                     run("mkdir -p \"%s\"" % folder_path)

   214                 run("chown -R -c :%s \"%s\"" % (env.web_group, folder_path))

   215                 run("chmod -R -c g+w \"%s\"" % folder_path)

   216 @task

   217 def relaunch_server(do_collectstatic=True):

   218     print("Relaunch server")

   219     check_folder_access()

   220     if do_collectstatic:

   221         collectstatic(env.remote_web_path, env.remote_virtualenv_path, env.platform_web_module)

   222     sudo(env.web_relaunch_cmd, shell=False)

   223 

   224 @task

   225 def sync_web(version):

   226     print(green("sync web with version %s" % version))

   227     export_path = export_version(version)

   228     do_sync_web(version, export_path)

   229     create_config(export_path)

   230     clean_export_folder(export_path)

   231     relaunch_server()

   232 

   233 @task

   234 def sync_ldt(version):

   235     print(green("sync ldt with version %s" % version))

   236     export_path = export_version(version)

   237     do_sync_ldt(version, export_path)

   238     clean_export_folder(export_path)

   239     relaunch_server()

   240     

   241 @task

   242 def update_lib(version, package):

   243     print(green("update ldt with version %s" % version))

   244     export_path = export_version(version)

   245     lib_path = os.path.join(export_path, "virtualenv", "res", "lib")

   246     

   247     f, pathname, description = imp.find_module("patch", [lib_path])

   248     patch = imp.load_module("patch", f, pathname, description)

   249     f, pathname, description = imp.find_module("lib_create_env", [lib_path])

   250     lib_create_env = imp.load_module("lib_create_env", f, pathname, description)

   251     

   252     package_path = os.path.join(export_path, "virtualenv", "res", "src", lib_create_env.URLS[package]['local'])

   253     

   254     sync_install_build(package_path)

   255     clean_export_folder(export_path)

   256     relaunch_server()

   257     

   258 @task

   259 def sync_platform(version):

   260     print(green("sync platform with version %s" % version))

   261     export_path = export_version(version)

   262     do_sync_ldt(version, export_path)

   263     do_sync_web(version, export_path)

   264     create_config(export_path)

   265     clean_export_folder(export_path)

   266     relaunch_server()

   267 

   268 @task

   269 def create_virtualenv(version):

   270     print(green("create virtualenv with version %s" % version))

   271     export_path = export_version(version)

   272     venv_remote_export_path = ""

   273     try:

   274         virtualenv_path = os.path.join(export_path, "virtualenv")

   275     

   276         venv_remote_export_path = os.path.join(env.remote_venv_export_path, env.export_prefix, version,"virtualenv")

   277         rsync_export(virtualenv_path, venv_remote_export_path, env.venv_rsync_filters)

   278         do_create_virtualenv(venv_remote_export_path, env.remote_virtualenv_path)

   279     finally:

   280         clean_export_folder(export_path)

   281         if venv_remote_export_path:

   282             clean_rsync_folder(venv_remote_export_path)