make hdalab work locally
authorymh <ymh.work@gmail.com>
Wed, 17 Jul 2024 22:46:52 +0200
changeset 702 2a6e667b1610
parent 691 8454f9cda0ca
child 703 a988e44c92d5
make hdalab work locally
.env.tmpl
.hgignore
docker-compose.yaml
docker/server/config.py
docker/server/hdalab.yml
docker/server/requirements.txt
docker/server/server.dockerfile
docker/web/Caddyfile
docker/web/web.dockerfile
src/hdalab/static/hdalab/js/gomina.js
src/hdalab/views/ajax.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.env.tmpl	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,8 @@
+ADMIN_EMAIL=
+POSTGRES_PASSWORD=
+DJANGO_SECRET=
+BROKER_PASSWORD=
+POSTGRES_USER=iri
+DEBUG=True
+ADMIN_EMAIL=admin@mail.com
+
--- a/.hgignore	Fri Mar 16 18:54:11 2018 +0100
+++ b/.hgignore	Wed Jul 17 22:46:52 2024 +0200
@@ -26,3 +26,8 @@
 ^src/MANIFEST.in$
 ^dev/hdalab/dist
 ^dev/out
+^data_docker
+.env$
+.envrc$
+.etchosts$
+erreur_import$
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker-compose.yaml	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,95 @@
+services:
+  server:
+    image: hdalab_server
+    build:
+      context: .
+      dockerfile: docker/server/server.dockerfile
+    environment:
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/hdalab2
+      DJANGO_SECRET: ${DJANGO_SECRET}
+      DEBUG: ${DEBUG}
+      ADMIN_EMAIL: ${ADMIN_EMAIL}
+      BROKER_PASSWORD: ${BROKER_PASSWORD}
+    depends_on:
+      broker:
+        condition: service_started
+      db:
+        condition: service_healthy
+    volumes:
+      - ./data_docker/static:/static
+
+  worker:
+    image: hdalab_server
+    command: celery worker -A hdalab --loglevel=INFO
+    environment:
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/hdalab2
+      DJANGO_SECRET: ${DJANGO_SECRET}
+      DEBUG: False
+      ADMIN_EMAIL: ${ADMIN_EMAIL}
+      BROKER_PASSWORD: ${BROKER_PASSWORD}
+    depends_on:
+      broker:
+        condition: service_started
+      db:
+        condition: service_healthy
+
+  beat:
+    image: hdalab_server
+    command: celery beat -A hdalab --schedule /celery/beat.db --loglevel=INFO
+    environment:
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/hdalab2
+      DJANGO_SECRET: ${DJANGO_SECRET}
+      DEBUG: ${DEBUG}
+      ADMIN_EMAIL: ${ADMIN_EMAIL}
+      BROKER_PASSWORD: ${BROKER_PASSWORD}
+    volumes:
+      - ./data_docker/celery:/celery
+    depends_on:
+      broker:
+        condition: service_started
+      db:
+        condition: service_healthy
+
+  broker:
+    image: rabbitmq:3-alpine
+    hostname: hdalab_rabbitmq
+    environment: 
+      RABBITMQ_DEFAULT_USER: iri
+      RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD}
+      RABBITMQ_DEFAULT_VHOST: hdalab
+
+  db:
+    image: postgres
+    restart: always
+    volumes:
+      - ./data_docker/db:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: "hdalab2"
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
+      POSTGRES_USER: "${POSTGRES_USER}"
+      POSTGRES_HOST_AUTH_METHOD: "md5"
+      POSTGRES_INITDB_ARGS: "--auth-local=md5"
+    healthcheck:
+      test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  es:
+    image: launcher.gcr.io/google/elasticsearch2
+    restart: always
+    volumes:
+      - ./data_docker/es:/usr/share/elasticsearch/data
+
+  caddy:
+    build:
+      context: docker/web
+      dockerfile: web.dockerfile
+    volumes:
+      - ./data_docker/static:/static
+    ports:
+      - "80:80"
+      - "443:443"
+
+  cache:
+    image: docker.io/memcached:alpine
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker/server/config.py	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+'''
+Created on Jan 26, 2012
+
+@author: ymh
+'''
+import logging
+import os
+
+from environs import Env
+ 
+env = Env()
+# Read .env into os.environ
+env.read_env()
+
+DEBUG = env.bool("DEBUG", default=False)
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    ('Hdalab admin', env.str("ADMIN_EMAIL")),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': env.dj_db_url("DATABASE_URL"),
+}
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'LOCATION': 'cache:11211',
+        'KEY_PREFIX': 'hdalab2',
+        'TIMEOUT': 500,
+    }
+}
+
+BASE_DIR = "/code/"
+BASE_URL = '/'
+WEB_URL = 'https://hdalab.iri-research.org'
+SCRIPT_PREFIX = BASE_URL + 'hdalab'
+
+LOCALE_PATHS = ( os.path.join(BASE_DIR, 'locale',), )
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/home/media/media.lawrence.com/media/"
+MEDIA_ROOT = "/static/media/"
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
+MEDIA_URL = BASE_URL + "static/media/"
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/home/media/media.lawrence.com/static/"
+STATIC_ROOT = "/static/site/"
+
+# URL prefix for static files.
+# Example: "http://media.lawrence.com/static/"
+STATIC_URL = BASE_URL + "static/site/"
+
+# URL prefix for admin static files -- CSS, JavaScript and images.
+# Make sure to use a trailing slash.
+# Examples: "http://foo.com/static/admin/", "/static/admin/".
+ADMIN_MEDIA_PREFIX = STATIC_URL + 'admin/'
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = ['hdalab.iri-research.org']
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# In a Windows environment this must be set to your system time zone.
+TIME_ZONE = 'Europe/Paris'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'fr-FR'
+
+SITE_ID = 1
+
+#if you set this to False, Django will not use timezone-aware datetimes.
+USE_TZ = True
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': "http://"+env.str("ES_HOST", default="es")+":9200",
+        'INDEX_NAME': 'jocondelab',
+    },
+}
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = env.str("DJANGO_SECRET")
+
+#LOG_FILE = os.path.abspath(os.path.join(BASE_DIR,"../../run/log/log.txt"))
+LOG_LEVEL = logging.INFO
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'formatters' : {
+        'simple' : {
+            'format': "%(asctime)s - %(levelname)s : %(message)s",
+        },
+        'semi-verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(message)s'
+        },
+    },
+    'handlers': {
+        'stream_to_console': {
+            'level': LOG_LEVEL,
+            'class': 'logging.StreamHandler'
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['stream_to_console'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+        'hdabo': {
+            'handlers': ['stream_to_console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'hdalab': {
+            'handlers': ['stream_to_console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'core': {
+            'handlers': ['stream_to_console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'rdflib_sqlalchemy': {
+            'handlers': ['stream_to_console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+    }
+}
+
+
+USE_ETAGS = False
+
+GOOGLE_ANALYTICS_CODE = None
+
+DEFAULT_FROM_EMAIL = "do-not-reply@iri-research.org"
+
+ENVELOPE_EMAIL_RECIPIENTS= ['histoiredesarts@culture.gouv.fr']
+
+RENKAN_PREVIEW_PHANTOMJS_PATH = '/usr/local/bin/phantomjs'
+
+BROKER_URL = 'amqp://iri:'+ env.str('BROKER_PASSWORD') +'@broker:5672/hdalab'
+
+RENKAN_TUTORIAL_VIDEO_URLS = [
+    {'format': 'video/mp4', 'url': 'http://media.iri.centrepompidou.fr/video/hdalab/hdalab_renkan_presentation_720p.mp4' },
+    {'format': 'video/webm', 'url': 'http://media.iri.centrepompidou.fr/video/hdalab/hdalab_renkan_presentation_720p.webm' },
+    {'format': 'video/ogg', 'url': 'http://media.iri.centrepompidou.fr/video/hdalab/hdalab_renkan_presentation_720p.ogv' }
+]
+
+WIKIPEDIA_API_URL = "https://fr.wikipedia.org/w/api.php"
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker/server/hdalab.yml	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,8 @@
+uwsgi:
+  master: true
+  socket: :8000
+  processes: 4
+  chdir: /code
+  module: hdalab.wsgi
+  pythonpath: "/code"
+  log-master: true
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker/server/requirements.txt	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,51 @@
+pathlib==1.0.1
+amqp==2.2.2
+anyjson==0.3.3
+billiard==3.5.0.3
+celery==4.1.0
+cssselect==0.9.1
+cssutils==1.0
+Django==1.8.2
+django-appconf==1.0.1
+django-celery-email==1.1.0
+django-envelope==1.0
+django-extensions==1.5.1
+django-haystack==2.4.0
+django-honeypot==0.4.0
+django-registration-redux==1.2
+easy-thumbnails==2.6
+elasticsearch==1.4.0
+dj-database-url==0.5.0
+environs [django]==4.2.0
+html5lib==0.999
+httplib2==0.22.0
+isodate==0.5.1
+jedi===0.17.2
+kombu==4.1.0
+lxml==3.4.2
+Pillow==6.2.2
+premailer==2.8.3
+psycopg2==2.8.4
+pycrypto==2.6.1
+pylibmc==1.6.1
+pyparsing==2.4.7
+pytz
+PyYAML==3.11
+rdflib==4.2.2
+#renkanmanager==0.12.17
+requests==2.6.0
+simplejson==3.6.5
+six==1.9.0
+SPARQLWrapper==1.6.4
+SQLAlchemy==0.9.9
+ua-parser==0.3.6
+Unidecode==0.4.17
+urllib3==1.10.2
+user-agents==0.3.0
+uWSGI==2.0.18
+vine==1.1.4
+wheel==0.24.0
+Whoosh==2.6.0
+wikitools==1.2
+python-memcached==1.59
+pylibmc==1.6.1
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker/server/server.dockerfile	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,63 @@
+# set base image (host OS)
+FROM docker.io/python:2.7-alpine as builder
+
+ENV VIRTUALENV=/opt/venv
+
+# set the working directory in the container
+WORKDIR /code
+
+RUN \
+    pip install virtualenv && \
+    virtualenv $VIRTUALENV
+
+ENV PATH="$VIRTUALENV/bin:$PATH"
+
+# copy the dependencies file to the working directory
+COPY docker/server/requirements.txt .
+
+
+RUN \
+ apk add --no-cache postgresql-client postgresql-libs libxml2 libxslt libmemcached curl tar && \
+ apk add --no-cache --virtual .build-deps git gcc musl-dev postgresql-dev libxml2-dev libxslt-dev linux-headers libmemcached-dev libffi-dev libgcc openssl-dev curl jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev && \
+ pip install -r requirements.txt && \
+ curl -L https://www.iri.centrepompidou.fr/dev/hg/renkan/archive/V00.13.04.tar.gz | tar zxvf - --strip-components=4 renkan-V00.13.04/server/python/django/renkanmanager 
+
+
+FROM docker.io/python:2.7-alpine 
+
+COPY --from=builder /opt/venv /opt/venv
+COPY --from=builder /code/renkanmanager /code/renkanmanager
+COPY src/ /code/
+
+
+ENV BASEDIR="/code"
+ENV PATH="/opt/venv/bin:$BASEDIR:$PATH"
+
+WORKDIR $BASEDIR
+
+RUN \
+ apk add --no-cache postgresql-client postgresql-libs libxml2 libxslt bash libmemcached py-pathlib2 jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev && \
+ mkdir -p /static
+
+
+COPY docker/server/config.py ./hdalab/
+
+COPY docker/server/hdalab.yml .
+COPY --chmod=0755 docker/server/entrypoint.sh /
+
+ENV PYTHONPATH "/code/"
+ENV DJANGO_SETTINGS_MODULE hdalab.settings
+ENV DEBUG "False"
+ENV DATABASE_URL ""
+ENV ES_HOST "es"
+ENV DJANGO_SECRET ""
+ENV ADMIN_EMAIL ""
+ENV BROKER_PASSWORD ""
+
+
+EXPOSE 8000
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+# command to run on container start
+CMD [ "/opt/venv/bin/uwsgi", "--yaml", "/code/hdalab.yml"]
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker/web/Caddyfile	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,20 @@
+hdalab.iri-research.org {
+    tls internal
+
+    redir / /hdalab/
+    rewrite /hdalab /hdalab/
+
+    handle_path /static/* {
+        root * /static
+        file_server
+    }
+    
+    handle_path /hdalab/* {
+        reverse_proxy server:8000 {
+            transport uwsgi {
+                uwsgi_param SCRIPT_NAME /hdalab
+                uwsgi_param UWSGI_SCHEME https
+            }
+        }
+    }
+}
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docker/web/web.dockerfile	Wed Jul 17 22:46:52 2024 +0200
@@ -0,0 +1,10 @@
+FROM docker.io/caddy:builder AS builder
+
+RUN xcaddy build \
+    --with github.com/BadAimWeeb/caddy-uwsgi-transport
+
+FROM docker.io/caddy:latest
+
+COPY --from=builder /usr/bin/caddy /usr/bin/caddy
+COPY Caddyfile /etc/caddy/Caddyfile
+
--- a/src/hdalab/static/hdalab/js/gomina.js	Fri Mar 16 18:54:11 2018 +0100
+++ b/src/hdalab/static/hdalab/js/gomina.js	Wed Jul 17 22:46:52 2024 +0200
@@ -274,22 +274,25 @@
         var _htmlCl = '<ul id="contentlist">'
             + contentdata.map(function(_d) {
                 var _dsurl = gomNs.urls.datasheet.replace(/ID$/,_d.hda_id);
+                var _maplet = '';
+                if(typeof _d.coords == "object") {
+                    var _zoom = 14;
+                    var _n = 2**_zoom;
+                    var _x = Math.round(_n * ((_d.coords.longitude + 180.0)/360.0));
+                    var _lat_rad = _d.coords.latitude * Math.PI / 180.0;
+                    var _y = Math.round(_n * (1.0 - (Math.log(Math.tan(_lat_rad) + 1.0/Math.cos(_lat_rad)) / Math.PI)) / 2.0);
+                    _maplet = `<div class="maplet"><img src="https://tile.openstreetmap.org/${_zoom}/${_x}/${_y}.png" /><h4>`
+                        + gettext('Localisation&nbsp;:')
+                        + ' '
+                        + _d.coords.city_name
+                        + '</h4></div>';
+                }
                 var _html = '<li class="content-item"><h3>'
                     + ((_d.hda_id)?'<a href="'+_dsurl+ '">':'')
                     + _d.title
                     + ((_d.hda_id)?'</a>&nbsp;<a class="mind-map-icon" href="' + gomNs.urls.renkan + '?notice=' + _d.hda_id + '" target="_blank">placeholder</a>':'')
                     + '</h3>'
-                    + ( typeof _d.coords == "object" ?
-                        '<div class="maplet"><img src="http://maps.googleapis.com/maps/api/staticmap?center=47,1.5&zoom=4&size=160x160&maptype=roadmap&markers=color:red%7C'
-                        + _d.coords.latitude
-                        + ','
-                        + _d.coords.longitude
-                        + '&sensor=false" /><h4>'
-                        + gettext('Localisation&nbsp;:')
-                        + ' '
-                        + _d.coords.city_name
-                        + '</h4></div>'
-                        : '')
+                    + _maplet
                     + '<h4><a href="'
                     + _d.url
                     + '" target="_blank">'
--- a/src/hdalab/views/ajax.py	Fri Mar 16 18:54:11 2018 +0100
+++ b/src/hdalab/views/ajax.py	Wed Jul 17 22:46:52 2024 +0200
@@ -124,8 +124,8 @@
                 'url': datasheet.url,
                 'description': datasheet.description,
                 'hda_id': datasheet.hda_id,
-                'organization': datasheet.organisation.name,
-                'organization_url': datasheet.organisation.website,
+                'organization': datasheet.organisation.name if datasheet.organisation else "",
+                'organization_url': datasheet.organisation.website if datasheet.organisation else "",
                 'score': max(dsscore, rootscore)
             })
     if resobj: