# HG changeset patch # User ymh # Date 1467720235 -7200 # Node ID 8e1440b83f0c65984d355b32da42877bfd8d0195 # Parent 06f627e804b6f2b80e89e5584dd9d5511264c43c put task under celery for better perf management diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/apps.py --- a/server/src/metaeducation/apps.py Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/metaeducation/apps.py Tue Jul 05 14:03:55 2016 +0200 @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from django.apps import AppConfig class MetaeducationConfig(AppConfig): @@ -6,3 +8,4 @@ def ready(self): import metaeducation.signals + from .celery import app as celery_app # noqa diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/celery.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/metaeducation/celery.py Tue Jul 05 14:03:55 2016 +0200 @@ -0,0 +1,15 @@ +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metaeducation.settings.dev') + +from django.conf import settings # noqa + +app = Celery('metaeducation') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/settings/dev.py.tmpl --- a/server/src/metaeducation/settings/dev.py.tmpl Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/metaeducation/settings/dev.py.tmpl Tue Jul 05 14:03:55 2016 +0200 @@ -2,6 +2,10 @@ from metaeducation.settings import * import os, logging, base64 +INSTALLED_APPS += ('kombu.transport.django',) + +BROKER_URL = 'django://' + BASE_URL = '/' WEB_URL = '' @@ -131,4 +135,3 @@ LRS_MTDC_RENKAN_PASSWORD = "" # Password of Renkan app for LRS auth TRACKING_RNKNS_NAMESPACE = "" # Namespace for renkan nodes and edges URI, must be of the format "http://{something}/" TRACKING_VOCABULARY_NAMESPACE = "" # Namespace for tracking custom vocabulary, must be of the format "http://{something}/" - diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/tasks.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/metaeducation/tasks.py Tue Jul 05 14:03:55 2016 +0200 @@ -0,0 +1,2 @@ +# just import this file for celery task autodiscovery +from metaeducation.tracking import send_tracking_data diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/tracking/messages.py --- a/server/src/metaeducation/tracking/messages.py Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/metaeducation/tracking/messages.py Tue Jul 05 14:03:55 2016 +0200 @@ -92,7 +92,7 @@ } } - send_tracking_data(msg) + send_tracking_data.delay(msg) def send_open_read_renkan(renkan, current_user, registration): @@ -113,7 +113,7 @@ } } - send_tracking_data(msg) + send_tracking_data.delay(msg) def send_open_edit_renkan(renkan, current_user, registration): msg = get_base_message('open_edit', renkan.renkan_guid, current_user, registration) @@ -133,7 +133,7 @@ } } - send_tracking_data(msg) + send_tracking_data.delay(msg) def _send_operation_renkan(action, renkan, current_user, content=None): if not content: @@ -159,7 +159,7 @@ } } } - send_tracking_data(msg) + send_tracking_data.delay(msg) send_delete_renkan = functools.partial(_send_operation_renkan, 'delete') @@ -188,4 +188,4 @@ 'http://liris.renkantracking.org/fromCreate': {'renkan': renkan.source_revision.parent_renkan.renkan_guid, 'revision': renkan.source_revision.revision_guid} } } - send_tracking_data(msg) + send_tracking_data.delay(msg) diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/tracking/middleware.py --- a/server/src/metaeducation/tracking/middleware.py Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/metaeducation/tracking/middleware.py Tue Jul 05 14:03:55 2016 +0200 @@ -10,6 +10,8 @@ logger = logging.getLogger(__name__) +# Inspired by https://github.com/Atomidata/django-audit-log/blob/master/audit_log/middleware.py + class TrackingMiddleware(object): def process_request(self, request): if not request.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'): diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/tracking/tasks.py --- a/server/src/metaeducation/tracking/tasks.py Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/metaeducation/tracking/tasks.py Tue Jul 05 14:03:55 2016 +0200 @@ -1,9 +1,11 @@ import logging, json import requests from django.conf import settings +from metaeducation.celery import app as celery_app logger = logging.getLogger(__name__) +@celery_app.task() def send_tracking_data(json_data): tracking_data = (json_data if isinstance(json_data, str) else json.dumps(json_data)).encode('utf-8') diff -r 06f627e804b6 -r 8e1440b83f0c server/src/metaeducation/views/tracking.py --- a/server/src/metaeducation/views/tracking.py Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/metaeducation/views/tracking.py Tue Jul 05 14:03:55 2016 +0200 @@ -15,7 +15,7 @@ def post(self, request): logger.debug("POSTING DATA %r", str(request.body, 'utf-8')) - send_tracking_data(str(request.body, 'utf-8')) + send_tracking_data.delay(str(request.body, 'utf-8')) # This is a fire and forget !!! # please track the log to see if everything is ok diff -r 06f627e804b6 -r 8e1440b83f0c server/src/requirement.txt --- a/server/src/requirement.txt Tue Jul 05 13:19:38 2016 +0200 +++ b/server/src/requirement.txt Tue Jul 05 14:03:55 2016 +0200 @@ -11,3 +11,4 @@ requests==2.9.1 requests-oauthlib==0.6.0 six==1.10.0 +celery==3.1.23