put task under celery for better perf management
authorymh <ymh.work@gmail.com>
Tue, 05 Jul 2016 14:03:55 +0200
changeset 78 8e1440b83f0c
parent 77 06f627e804b6
child 79 49ca16704275
put task under celery for better perf management
server/src/metaeducation/apps.py
server/src/metaeducation/celery.py
server/src/metaeducation/settings/dev.py.tmpl
server/src/metaeducation/tasks.py
server/src/metaeducation/tracking/messages.py
server/src/metaeducation/tracking/middleware.py
server/src/metaeducation/tracking/tasks.py
server/src/metaeducation/views/tracking.py
server/src/requirement.txt
--- a/server/src/metaeducation/apps.py	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/metaeducation/apps.py	Tue Jul 05 14:03:55 2016 +0200
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+
 from django.apps import AppConfig
 
 class MetaeducationConfig(AppConfig):
@@ -6,3 +8,4 @@
 
     def ready(self):
         import metaeducation.signals
+        from .celery import app as celery_app  # noqa
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/src/metaeducation/celery.py	Tue Jul 05 14:03:55 2016 +0200
@@ -0,0 +1,15 @@
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metaeducation.settings.dev')
+
+from django.conf import settings  # noqa
+
+app = Celery('metaeducation')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
--- a/server/src/metaeducation/settings/dev.py.tmpl	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/metaeducation/settings/dev.py.tmpl	Tue Jul 05 14:03:55 2016 +0200
@@ -2,6 +2,10 @@
 from metaeducation.settings import *
 import os, logging, base64
 
+INSTALLED_APPS += ('kombu.transport.django',)
+
+BROKER_URL = 'django://'
+
 BASE_URL = '/'
 WEB_URL = ''
 
@@ -131,4 +135,3 @@
 LRS_MTDC_RENKAN_PASSWORD = "" # Password of Renkan app for LRS auth
 TRACKING_RNKNS_NAMESPACE = "" # Namespace for renkan nodes and edges URI, must be of the format "http://{something}/"
 TRACKING_VOCABULARY_NAMESPACE = "" # Namespace for tracking custom vocabulary, must be of the format "http://{something}/"
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/src/metaeducation/tasks.py	Tue Jul 05 14:03:55 2016 +0200
@@ -0,0 +1,2 @@
+# just import this file for celery task autodiscovery
+from metaeducation.tracking import send_tracking_data
--- a/server/src/metaeducation/tracking/messages.py	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/metaeducation/tracking/messages.py	Tue Jul 05 14:03:55 2016 +0200
@@ -92,7 +92,7 @@
         }
     }
 
-    send_tracking_data(msg)
+    send_tracking_data.delay(msg)
 
 
 def send_open_read_renkan(renkan, current_user, registration):
@@ -113,7 +113,7 @@
         }
     }
 
-    send_tracking_data(msg)
+    send_tracking_data.delay(msg)
 
 def send_open_edit_renkan(renkan, current_user, registration):
     msg = get_base_message('open_edit', renkan.renkan_guid, current_user, registration)
@@ -133,7 +133,7 @@
         }
     }
 
-    send_tracking_data(msg)
+    send_tracking_data.delay(msg)
 
 def _send_operation_renkan(action, renkan, current_user, content=None):
     if not content:
@@ -159,7 +159,7 @@
             }
         }
     }
-    send_tracking_data(msg)
+    send_tracking_data.delay(msg)
 
 
 send_delete_renkan = functools.partial(_send_operation_renkan, 'delete')
@@ -188,4 +188,4 @@
                 'http://liris.renkantracking.org/fromCreate': {'renkan': renkan.source_revision.parent_renkan.renkan_guid, 'revision': renkan.source_revision.revision_guid}
             }
         }
-    send_tracking_data(msg)
+    send_tracking_data.delay(msg)
--- a/server/src/metaeducation/tracking/middleware.py	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/metaeducation/tracking/middleware.py	Tue Jul 05 14:03:55 2016 +0200
@@ -10,6 +10,8 @@
 
 logger = logging.getLogger(__name__)
 
+# Inspired by https://github.com/Atomidata/django-audit-log/blob/master/audit_log/middleware.py
+
 class TrackingMiddleware(object):
     def process_request(self, request):
         if not request.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
--- a/server/src/metaeducation/tracking/tasks.py	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/metaeducation/tracking/tasks.py	Tue Jul 05 14:03:55 2016 +0200
@@ -1,9 +1,11 @@
 import logging, json
 import requests
 from django.conf import settings
+from metaeducation.celery import app as celery_app
 
 logger = logging.getLogger(__name__)
 
+@celery_app.task()
 def send_tracking_data(json_data):
 
     tracking_data = (json_data if isinstance(json_data, str) else json.dumps(json_data)).encode('utf-8')
--- a/server/src/metaeducation/views/tracking.py	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/metaeducation/views/tracking.py	Tue Jul 05 14:03:55 2016 +0200
@@ -15,7 +15,7 @@
     def post(self, request):
         logger.debug("POSTING DATA %r", str(request.body, 'utf-8'))
 
-        send_tracking_data(str(request.body, 'utf-8'))
+        send_tracking_data.delay(str(request.body, 'utf-8'))
 
         # This is a fire and forget !!!
         # please track the log to see if everything is ok
--- a/server/src/requirement.txt	Tue Jul 05 13:19:38 2016 +0200
+++ b/server/src/requirement.txt	Tue Jul 05 14:03:55 2016 +0200
@@ -11,3 +11,4 @@
 requests==2.9.1
 requests-oauthlib==0.6.0
 six==1.10.0
+celery==3.1.23