export sync code in utils, and add button to synchronize annotation on eventsessionview
--- a/annot-server/settings.py.tmpl Sat Oct 25 05:41:51 2014 +0200
+++ b/annot-server/settings.py.tmpl Sat Oct 25 05:43:01 2014 +0200
@@ -9,3 +9,5 @@
STATIC_URL = '/static'
STATIC_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)),'static')
LDT_PLATFORM_BASE_URL = 'http://ldt.iri.centrepompidou.fr/ldtplatform'
+
+SYNC_POST_PARAM = {}
--- a/annot-server/utils.py Sat Oct 25 05:41:51 2014 +0200
+++ b/annot-server/utils.py Sat Oct 25 05:43:01 2014 +0200
@@ -4,6 +4,360 @@
# Copyright (c) 2014 IRI
#
+import bisect
+import datetime
+import json
+import logging
+import re
+import uuid
+
+from dateutil.parser import parse as parse_date_raw
+from dateutil.tz import tzutc
+
+from lxml import etree
+
+import requests
+
+
+logger = logging.getLogger(__name__)
PIANOROLL_CHANNEL = 'PIANOROLL'
ANNOTATION_CHANNEL = 'ANNOT'
+
+
+class AnnotationsSynchronizer(object):
+
+ LDT_CONTENT_REST_API_PATH = "api/ldt/1.0/contents/"
+ LDT_PROJECT_REST_API_PATH = "api/ldt/1.0/projects/"
+ DEFAULT_ANNOTATION_CHANNEL = 'ANNOT'
+
+ def parse_date(self, datestr):
+ res = parse_date_raw(datestr)
+ if res.tzinfo is None:
+ res = res.replace(tzinfo=tzutc())
+ return res
+
+ def find_delta(self, ts):
+ if self.deltas:
+ i = bisect.bisect_right(self.deltas, (ts+1,0))
+ if i:
+ return self.deltas[i-1]
+ return (0,0)
+
+
+ def get_filter(self):
+ res = []
+ if self.start_date:
+ res.append({'name': 'ts', 'op': ">=", 'val':self.start_date.isoformat() })
+ if self.end_date:
+ res.append({'name': 'ts', 'op': "<=", 'val':self.end_date.isoformat() })
+ if self.events:
+ res.append({'name': 'event_code', 'op': "in", 'val': self.events })
+ if self.channels:
+ res.append({'name': 'channel', 'op': "in", 'val': self.channels })
+ if self.user_whitelist:
+ res.append({'name': 'user', 'op': "in", 'val': self.user_whitelist })
+ return res
+
+
+ def build_annotation_iterator(self, params):
+ page = 0
+ page_nb = 1
+ while page < page_nb:
+ page += 1
+ params['page'] = page
+ resp = requests.get(self.annot_url, params=params, headers={'Content-Type': 'application/json'})
+ if resp.status_code != requests.codes.ok:
+ self.logger.debug("build_annotation_iterator : request %s : return code %r " % (resp.request.url, resp.status_code))
+ return
+ resp_json = resp.json()
+ page_nb = resp_json.get('total_pages', 1)
+ for item in resp_json.get('objects', []):
+ #TODO: add progress log
+ yield item
+
+
+
+ def __init__(self, start_date=None, end_date=None, duration=None, content_file=None,
+ content_file_write=None, project_id=None,
+ channels=[DEFAULT_ANNOTATION_CHANNEL], events=[], annot_url=None,
+ user_whitelist=None, post_param={}, deltas=None, base_url="http://ldt.iri.centrepompidou.fr/ldtplatform/",
+ content=None, content_id=None, video=None,
+ replace=True, merge=False, name="", batch_size=500,
+ filename="project.ldt", color="16763904", logger=logger):
+
+ self.logger = logger
+ self.base_url = base_url.rstrip("/")+"/" if base_url else base_url
+ self.deltas = deltas
+ self.post_param = {}
+ if isinstance(post_param, basestring):
+ self.post_param = json.loads(post_param)
+ elif post_param:
+ self.post_param = post_param
+
+ self.start_date = None
+ if start_date:
+ if isinstance(start_date, datetime.datetime):
+ self.start_date = start_date
+ else:
+ self.start_date = self.parse_date(str(start_date))
+
+ self.duration = duration
+ self.end_date = None
+ if end_date:
+ if isinstance(end_date, datetime.datetime):
+ self.end_date = end_date
+ else:
+ self.end_date= self.parse_date(str(end_date))
+
+ if end_date:
+ if isinstance(end_date, datetime.datetime):
+ self.end_date = end_date
+ else:
+ self.end_date= self.parse_date(str(end_date))
+ elif self.start_date and self.duration:
+ self.end_date = self.start_date + datetime.timedelta(seconds=self.duration)
+ elif self.start_date and self.base_url:
+ # get duration from api
+ content_url = self.base_url + AnnotationsSynchronizer.LDT_CONTENT_REST_API_PATH + content_id + "/?format=json"
+ self.logger.debug("get duration " + content_url) #@UndefinedVariable
+ r = requests.get(content_url, params=self.post_param)
+ self.logger.debug("get duration resp " + repr(r)) #@UndefinedVariable
+ self.duration = int(r.json()['duration'])
+ self.logger.debug("get duration " + repr(self.duration)) #@UndefinedVariable
+
+ self.end_date = self.start_date + datetime.timedelta(seconds=int(self.duration/1000))
+
+ if self.end_date and self.deltas:
+ self.end_date = self.end_date + datetime.timedelta(milliseconds=self.deltas[-1][1])
+
+ self.content_file = content_file
+ self.project_id = project_id
+ if self.project_id is not None:
+ self.content_file = self.base_url + AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + self.project_id + "/?format=json"
+ self.content_file_write = content_file_write
+ if self.content_file_write is None and self.project_id:
+ self.content_file_write = self.content_file
+ self.channels = list(set(channels))
+ self.annot_url = annot_url
+ self.events = list(set(events))
+ self.user_whitelist_file = user_whitelist
+ if self.user_whitelist_file:
+ with open(user_whitelist_file, 'r+') as f:
+ self.user_whitelist = list(set([s.strip() for s in f]))
+ else:
+ self.user_whitelist = None
+ self.content = content
+ self.content_id = content_id
+ self.video = video
+ self.replace = replace
+ self.merge = merge
+ self.name = name
+ self.batch_size = batch_size
+ self.filename = filename
+ self.color = color
+
+ def export_annotations(self):
+
+ root = None
+ ensemble_parent = None
+
+ #to do : analyse situation ldt or iri ? filename set or not ?
+
+ if self.content_file and self.content_file.find("http://") == 0:
+
+ self.logger.debug("url : " + self.content_file) #@UndefinedVariable
+
+ r = requests.get(self.content_file, params=self.post_param)
+ self.logger.debug("url response " + repr(r) + " content " + repr(r.text)) #@UndefinedVariable
+ project = r.json()
+ text_match = re.match(r"\<\?\s*xml.*?\?\>(.*)", project['ldt'], re.I|re.S)
+ root = etree.fromstring(text_match.group(1) if text_match else project['ldt'])
+
+ elif self.content_file and os.path.exists(self.content_file):
+
+ doc = etree.parse(self.content_file)
+ root = doc.getroot()
+
+ content_id = None
+
+ if root is None:
+
+ root = etree.Element(u"iri")
+
+ project = etree.SubElement(root, u"project", {u"abstract":u"Annotations",u"title":u"Annotations", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())})
+
+ medias = etree.SubElement(root, u"medias")
+ media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(self.content), u"video":unicode(self.video), u"id":unicode(self.content_id), u"extra":u""})
+
+ annotations = etree.SubElement(root, u"annotations")
+ content = etree.SubElement(annotations, u"content", {u"id":unicode(self.content_id)})
+ ensemble_parent = content
+
+ content_id = self.content_id
+
+
+ if ensemble_parent is None:
+ file_type = None
+ for node in root:
+ if node.tag == "project":
+ file_type = "ldt"
+ break
+ elif node.tag == "head":
+ file_type = "iri"
+ break
+
+ if file_type == "ldt":
+ media_nodes = root.xpath("//media")
+ if len(media_nodes) > 0:
+ media = media_nodes[0]
+ annotations_node = root.find(u"annotations")
+ if annotations_node is None:
+ annotations_node = etree.SubElement(root, u"annotations")
+ content_node = annotations_node.find(u"content")
+ if content_node is None:
+ content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id"))
+ ensemble_parent = content_node
+ content_id = content_node.get(u"id")
+ display_nodes = root.xpath("//displays/display/content[@id='%s']" % content_id)
+ if len(display_nodes) == 0:
+ self.logger.info("No display node found. Will not update display")
+ display_content_node = None
+ else:
+ display_content_node = display_nodes[0]
+
+ elif file_type == "iri":
+ body_node = root.find(u"body")
+ if body_node is None:
+ body_node = etree.SubElement(root, u"body")
+ ensembles_node = body_node.find(u"ensembles")
+ if ensembles_node is None:
+ ensembles_node = etree.SubElement(body_node, u"ensembles")
+ ensemble_parent = ensembles_node
+ content_id = root.xpath("head/meta[@name='id']/@content")[0]
+ display_content_node = None
+
+
+ if ensemble_parent is None:
+ self.logger.error("Can not process file") #@UndefinedVariable
+ sys.exit()
+
+ if self.replace:
+ for ens in ensemble_parent.iterchildren(tag=u"ensemble"):
+ ens_id = ens.get("id","")
+ if ens_id.startswith("annot_"):
+ ensemble_parent.remove(ens)
+ # remove in display nodes
+ if display_content_node is not None:
+ for cut_display in display_content_node.iterchildren():
+ if cut_display.get('idens','') == ens_id:
+ display_content_node.remove(cut_display)
+
+ ensemble = None
+ elements = None
+
+ if self.merge:
+ for ens in ensemble_parent.findall(u"ensemble"):
+ if ens.get('id',"").startswith("annot_"):
+ ensemble = ens
+ break
+ if ensemble is not None:
+ elements = ensemble.find(u".//elements")
+ decoupage = ensemble.find(u"decoupage")
+
+ if ensemble is None or elements is None:
+ ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"annot_" + unicode(uuid.uuid4()), u"title":u"Ensemble Annotation", u"author":u"IRI Web", u"abstract":u"Ensemble Annotation"})
+ decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"})
+
+ etree.SubElement(decoupage, u"title").text = unicode(self.name)
+ etree.SubElement(decoupage, u"abstract").text = unicode(self.name)
+
+ elements = etree.SubElement(decoupage, u"elements")
+
+ ensemble_id = ensemble.get('id', '')
+ decoupage_id = decoupage.get('id', '') if decoupage is not None else None
+
+
+ filters = self.get_filter()
+
+
+ params = { 'q':json.dumps({'filters':filters}), 'results_per_page': self.batch_size}
+
+
+ for annot in self.build_annotation_iterator(params):
+ annot_ts = self.parse_date(annot['ts'])
+ if self.start_date is None:
+ star_date = annot_ts
+ annot_ts_rel = annot_ts-self.start_date
+ annot_ts_rel_milli = int(round(annot_ts_rel.total_seconds()*1000))
+ d = self.find_delta(annot_ts_rel_milli)
+ if d[1] < 0:
+ continue
+ else :
+ annot_ts_rel_milli -= d[1]
+ annot_content = annot.get('content',{'category':'', 'user':None})
+
+ username = annot_content.get('user', 'anon.') or 'anon.'
+
+ category = annot_content.get('category', None)
+ if category is None:
+ continue
+
+ element = etree.SubElement(elements, u"element" , {u"id":annot.get('uuid', uuid.uuid4()), u"color":unicode(self.color), u"author":unicode(username), u"date":unicode(annot_ts.strftime("%Y/%m/%d")), u"begin": unicode(annot_ts_rel_milli), u"dur":u"0"})
+ etree.SubElement(element, u"title").text = unicode(username) + u": " + unicode(category.get('label', category.get('code', '')))
+ etree.SubElement(element, u"abstract").text = unicode(category.get('label', category.get('code', '')))
+
+ tags_node = etree.SubElement(element, u"tags")
+ etree.SubElement(tags_node,u"tag").text = category.get('code', '')
+
+ meta_element = etree.SubElement(element, u'meta')
+
+ polemics_element = etree.Element(u'polemics')
+ etree.SubElement(polemics_element, u'polemic').text = category.get('code', '')
+ meta_element.append(polemics_element)
+
+ etree.SubElement(meta_element, u"source", attrib={"url":self.annot_url + "/" + annot['uuid'], "mimetype":u"application/json"}).text = etree.CDATA(json.dumps(annot))
+
+ # sort by tc in
+ if self.merge :
+ # remove all elements and put them in a array
+ # sort them with tc
+ #put them back
+ elements[:] = sorted(elements,key=lambda n: int(n.get('begin')))
+
+ #add to display node
+ if display_content_node is not None:
+ display_dec = None
+ for dec in display_content_node.iterchildren(tag=u"decoupage"):
+ if dec.get('idens','') == ensemble_id and dec.get('id', '') == decoupage_id:
+ display_dec = dec
+ break
+ if display_dec is None and ensemble_id and decoupage_id:
+ etree.SubElement(display_content_node, u"decoupage", attrib={'idens': ensemble_id, 'id': decoupage_id, 'tagsSelect':''})
+
+ output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=False, xml_declaration=True)
+
+ if self.content_file_write and self.content_file_write.find("http://") == 0:
+
+ project["ldt"] = output_data
+ project['owner'] = project['owner'].replace('%7E','~')
+ project['contents'] = [c_url.replace('%7E','~') for c_url in project['contents']]
+
+ self.logger.debug("write http " + self.content_file_write) #@UndefinedVariable
+ self.logger.debug("write http " + repr(self.post_param)) #@UndefinedVariable
+ self.logger.debug("write http " + repr(project)) #@UndefinedVariable
+ r = requests.put(self.content_file_write, data=json.dumps(project), headers={'content-type':'application/json'}, params=self.post_param);
+ self.logger.debug("write http " + repr(r) + " content " + r.text) #@UndefinedVariable
+ if r.status_code != requests.codes.ok: # @UndefinedVariable
+ r.raise_for_status()
+ else:
+ if self.content_file_write and os.path.exists(self.content_file_write):
+ dest_file_name = self.content_file_write
+ else:
+ dest_file_name = self.filename
+
+ self.logger.debug("WRITE : " + dest_file_name) #@UndefinedVariable
+ output = open(dest_file_name, "w")
+ output.write(output_data)
+ output.flush()
+ output.close()
--- a/annot-server/webapp/admin.py Sat Oct 25 05:41:51 2014 +0200
+++ b/annot-server/webapp/admin.py Sat Oct 25 05:43:01 2014 +0200
@@ -4,11 +4,14 @@
#
import json
+import logging
+from StringIO import StringIO
import urllib
from dateutil.tz import tzutc
-from flask.ext.admin import Admin
+from flask import url_for, render_template, request
+from flask.ext.admin import Admin, BaseView, expose
from flask.ext.admin.contrib.sqla import ModelView
from flask.ext.admin.contrib.sqla.form import AdminModelConverter
from flask.ext.admin.contrib.sqla.filters import BooleanEqualFilter
@@ -19,9 +22,10 @@
from wtforms import fields
from wtforms.ext.dateutil import fields as dateutil_fields
+import config
import database
import models
-import config
+import utils
from webapp import app
admin = Admin(app)
@@ -41,10 +45,6 @@
output = []
for c in model.categories_json.get("categories",[]):
- #if "subcategories" in c and len(c["subcategories"])>0:
- # for sc in c["subcategories"]:
- # output.append({"name": sc["code"],"keywords": [ sc["label"] ], "color" : sc["color"] })
- #else:
output.append({"name": c["code"] if "code" in c else "","keywords": [ c["label"] ], "color" : c["color"] })
base_url = getattr(config,'LDT_PLATFORM_BASE_URL','') + "/ldt/embediframe/?"
@@ -62,6 +62,21 @@
return Markup("<a href=\"%s\" target=\"_blank\">Lien</a> :<br/><input value=\"%s\"/><br/>Code embed :<input value=\"%s\"/>" % (url, url, embed))
+def build_sync_form(view, context, model, name):
+ # `view` is current administrative view
+ # `context` is instance of jinja2.runtime.Context
+ # `model` is model instance
+ # `name` is property name
+ if not model.project_id \
+ or not model.start_ts \
+ or not getattr(config,'LDT_PLATFORM_BASE_URL',None):
+ return ''
+
+ url = url_for('eventsessionview.sync')
+
+ return Markup(render_template('admin/sync_form.html', model=model, url=url))
+
+
class StartTSField(dateutil_fields.DateTimeField):
def __init__(self, *args, **kwargs):
if 'display_format' not in kwargs:
@@ -118,9 +133,13 @@
column_default_sort = ( models.Event.code, True)
column_sortable_list = (('event', models.Event.code),)
column_auto_select_related = True
- column_list = ('event', 'project_id', 'order', 'categories_json', 'url')
+ column_list = ('event', 'project_id', 'order', 'categories_json', 'url', 'sync')
column_filters = ('event',)
- column_formatters = {'url': build_ldt_url, 'categories_json': lambda v, c, m, p: json.dumps(m.categories_json)[:80] if m.categories_json else ''}
+ column_formatters = {
+ 'sync': build_sync_form,
+ 'url': build_ldt_url,
+ 'categories_json': lambda v, c, m, p: json.dumps(m.categories_json)[:80] if m.categories_json else ''
+ }
form_overrides = { "start_ts": StartTSField, }
form_widget_args = {
'categories_json': {
@@ -140,6 +159,41 @@
def __init__(self, session, **kwargs):
super(EventSessionView, self).__init__(models.EventSession, session, **kwargs)
+ @expose("/sync/",methods=('POST', ))
+ def sync(self):
+ event_session = database.db_session.query(models.EventSession).filter_by(id=int(request.form['event_session_id'])).first()
+
+ if not event_session:
+ abort(404)
+
+ stream = StringIO()
+ handler = logging.StreamHandler(stream)
+ logger = logging.getLogger('sync_logger')
+ logger.setLevel(logging.DEBUG)
+ for h in logger.handlers:
+ logger.removeHandler(h)
+ logger.addHandler(handler)
+
+ sync_args = {
+ 'start_date' : event_session.start_ts,
+ 'duration' : event_session.duration or None,
+ 'events' : [event_session.event.code],
+ 'channels' : [utils.AnnotationsSynchronizer.DEFAULT_ANNOTATION_CHANNEL],
+ 'project_id' : event_session.project_id,
+ 'annot_url' : url_for('annotationapi0.annotationapi', _external=True),
+ 'post_param' : getattr(config, 'SYNC_POST_PARAM', {}),
+ 'base_url' : getattr(config, 'LDT_PLATFORM_BASE_URL', "http://ldt.iri.centrepompidou.fr/ldtplatform/"),
+ 'replace' : True,
+ 'merge' : False,
+ 'batch_size' : 500,
+ 'logger' : logger,
+ }
+
+ sync = utils.AnnotationsSynchronizer(**sync_args)
+ sync.export_annotations()
+ logs = [ line for line in stream.getvalue().split("\n")]
+
+ return self.render('admin/sync_event_session.html', event_session=event_session, sync_args=repr(sync_args), logs=logs)
admin.add_view(EventView(database.db_session))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/annot-server/webapp/templates/admin/sync_event_session.html Sat Oct 25 05:43:01 2014 +0200
@@ -0,0 +1,17 @@
+{% extends 'admin/master.html' %}
+{% block body %}
+ <h3>Sync on project {{event_session.project_id}}</h3>
+
+ <h4>Sync arguments</h4>
+ <pre>
+ {{sync_args}}
+ </pre>
+
+ <h4>Logs</h4>
+ <pre>
+ {% for line in logs %}
+ {{line}}<br>
+ {% endfor %}
+ </pre>
+
+{% endblock %}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/annot-server/webapp/templates/admin/sync_form.html Sat Oct 25 05:43:01 2014 +0200
@@ -0,0 +1,4 @@
+<form action="{{url}}" method="post">
+ <input type="hidden" name="event_session_id" value="{{model.id}}"/>
+ <button type="submit">Sync</button>
+</form>
--- a/annot-server/websockets.py Sat Oct 25 05:41:51 2014 +0200
+++ b/annot-server/websockets.py Sat Oct 25 05:43:01 2014 +0200
@@ -79,6 +79,12 @@
self.event = event_ids[0]
self._init_props(self.factory.ws_factory, self.factory.conn)
+ def clean_annot(self, annot):
+ if 'ts' in annot:
+ annot['ts'] = annot['ts'].isoformat()
+ if 'uuid' in annot:
+ annot['uuid'] = str(annot['uuid'])
+
#TODO: add error handling
def onMessage(self, payload, isBinary):
if isBinary:
@@ -94,15 +100,13 @@
def error_callback(failure):
res = params.copy()
+ self.clean_annot(res)
res['status'] = 'KO'
res['failure'] = str(failure)
self.sendMessage(json.dumps(res))
def annot_callback(res):
- if 'ts' in res:
- res['ts'] = res['ts'].isoformat()+'Z'
- if 'uuid' in res:
- res['uuid'] = str(res['uuid'])
+ self.clean_annot(res)
res['status'] = 'OK'
self.sendMessage(json.dumps(res))
--- a/client/gulpfile.js Sat Oct 25 05:41:51 2014 +0200
+++ b/client/gulpfile.js Sat Oct 25 05:43:01 2014 +0200
@@ -59,6 +59,7 @@
'!./bower_components/jquery/src/**/*',
'!./bower_components/angular-bootstrap/ui-bootstrap.js',
'./bower_components/jquery/dist/jquery.js',
+ './bower_components/angular/angular.js',
'./bower_components/**/*.js'
];
--- a/utils/export_annotations.py Sat Oct 25 05:41:51 2014 +0200
+++ b/utils/export_annotations.py Sat Oct 25 05:43:01 2014 +0200
@@ -1,58 +1,18 @@
#!/usr/bin/env python
# coding=utf-8
-from lxml import etree
import argparse
-import json
import datetime
-import requests
-import os.path
-import re
import sys
-import time
-import uuid #@UnresolvedImport
-from dateutil.parser import parse as parse_date_raw
-from dateutil.tz import tzutc
-import bisect
import logging
-#class TweetExclude(object):
-# def __init__(self, id):
-# self.id = id
-#
-# def __repr__(self):
-# return "<TweetExclude(id=%d)>" % (self.id)
+from lxml import etree
-LDT_CONTENT_REST_API_PATH = "api/ldt/1.0/contents/"
-LDT_PROJECT_REST_API_PATH = "api/ldt/1.0/projects/"
-DEFAULT_ANNOTATION_CHANNEL = 'ANNOT'
-
-def parse_date(datestr):
- res = parse_date_raw(datestr)
- if res.tzinfo is None:
- res = res.replace(tzinfo=tzutc())
- return res
+import utils
def get_logger():
return logging.getLogger(__name__)
-
-def get_filter(start_date, end_date, events, channels, user_whitelist):
- res = []
- #TODO: check timezone...
- if start_date:
- res.append({'name': 'ts', 'op': ">=", 'val':start_date.isoformat() })
- if end_date:
- res.append({'name': 'ts', 'op': "<=", 'val':end_date.isoformat() })
- if events:
- res.append({'name': 'event_code', 'op': "in", 'val':events })
- if channels:
- res.append({'name': 'channel', 'op': "in", 'val':channels })
- if user_whitelist:
- res.append({'name': 'user', 'op': "in", 'val':user_whitelist })
- return res
-
-
def set_logging(options, plogger=None, queue=None):
logging_config = {
@@ -124,12 +84,10 @@
help="video url", metavar="VIDEO")
parser.add_argument("-i", "--content-id", dest="content_id",
help="Content id", metavar="CONTENT_ID")
- parser.add_argument("-x", "--exclude", dest="exclude",
- help="file containing the id to exclude", metavar="EXCLUDE")
parser.add_argument("-C", "--color", dest="color",
help="Color code", metavar="COLOR", default="16763904")
parser.add_argument("-H", "--channel", dest="channels",
- help="Channel", metavar="CHANNEL", default=[DEFAULT_ANNOTATION_CHANNEL], action="append")
+ help="Channel", metavar="CHANNEL", default=[utils.AnnotationsSynchronizer.DEFAULT_ANNOTATION_CHANNEL], action="append")
parser.add_argument("-E", "--event", dest="events",
help="Event", metavar="EVENT", default=[], action="append")
parser.add_argument("-D", "--duration", dest="duration", type=int,
@@ -160,13 +118,6 @@
return (parser.parse_args(), parser)
-def find_delta(deltas, ts):
- i = bisect.bisect_right(deltas, (ts+1,0))
- if i:
- return deltas[i-1]
- return (0,0)
-
-
def parse_duration(s):
try:
return int(s)
@@ -181,21 +132,6 @@
}
return int(round(datetime.timedelta(**time_params).total_seconds()*1000))
-def build_annotation_iterator(url, params, headers):
- page = 0
- page_nb = 1
- while page < page_nb:
- page += 1
- params['page'] = page
- resp = requests.get(url, params=params, headers=headers)
- if resp.status_code != requests.codes.ok:
- return
- resp_json = resp.json()
- page_nb = resp_json.get('total_pages', 1)
- for item in resp_json.get('objects', []):
- #TODO: add progress log
- yield item
-
if __name__ == "__main__" :
@@ -219,11 +155,6 @@
parser.print_help()
sys.exit(1)
- user_whitelist_file = options.user_whitelist
- user_whitelist = None
-
- annotation_url = options.annot_url
-
if options.listconf:
parameters = []
@@ -235,8 +166,8 @@
params['content_file'] = snode.text
params['content_file_write'] = snode.text
elif snode.tag == "project_id":
- params['content_file'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"
- params['content_file_write'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"
+ params['content_file'] = options.base_url + utils.AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"
+ params['content_file_write'] = options.base_url + utils.AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json"
params['project_id'] = snode.text
elif snode.tag == "start_date":
params['start_date'] = snode.text
@@ -256,7 +187,7 @@
parameters.append(params)
else:
if options.project_id:
- content_file = options.base_url + LDT_PROJECT_REST_API_PATH + options.project_id + "/?format=json"
+ content_file = options.base_url + utils.AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + options.project_id + "/?format=json"
else:
content_file = options.content_file
parameters = [{
@@ -269,255 +200,23 @@
'content_file_write' : content_file,
'project_id' : options.project_id
}]
- post_param = {}
- if options.post_param:
- post_param = json.loads(options.post_param)
for params in parameters:
get_logger().debug("PARAMETERS " + repr(params)) #@UndefinedVariable
- start_date_str = params.get("start_date",None)
- end_date_str = params.get("end_date", None)
- duration = params.get("duration", None)
- content_file = params.get("content_file", None)
- content_file_write = params.get("content_file_write", None)
- channels = list(set(params.get('channels', [DEFAULT_ANNOTATION_CHANNEL])))
- events = list(set(params.get('events', [])))
-
- if user_whitelist_file:
- with open(user_whitelist_file, 'r+') as f:
- user_whitelist = list(set([s.strip() for s in f]))
-
- start_date = None
- if start_date_str:
- start_date= parse_date(start_date_str)
-
-
- root = None
- ensemble_parent = None
-
- #to do : analyse situation ldt or iri ? filename set or not ?
-
- if content_file and content_file.find("http") == 0:
-
- get_logger().debug("url : " + content_file) #@UndefinedVariable
-
- r = requests.get(content_file, params=post_param)
- get_logger().debug("url response " + repr(r) + " content " + repr(r.text)) #@UndefinedVariable
- project = r.json()
- text_match = re.match(r"\<\?\s*xml.*?\?\>(.*)", project['ldt'], re.I|re.S)
- root = etree.fromstring(text_match.group(1) if text_match else project['ldt'])
-
- elif content_file and os.path.exists(content_file):
-
- doc = etree.parse(content_file)
- root = doc.getroot()
-
- content_id = None
-
- if root is None:
-
- root = etree.Element(u"iri")
-
- project = etree.SubElement(root, u"project", {u"abstract":u"Annotations",u"title":u"Annotations", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())})
-
- medias = etree.SubElement(root, u"medias")
- media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(options.content), u"video":unicode(options.video), u"id":unicode(options.content_id), u"extra":u""})
-
- annotations = etree.SubElement(root, u"annotations")
- content = etree.SubElement(annotations, u"content", {u"id":unicode(options.content_id)})
- ensemble_parent = content
-
- content_id = options.content_id
-
-
- if ensemble_parent is None:
- file_type = None
- for node in root:
- if node.tag == "project":
- file_type = "ldt"
- break
- elif node.tag == "head":
- file_type = "iri"
- break
-
- if file_type == "ldt":
- media_nodes = root.xpath("//media")
- if len(media_nodes) > 0:
- media = media_nodes[0]
- annotations_node = root.find(u"annotations")
- if annotations_node is None:
- annotations_node = etree.SubElement(root, u"annotations")
- content_node = annotations_node.find(u"content")
- if content_node is None:
- content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id"))
- ensemble_parent = content_node
- content_id = content_node.get(u"id")
- display_nodes = root.xpath("//displays/display/content[@id='%s']" % content_id)
- if len(display_nodes) == 0:
- get_logger().info("No display node found. Will not update display")
- display_content_node = None
- else:
- display_content_node = display_nodes[0]
-
- elif file_type == "iri":
- body_node = root.find(u"body")
- if body_node is None:
- body_node = etree.SubElement(root, u"body")
- ensembles_node = body_node.find(u"ensembles")
- if ensembles_node is None:
- ensembles_node = etree.SubElement(body_node, u"ensembles")
- ensemble_parent = ensembles_node
- content_id = root.xpath("head/meta[@name='id']/@content")[0]
- display_content_node = None
-
-
- if ensemble_parent is None:
- get_logger().error("Can not process file") #@UndefinedVariable
- sys.exit()
-
- if options.replace:
- for ens in ensemble_parent.iterchildren(tag=u"ensemble"):
- ens_id = ens.get("id","")
- if ens_id.startswith("annot_"):
- ensemble_parent.remove(ens)
- # remove in display nodes
- if display_content_node is not None:
- for cut_display in display_content_node.iterchildren():
- if cut_display.get('idens','') == ens_id:
- display_content_node.remove(cut_display)
-
- ensemble = None
- elements = None
+ sync_args = dict(vars(options))
+ sync_args.pop('cuts')
+ sync_args.pop('verbose')
+ sync_args.pop('quiet')
+ sync_args.pop('debug')
+ sync_args.pop('listconf')
+ sync_args.pop('logfile')
+ sync_args['deltas'] = deltas
+ sync_args['logger'] = get_logger()
+ sync_args.update(params)
- if options.merge:
- for ens in ensemble_parent.findall(u"ensemble"):
- if ens.get('id',"").startswith("annot_"):
- ensemble = ens
- break
- if ensemble is not None:
- elements = ensemble.find(u".//elements")
- decoupage = ensemble.find(u"decoupage")
-
- if ensemble is None or elements is None:
- ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"annot_" + unicode(uuid.uuid4()), u"title":u"Ensemble Annotation", u"author":u"IRI Web", u"abstract":u"Ensemble Annotation"})
- decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"})
-
- etree.SubElement(decoupage, u"title").text = unicode(options.name)
- etree.SubElement(decoupage, u"abstract").text = unicode(options.name)
-
- elements = etree.SubElement(decoupage, u"elements")
-
- ensemble_id = ensemble.get('id', '')
- decoupage_id = decoupage.get('id', '') if decoupage is not None else None
-
- end_date = None
- if end_date_str:
- end_date = parse_date(end_date_str)
- elif start_date and duration:
- end_date = start_date + datetime.timedelta(seconds=duration)
- elif start_date and options.base_url:
- # get duration from api
- content_url = options.base_url + LDT_CONTENT_REST_API_PATH + content_id + "/?format=json"
- get_logger().debug("get duration " + content_url) #@UndefinedVariable
- r = requests.get(content_url, params=post_param)
- get_logger().debug("get duration resp " + repr(r)) #@UndefinedVariable
- duration = int(r.json()['duration'])
- get_logger().debug("get duration " + repr(duration)) #@UndefinedVariable
-
- end_date = start_date + datetime.timedelta(seconds=int(duration/1000))
-
- if end_date and deltas:
- end_date = end_date + datetime.timedelta(milliseconds=deltas[-1][1])
-
-
- filters = get_filter(start_date, end_date, events, channels, user_whitelist)
-
- headers = {'Content-Type': 'application/json'}
-
- params = { 'q':json.dumps({'filters':filters}), 'results_per_page': options.batch_size}
-
-
- for annot in build_annotation_iterator(annotation_url, params, headers):
- annot_ts = parse_date(annot['ts'])
- if start_date is None:
- star_date = annot_ts
- annot_ts_rel = annot_ts-start_date
- annot_ts_rel_milli = int(round(annot_ts_rel.total_seconds()*1000))
- if deltas:
- d = find_delta(deltas, annot_ts_rel_milli)
- if d[1] < 0:
- continue
- else :
- annot_ts_rel_milli -= d[1]
- annot_content = annot.get('content',{'category':'', 'user':None})
-
- username = annot_content.get('user', 'anon.') or 'anon.'
+ get_logger().debug("SYNC ARGS " + repr(sync_args))
- category = annot_content.get('category', None)
- if category is None:
- continue
-
- element = etree.SubElement(elements, u"element" , {u"id":annot.get('uuid', uuid.uuid4()), u"color":unicode(options.color), u"author":unicode(username), u"date":unicode(annot_ts.strftime("%Y/%m/%d")), u"begin": unicode(annot_ts_rel_milli), u"dur":u"0"})
- etree.SubElement(element, u"title").text = unicode(username) + u": " + unicode(category.get('label', category.get('code', '')))
- etree.SubElement(element, u"abstract").text = unicode(category.get('label', category.get('code', '')))
-
- tags_node = etree.SubElement(element, u"tags")
- etree.SubElement(tags_node,u"tag").text = category.get('code', '')
-
- meta_element = etree.SubElement(element, u'meta')
-
- polemics_element = etree.Element(u'polemics')
- etree.SubElement(polemics_element, u'polemic').text = category.get('code', '')
- meta_element.append(polemics_element)
-
- etree.SubElement(meta_element, u"source", attrib={"url":annotation_url + "/" + annot['uuid'], "mimetype":u"application/json"}).text = etree.CDATA(json.dumps(annot))
-
- # sort by tc in
- if options.merge :
- # remove all elements and put them in a array
- # sort them with tc
- #put them back
- elements[:] = sorted(elements,key=lambda n: int(n.get('begin')))
-
- #add to display node
- if display_content_node is not None:
- display_dec = None
- for dec in display_content_node.iterchildren(tag=u"decoupage"):
- if dec.get('idens','') == ensemble_id and dec.get('id', '') == decoupage_id:
- display_dec = dec
- break
- if display_dec is None and ensemble_id and decoupage_id:
- etree.SubElement(display_content_node, u"decoupage", attrib={'idens': ensemble_id, 'id': decoupage_id, 'tagsSelect':''})
-
- output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=False, xml_declaration=True)
-
- if content_file_write and content_file_write.find("http") == 0:
-
- project["ldt"] = output_data
- project['owner'] = project['owner'].replace('%7E','~')
- project['contents'] = [c_url.replace('%7E','~') for c_url in project['contents']]
-
- post_param = {}
- if options.post_param:
- post_param = json.loads(options.post_param)
-
- get_logger().debug("write http " + content_file_write) #@UndefinedVariable
- get_logger().debug("write http " + repr(post_param)) #@UndefinedVariable
- get_logger().debug("write http " + repr(project)) #@UndefinedVariable
- r = requests.put(content_file_write, data=json.dumps(project), headers={'content-type':'application/json'}, params=post_param);
- get_logger().debug("write http " + repr(r) + " content " + r.text) #@UndefinedVariable
- if r.status_code != requests.codes.ok: # @UndefinedVariable
- r.raise_for_status()
- else:
- if content_file_write and os.path.exists(content_file_write):
- dest_file_name = content_file_write
- else:
- dest_file_name = options.filename
-
- get_logger().debug("WRITE : " + dest_file_name) #@UndefinedVariable
- output = open(dest_file_name, "w")
- output.write(output_data)
- output.flush()
- output.close()
+ sync = utils.AnnotationsSynchronizer(**sync_args)
+ sync.export_annotations()