# HG changeset patch # User ymh # Date 1414208581 -7200 # Node ID 658561ea9e6574b6512eb734b40ab4765cb0128a # Parent 175826fb5b4220360601a1efb72938a81fd77bac export sync code in utils, and add button to synchronize annotation on eventsessionview diff -r 175826fb5b42 -r 658561ea9e65 annot-server/settings.py.tmpl --- a/annot-server/settings.py.tmpl Sat Oct 25 05:41:51 2014 +0200 +++ b/annot-server/settings.py.tmpl Sat Oct 25 05:43:01 2014 +0200 @@ -9,3 +9,5 @@ STATIC_URL = '/static' STATIC_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)),'static') LDT_PLATFORM_BASE_URL = 'http://ldt.iri.centrepompidou.fr/ldtplatform' + +SYNC_POST_PARAM = {} diff -r 175826fb5b42 -r 658561ea9e65 annot-server/utils.py --- a/annot-server/utils.py Sat Oct 25 05:41:51 2014 +0200 +++ b/annot-server/utils.py Sat Oct 25 05:43:01 2014 +0200 @@ -4,6 +4,360 @@ # Copyright (c) 2014 IRI # +import bisect +import datetime +import json +import logging +import re +import uuid + +from dateutil.parser import parse as parse_date_raw +from dateutil.tz import tzutc + +from lxml import etree + +import requests + + +logger = logging.getLogger(__name__) PIANOROLL_CHANNEL = 'PIANOROLL' ANNOTATION_CHANNEL = 'ANNOT' + + +class AnnotationsSynchronizer(object): + + LDT_CONTENT_REST_API_PATH = "api/ldt/1.0/contents/" + LDT_PROJECT_REST_API_PATH = "api/ldt/1.0/projects/" + DEFAULT_ANNOTATION_CHANNEL = 'ANNOT' + + def parse_date(self, datestr): + res = parse_date_raw(datestr) + if res.tzinfo is None: + res = res.replace(tzinfo=tzutc()) + return res + + def find_delta(self, ts): + if self.deltas: + i = bisect.bisect_right(self.deltas, (ts+1,0)) + if i: + return self.deltas[i-1] + return (0,0) + + + def get_filter(self): + res = [] + if self.start_date: + res.append({'name': 'ts', 'op': ">=", 'val':self.start_date.isoformat() }) + if self.end_date: + res.append({'name': 'ts', 'op': "<=", 'val':self.end_date.isoformat() }) + if self.events: + res.append({'name': 'event_code', 'op': "in", 'val': self.events }) + if self.channels: + res.append({'name': 'channel', 'op': "in", 'val': self.channels }) + if self.user_whitelist: + res.append({'name': 'user', 'op': "in", 'val': self.user_whitelist }) + return res + + + def build_annotation_iterator(self, params): + page = 0 + page_nb = 1 + while page < page_nb: + page += 1 + params['page'] = page + resp = requests.get(self.annot_url, params=params, headers={'Content-Type': 'application/json'}) + if resp.status_code != requests.codes.ok: + self.logger.debug("build_annotation_iterator : request %s : return code %r " % (resp.request.url, resp.status_code)) + return + resp_json = resp.json() + page_nb = resp_json.get('total_pages', 1) + for item in resp_json.get('objects', []): + #TODO: add progress log + yield item + + + + def __init__(self, start_date=None, end_date=None, duration=None, content_file=None, + content_file_write=None, project_id=None, + channels=[DEFAULT_ANNOTATION_CHANNEL], events=[], annot_url=None, + user_whitelist=None, post_param={}, deltas=None, base_url="http://ldt.iri.centrepompidou.fr/ldtplatform/", + content=None, content_id=None, video=None, + replace=True, merge=False, name="", batch_size=500, + filename="project.ldt", color="16763904", logger=logger): + + self.logger = logger + self.base_url = base_url.rstrip("/")+"/" if base_url else base_url + self.deltas = deltas + self.post_param = {} + if isinstance(post_param, basestring): + self.post_param = json.loads(post_param) + elif post_param: + self.post_param = post_param + + self.start_date = None + if start_date: + if isinstance(start_date, datetime.datetime): + self.start_date = start_date + else: + self.start_date = self.parse_date(str(start_date)) + + self.duration = duration + self.end_date = None + if end_date: + if isinstance(end_date, datetime.datetime): + self.end_date = end_date + else: + self.end_date= self.parse_date(str(end_date)) + + if end_date: + if isinstance(end_date, datetime.datetime): + self.end_date = end_date + else: + self.end_date= self.parse_date(str(end_date)) + elif self.start_date and self.duration: + self.end_date = self.start_date + datetime.timedelta(seconds=self.duration) + elif self.start_date and self.base_url: + # get duration from api + content_url = self.base_url + AnnotationsSynchronizer.LDT_CONTENT_REST_API_PATH + content_id + "/?format=json" + self.logger.debug("get duration " + content_url) #@UndefinedVariable + r = requests.get(content_url, params=self.post_param) + self.logger.debug("get duration resp " + repr(r)) #@UndefinedVariable + self.duration = int(r.json()['duration']) + self.logger.debug("get duration " + repr(self.duration)) #@UndefinedVariable + + self.end_date = self.start_date + datetime.timedelta(seconds=int(self.duration/1000)) + + if self.end_date and self.deltas: + self.end_date = self.end_date + datetime.timedelta(milliseconds=self.deltas[-1][1]) + + self.content_file = content_file + self.project_id = project_id + if self.project_id is not None: + self.content_file = self.base_url + AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + self.project_id + "/?format=json" + self.content_file_write = content_file_write + if self.content_file_write is None and self.project_id: + self.content_file_write = self.content_file + self.channels = list(set(channels)) + self.annot_url = annot_url + self.events = list(set(events)) + self.user_whitelist_file = user_whitelist + if self.user_whitelist_file: + with open(user_whitelist_file, 'r+') as f: + self.user_whitelist = list(set([s.strip() for s in f])) + else: + self.user_whitelist = None + self.content = content + self.content_id = content_id + self.video = video + self.replace = replace + self.merge = merge + self.name = name + self.batch_size = batch_size + self.filename = filename + self.color = color + + def export_annotations(self): + + root = None + ensemble_parent = None + + #to do : analyse situation ldt or iri ? filename set or not ? + + if self.content_file and self.content_file.find("http://") == 0: + + self.logger.debug("url : " + self.content_file) #@UndefinedVariable + + r = requests.get(self.content_file, params=self.post_param) + self.logger.debug("url response " + repr(r) + " content " + repr(r.text)) #@UndefinedVariable + project = r.json() + text_match = re.match(r"\<\?\s*xml.*?\?\>(.*)", project['ldt'], re.I|re.S) + root = etree.fromstring(text_match.group(1) if text_match else project['ldt']) + + elif self.content_file and os.path.exists(self.content_file): + + doc = etree.parse(self.content_file) + root = doc.getroot() + + content_id = None + + if root is None: + + root = etree.Element(u"iri") + + project = etree.SubElement(root, u"project", {u"abstract":u"Annotations",u"title":u"Annotations", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())}) + + medias = etree.SubElement(root, u"medias") + media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(self.content), u"video":unicode(self.video), u"id":unicode(self.content_id), u"extra":u""}) + + annotations = etree.SubElement(root, u"annotations") + content = etree.SubElement(annotations, u"content", {u"id":unicode(self.content_id)}) + ensemble_parent = content + + content_id = self.content_id + + + if ensemble_parent is None: + file_type = None + for node in root: + if node.tag == "project": + file_type = "ldt" + break + elif node.tag == "head": + file_type = "iri" + break + + if file_type == "ldt": + media_nodes = root.xpath("//media") + if len(media_nodes) > 0: + media = media_nodes[0] + annotations_node = root.find(u"annotations") + if annotations_node is None: + annotations_node = etree.SubElement(root, u"annotations") + content_node = annotations_node.find(u"content") + if content_node is None: + content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id")) + ensemble_parent = content_node + content_id = content_node.get(u"id") + display_nodes = root.xpath("//displays/display/content[@id='%s']" % content_id) + if len(display_nodes) == 0: + self.logger.info("No display node found. Will not update display") + display_content_node = None + else: + display_content_node = display_nodes[0] + + elif file_type == "iri": + body_node = root.find(u"body") + if body_node is None: + body_node = etree.SubElement(root, u"body") + ensembles_node = body_node.find(u"ensembles") + if ensembles_node is None: + ensembles_node = etree.SubElement(body_node, u"ensembles") + ensemble_parent = ensembles_node + content_id = root.xpath("head/meta[@name='id']/@content")[0] + display_content_node = None + + + if ensemble_parent is None: + self.logger.error("Can not process file") #@UndefinedVariable + sys.exit() + + if self.replace: + for ens in ensemble_parent.iterchildren(tag=u"ensemble"): + ens_id = ens.get("id","") + if ens_id.startswith("annot_"): + ensemble_parent.remove(ens) + # remove in display nodes + if display_content_node is not None: + for cut_display in display_content_node.iterchildren(): + if cut_display.get('idens','') == ens_id: + display_content_node.remove(cut_display) + + ensemble = None + elements = None + + if self.merge: + for ens in ensemble_parent.findall(u"ensemble"): + if ens.get('id',"").startswith("annot_"): + ensemble = ens + break + if ensemble is not None: + elements = ensemble.find(u".//elements") + decoupage = ensemble.find(u"decoupage") + + if ensemble is None or elements is None: + ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"annot_" + unicode(uuid.uuid4()), u"title":u"Ensemble Annotation", u"author":u"IRI Web", u"abstract":u"Ensemble Annotation"}) + decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"}) + + etree.SubElement(decoupage, u"title").text = unicode(self.name) + etree.SubElement(decoupage, u"abstract").text = unicode(self.name) + + elements = etree.SubElement(decoupage, u"elements") + + ensemble_id = ensemble.get('id', '') + decoupage_id = decoupage.get('id', '') if decoupage is not None else None + + + filters = self.get_filter() + + + params = { 'q':json.dumps({'filters':filters}), 'results_per_page': self.batch_size} + + + for annot in self.build_annotation_iterator(params): + annot_ts = self.parse_date(annot['ts']) + if self.start_date is None: + star_date = annot_ts + annot_ts_rel = annot_ts-self.start_date + annot_ts_rel_milli = int(round(annot_ts_rel.total_seconds()*1000)) + d = self.find_delta(annot_ts_rel_milli) + if d[1] < 0: + continue + else : + annot_ts_rel_milli -= d[1] + annot_content = annot.get('content',{'category':'', 'user':None}) + + username = annot_content.get('user', 'anon.') or 'anon.' + + category = annot_content.get('category', None) + if category is None: + continue + + element = etree.SubElement(elements, u"element" , {u"id":annot.get('uuid', uuid.uuid4()), u"color":unicode(self.color), u"author":unicode(username), u"date":unicode(annot_ts.strftime("%Y/%m/%d")), u"begin": unicode(annot_ts_rel_milli), u"dur":u"0"}) + etree.SubElement(element, u"title").text = unicode(username) + u": " + unicode(category.get('label', category.get('code', ''))) + etree.SubElement(element, u"abstract").text = unicode(category.get('label', category.get('code', ''))) + + tags_node = etree.SubElement(element, u"tags") + etree.SubElement(tags_node,u"tag").text = category.get('code', '') + + meta_element = etree.SubElement(element, u'meta') + + polemics_element = etree.Element(u'polemics') + etree.SubElement(polemics_element, u'polemic').text = category.get('code', '') + meta_element.append(polemics_element) + + etree.SubElement(meta_element, u"source", attrib={"url":self.annot_url + "/" + annot['uuid'], "mimetype":u"application/json"}).text = etree.CDATA(json.dumps(annot)) + + # sort by tc in + if self.merge : + # remove all elements and put them in a array + # sort them with tc + #put them back + elements[:] = sorted(elements,key=lambda n: int(n.get('begin'))) + + #add to display node + if display_content_node is not None: + display_dec = None + for dec in display_content_node.iterchildren(tag=u"decoupage"): + if dec.get('idens','') == ensemble_id and dec.get('id', '') == decoupage_id: + display_dec = dec + break + if display_dec is None and ensemble_id and decoupage_id: + etree.SubElement(display_content_node, u"decoupage", attrib={'idens': ensemble_id, 'id': decoupage_id, 'tagsSelect':''}) + + output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=False, xml_declaration=True) + + if self.content_file_write and self.content_file_write.find("http://") == 0: + + project["ldt"] = output_data + project['owner'] = project['owner'].replace('%7E','~') + project['contents'] = [c_url.replace('%7E','~') for c_url in project['contents']] + + self.logger.debug("write http " + self.content_file_write) #@UndefinedVariable + self.logger.debug("write http " + repr(self.post_param)) #@UndefinedVariable + self.logger.debug("write http " + repr(project)) #@UndefinedVariable + r = requests.put(self.content_file_write, data=json.dumps(project), headers={'content-type':'application/json'}, params=self.post_param); + self.logger.debug("write http " + repr(r) + " content " + r.text) #@UndefinedVariable + if r.status_code != requests.codes.ok: # @UndefinedVariable + r.raise_for_status() + else: + if self.content_file_write and os.path.exists(self.content_file_write): + dest_file_name = self.content_file_write + else: + dest_file_name = self.filename + + self.logger.debug("WRITE : " + dest_file_name) #@UndefinedVariable + output = open(dest_file_name, "w") + output.write(output_data) + output.flush() + output.close() diff -r 175826fb5b42 -r 658561ea9e65 annot-server/webapp/admin.py --- a/annot-server/webapp/admin.py Sat Oct 25 05:41:51 2014 +0200 +++ b/annot-server/webapp/admin.py Sat Oct 25 05:43:01 2014 +0200 @@ -4,11 +4,14 @@ # import json +import logging +from StringIO import StringIO import urllib from dateutil.tz import tzutc -from flask.ext.admin import Admin +from flask import url_for, render_template, request +from flask.ext.admin import Admin, BaseView, expose from flask.ext.admin.contrib.sqla import ModelView from flask.ext.admin.contrib.sqla.form import AdminModelConverter from flask.ext.admin.contrib.sqla.filters import BooleanEqualFilter @@ -19,9 +22,10 @@ from wtforms import fields from wtforms.ext.dateutil import fields as dateutil_fields +import config import database import models -import config +import utils from webapp import app admin = Admin(app) @@ -41,10 +45,6 @@ output = [] for c in model.categories_json.get("categories",[]): - #if "subcategories" in c and len(c["subcategories"])>0: - # for sc in c["subcategories"]: - # output.append({"name": sc["code"],"keywords": [ sc["label"] ], "color" : sc["color"] }) - #else: output.append({"name": c["code"] if "code" in c else "","keywords": [ c["label"] ], "color" : c["color"] }) base_url = getattr(config,'LDT_PLATFORM_BASE_URL','') + "/ldt/embediframe/?" @@ -62,6 +62,21 @@ return Markup("Lien :

Code embed :" % (url, url, embed)) +def build_sync_form(view, context, model, name): + # `view` is current administrative view + # `context` is instance of jinja2.runtime.Context + # `model` is model instance + # `name` is property name + if not model.project_id \ + or not model.start_ts \ + or not getattr(config,'LDT_PLATFORM_BASE_URL',None): + return '' + + url = url_for('eventsessionview.sync') + + return Markup(render_template('admin/sync_form.html', model=model, url=url)) + + class StartTSField(dateutil_fields.DateTimeField): def __init__(self, *args, **kwargs): if 'display_format' not in kwargs: @@ -118,9 +133,13 @@ column_default_sort = ( models.Event.code, True) column_sortable_list = (('event', models.Event.code),) column_auto_select_related = True - column_list = ('event', 'project_id', 'order', 'categories_json', 'url') + column_list = ('event', 'project_id', 'order', 'categories_json', 'url', 'sync') column_filters = ('event',) - column_formatters = {'url': build_ldt_url, 'categories_json': lambda v, c, m, p: json.dumps(m.categories_json)[:80] if m.categories_json else ''} + column_formatters = { + 'sync': build_sync_form, + 'url': build_ldt_url, + 'categories_json': lambda v, c, m, p: json.dumps(m.categories_json)[:80] if m.categories_json else '' + } form_overrides = { "start_ts": StartTSField, } form_widget_args = { 'categories_json': { @@ -140,6 +159,41 @@ def __init__(self, session, **kwargs): super(EventSessionView, self).__init__(models.EventSession, session, **kwargs) + @expose("/sync/",methods=('POST', )) + def sync(self): + event_session = database.db_session.query(models.EventSession).filter_by(id=int(request.form['event_session_id'])).first() + + if not event_session: + abort(404) + + stream = StringIO() + handler = logging.StreamHandler(stream) + logger = logging.getLogger('sync_logger') + logger.setLevel(logging.DEBUG) + for h in logger.handlers: + logger.removeHandler(h) + logger.addHandler(handler) + + sync_args = { + 'start_date' : event_session.start_ts, + 'duration' : event_session.duration or None, + 'events' : [event_session.event.code], + 'channels' : [utils.AnnotationsSynchronizer.DEFAULT_ANNOTATION_CHANNEL], + 'project_id' : event_session.project_id, + 'annot_url' : url_for('annotationapi0.annotationapi', _external=True), + 'post_param' : getattr(config, 'SYNC_POST_PARAM', {}), + 'base_url' : getattr(config, 'LDT_PLATFORM_BASE_URL', "http://ldt.iri.centrepompidou.fr/ldtplatform/"), + 'replace' : True, + 'merge' : False, + 'batch_size' : 500, + 'logger' : logger, + } + + sync = utils.AnnotationsSynchronizer(**sync_args) + sync.export_annotations() + logs = [ line for line in stream.getvalue().split("\n")] + + return self.render('admin/sync_event_session.html', event_session=event_session, sync_args=repr(sync_args), logs=logs) admin.add_view(EventView(database.db_session)) diff -r 175826fb5b42 -r 658561ea9e65 annot-server/webapp/templates/admin/sync_event_session.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/annot-server/webapp/templates/admin/sync_event_session.html Sat Oct 25 05:43:01 2014 +0200 @@ -0,0 +1,17 @@ +{% extends 'admin/master.html' %} +{% block body %} +

Sync on project {{event_session.project_id}}

+ +

Sync arguments

+ 
+        {{sync_args}}
+
+ +

Logs

+ 
+        {% for line in logs %}
+        {{line}}

+        {% endfor %}
+
+ +{% endblock %} diff -r 175826fb5b42 -r 658561ea9e65 annot-server/webapp/templates/admin/sync_form.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/annot-server/webapp/templates/admin/sync_form.html Sat Oct 25 05:43:01 2014 +0200 @@ -0,0 +1,4 @@ +
+ + +
diff -r 175826fb5b42 -r 658561ea9e65 annot-server/websockets.py --- a/annot-server/websockets.py Sat Oct 25 05:41:51 2014 +0200 +++ b/annot-server/websockets.py Sat Oct 25 05:43:01 2014 +0200 @@ -79,6 +79,12 @@ self.event = event_ids[0] self._init_props(self.factory.ws_factory, self.factory.conn) + def clean_annot(self, annot): + if 'ts' in annot: + annot['ts'] = annot['ts'].isoformat() + if 'uuid' in annot: + annot['uuid'] = str(annot['uuid']) + #TODO: add error handling def onMessage(self, payload, isBinary): if isBinary: @@ -94,15 +100,13 @@ def error_callback(failure): res = params.copy() + self.clean_annot(res) res['status'] = 'KO' res['failure'] = str(failure) self.sendMessage(json.dumps(res)) def annot_callback(res): - if 'ts' in res: - res['ts'] = res['ts'].isoformat()+'Z' - if 'uuid' in res: - res['uuid'] = str(res['uuid']) + self.clean_annot(res) res['status'] = 'OK' self.sendMessage(json.dumps(res)) diff -r 175826fb5b42 -r 658561ea9e65 client/gulpfile.js --- a/client/gulpfile.js Sat Oct 25 05:41:51 2014 +0200 +++ b/client/gulpfile.js Sat Oct 25 05:43:01 2014 +0200 @@ -59,6 +59,7 @@ '!./bower_components/jquery/src/**/*', '!./bower_components/angular-bootstrap/ui-bootstrap.js', './bower_components/jquery/dist/jquery.js', + './bower_components/angular/angular.js', './bower_components/**/*.js' ]; diff -r 175826fb5b42 -r 658561ea9e65 utils/export_annotations.py --- a/utils/export_annotations.py Sat Oct 25 05:41:51 2014 +0200 +++ b/utils/export_annotations.py Sat Oct 25 05:43:01 2014 +0200 @@ -1,58 +1,18 @@ #!/usr/bin/env python # coding=utf-8 -from lxml import etree import argparse -import json import datetime -import requests -import os.path -import re import sys -import time -import uuid #@UnresolvedImport -from dateutil.parser import parse as parse_date_raw -from dateutil.tz import tzutc -import bisect import logging -#class TweetExclude(object): -# def __init__(self, id): -# self.id = id -# -# def __repr__(self): -# return "" % (self.id) +from lxml import etree -LDT_CONTENT_REST_API_PATH = "api/ldt/1.0/contents/" -LDT_PROJECT_REST_API_PATH = "api/ldt/1.0/projects/" -DEFAULT_ANNOTATION_CHANNEL = 'ANNOT' - -def parse_date(datestr): - res = parse_date_raw(datestr) - if res.tzinfo is None: - res = res.replace(tzinfo=tzutc()) - return res +import utils def get_logger(): return logging.getLogger(__name__) - -def get_filter(start_date, end_date, events, channels, user_whitelist): - res = [] - #TODO: check timezone... - if start_date: - res.append({'name': 'ts', 'op': ">=", 'val':start_date.isoformat() }) - if end_date: - res.append({'name': 'ts', 'op': "<=", 'val':end_date.isoformat() }) - if events: - res.append({'name': 'event_code', 'op': "in", 'val':events }) - if channels: - res.append({'name': 'channel', 'op': "in", 'val':channels }) - if user_whitelist: - res.append({'name': 'user', 'op': "in", 'val':user_whitelist }) - return res - - def set_logging(options, plogger=None, queue=None): logging_config = { @@ -124,12 +84,10 @@ help="video url", metavar="VIDEO") parser.add_argument("-i", "--content-id", dest="content_id", help="Content id", metavar="CONTENT_ID") - parser.add_argument("-x", "--exclude", dest="exclude", - help="file containing the id to exclude", metavar="EXCLUDE") parser.add_argument("-C", "--color", dest="color", help="Color code", metavar="COLOR", default="16763904") parser.add_argument("-H", "--channel", dest="channels", - help="Channel", metavar="CHANNEL", default=[DEFAULT_ANNOTATION_CHANNEL], action="append") + help="Channel", metavar="CHANNEL", default=[utils.AnnotationsSynchronizer.DEFAULT_ANNOTATION_CHANNEL], action="append") parser.add_argument("-E", "--event", dest="events", help="Event", metavar="EVENT", default=[], action="append") parser.add_argument("-D", "--duration", dest="duration", type=int, @@ -160,13 +118,6 @@ return (parser.parse_args(), parser) -def find_delta(deltas, ts): - i = bisect.bisect_right(deltas, (ts+1,0)) - if i: - return deltas[i-1] - return (0,0) - - def parse_duration(s): try: return int(s) @@ -181,21 +132,6 @@ } return int(round(datetime.timedelta(**time_params).total_seconds()*1000)) -def build_annotation_iterator(url, params, headers): - page = 0 - page_nb = 1 - while page < page_nb: - page += 1 - params['page'] = page - resp = requests.get(url, params=params, headers=headers) - if resp.status_code != requests.codes.ok: - return - resp_json = resp.json() - page_nb = resp_json.get('total_pages', 1) - for item in resp_json.get('objects', []): - #TODO: add progress log - yield item - if __name__ == "__main__" : @@ -219,11 +155,6 @@ parser.print_help() sys.exit(1) - user_whitelist_file = options.user_whitelist - user_whitelist = None - - annotation_url = options.annot_url - if options.listconf: parameters = [] @@ -235,8 +166,8 @@ params['content_file'] = snode.text params['content_file_write'] = snode.text elif snode.tag == "project_id": - params['content_file'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json" - params['content_file_write'] = options.base_url + LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json" + params['content_file'] = options.base_url + utils.AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json" + params['content_file_write'] = options.base_url + utils.AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + snode.text + "/?format=json" params['project_id'] = snode.text elif snode.tag == "start_date": params['start_date'] = snode.text @@ -256,7 +187,7 @@ parameters.append(params) else: if options.project_id: - content_file = options.base_url + LDT_PROJECT_REST_API_PATH + options.project_id + "/?format=json" + content_file = options.base_url + utils.AnnotationsSynchronizer.LDT_PROJECT_REST_API_PATH + options.project_id + "/?format=json" else: content_file = options.content_file parameters = [{ @@ -269,255 +200,23 @@ 'content_file_write' : content_file, 'project_id' : options.project_id }] - post_param = {} - if options.post_param: - post_param = json.loads(options.post_param) for params in parameters: get_logger().debug("PARAMETERS " + repr(params)) #@UndefinedVariable - start_date_str = params.get("start_date",None) - end_date_str = params.get("end_date", None) - duration = params.get("duration", None) - content_file = params.get("content_file", None) - content_file_write = params.get("content_file_write", None) - channels = list(set(params.get('channels', [DEFAULT_ANNOTATION_CHANNEL]))) - events = list(set(params.get('events', []))) - - if user_whitelist_file: - with open(user_whitelist_file, 'r+') as f: - user_whitelist = list(set([s.strip() for s in f])) - - start_date = None - if start_date_str: - start_date= parse_date(start_date_str) - - - root = None - ensemble_parent = None - - #to do : analyse situation ldt or iri ? filename set or not ? - - if content_file and content_file.find("http") == 0: - - get_logger().debug("url : " + content_file) #@UndefinedVariable - - r = requests.get(content_file, params=post_param) - get_logger().debug("url response " + repr(r) + " content " + repr(r.text)) #@UndefinedVariable - project = r.json() - text_match = re.match(r"\<\?\s*xml.*?\?\>(.*)", project['ldt'], re.I|re.S) - root = etree.fromstring(text_match.group(1) if text_match else project['ldt']) - - elif content_file and os.path.exists(content_file): - - doc = etree.parse(content_file) - root = doc.getroot() - - content_id = None - - if root is None: - - root = etree.Element(u"iri") - - project = etree.SubElement(root, u"project", {u"abstract":u"Annotations",u"title":u"Annotations", u"user":u"IRI Web", u"id":unicode(uuid.uuid4())}) - - medias = etree.SubElement(root, u"medias") - media = etree.SubElement(medias, u"media", {u"pict":u"", u"src":unicode(options.content), u"video":unicode(options.video), u"id":unicode(options.content_id), u"extra":u""}) - - annotations = etree.SubElement(root, u"annotations") - content = etree.SubElement(annotations, u"content", {u"id":unicode(options.content_id)}) - ensemble_parent = content - - content_id = options.content_id - - - if ensemble_parent is None: - file_type = None - for node in root: - if node.tag == "project": - file_type = "ldt" - break - elif node.tag == "head": - file_type = "iri" - break - - if file_type == "ldt": - media_nodes = root.xpath("//media") - if len(media_nodes) > 0: - media = media_nodes[0] - annotations_node = root.find(u"annotations") - if annotations_node is None: - annotations_node = etree.SubElement(root, u"annotations") - content_node = annotations_node.find(u"content") - if content_node is None: - content_node = etree.SubElement(annotations_node,u"content", id=media.get(u"id")) - ensemble_parent = content_node - content_id = content_node.get(u"id") - display_nodes = root.xpath("//displays/display/content[@id='%s']" % content_id) - if len(display_nodes) == 0: - get_logger().info("No display node found. Will not update display") - display_content_node = None - else: - display_content_node = display_nodes[0] - - elif file_type == "iri": - body_node = root.find(u"body") - if body_node is None: - body_node = etree.SubElement(root, u"body") - ensembles_node = body_node.find(u"ensembles") - if ensembles_node is None: - ensembles_node = etree.SubElement(body_node, u"ensembles") - ensemble_parent = ensembles_node - content_id = root.xpath("head/meta[@name='id']/@content")[0] - display_content_node = None - - - if ensemble_parent is None: - get_logger().error("Can not process file") #@UndefinedVariable - sys.exit() - - if options.replace: - for ens in ensemble_parent.iterchildren(tag=u"ensemble"): - ens_id = ens.get("id","") - if ens_id.startswith("annot_"): - ensemble_parent.remove(ens) - # remove in display nodes - if display_content_node is not None: - for cut_display in display_content_node.iterchildren(): - if cut_display.get('idens','') == ens_id: - display_content_node.remove(cut_display) - - ensemble = None - elements = None + sync_args = dict(vars(options)) + sync_args.pop('cuts') + sync_args.pop('verbose') + sync_args.pop('quiet') + sync_args.pop('debug') + sync_args.pop('listconf') + sync_args.pop('logfile') + sync_args['deltas'] = deltas + sync_args['logger'] = get_logger() + sync_args.update(params) - if options.merge: - for ens in ensemble_parent.findall(u"ensemble"): - if ens.get('id',"").startswith("annot_"): - ensemble = ens - break - if ensemble is not None: - elements = ensemble.find(u".//elements") - decoupage = ensemble.find(u"decoupage") - - if ensemble is None or elements is None: - ensemble = etree.SubElement(ensemble_parent, u"ensemble", {u"id":u"annot_" + unicode(uuid.uuid4()), u"title":u"Ensemble Annotation", u"author":u"IRI Web", u"abstract":u"Ensemble Annotation"}) - decoupage = etree.SubElement(ensemble, u"decoupage", {u"id": unicode(uuid.uuid4()), u"author": u"IRI Web"}) - - etree.SubElement(decoupage, u"title").text = unicode(options.name) - etree.SubElement(decoupage, u"abstract").text = unicode(options.name) - - elements = etree.SubElement(decoupage, u"elements") - - ensemble_id = ensemble.get('id', '') - decoupage_id = decoupage.get('id', '') if decoupage is not None else None - - end_date = None - if end_date_str: - end_date = parse_date(end_date_str) - elif start_date and duration: - end_date = start_date + datetime.timedelta(seconds=duration) - elif start_date and options.base_url: - # get duration from api - content_url = options.base_url + LDT_CONTENT_REST_API_PATH + content_id + "/?format=json" - get_logger().debug("get duration " + content_url) #@UndefinedVariable - r = requests.get(content_url, params=post_param) - get_logger().debug("get duration resp " + repr(r)) #@UndefinedVariable - duration = int(r.json()['duration']) - get_logger().debug("get duration " + repr(duration)) #@UndefinedVariable - - end_date = start_date + datetime.timedelta(seconds=int(duration/1000)) - - if end_date and deltas: - end_date = end_date + datetime.timedelta(milliseconds=deltas[-1][1]) - - - filters = get_filter(start_date, end_date, events, channels, user_whitelist) - - headers = {'Content-Type': 'application/json'} - - params = { 'q':json.dumps({'filters':filters}), 'results_per_page': options.batch_size} - - - for annot in build_annotation_iterator(annotation_url, params, headers): - annot_ts = parse_date(annot['ts']) - if start_date is None: - star_date = annot_ts - annot_ts_rel = annot_ts-start_date - annot_ts_rel_milli = int(round(annot_ts_rel.total_seconds()*1000)) - if deltas: - d = find_delta(deltas, annot_ts_rel_milli) - if d[1] < 0: - continue - else : - annot_ts_rel_milli -= d[1] - annot_content = annot.get('content',{'category':'', 'user':None}) - - username = annot_content.get('user', 'anon.') or 'anon.' + get_logger().debug("SYNC ARGS " + repr(sync_args)) - category = annot_content.get('category', None) - if category is None: - continue - - element = etree.SubElement(elements, u"element" , {u"id":annot.get('uuid', uuid.uuid4()), u"color":unicode(options.color), u"author":unicode(username), u"date":unicode(annot_ts.strftime("%Y/%m/%d")), u"begin": unicode(annot_ts_rel_milli), u"dur":u"0"}) - etree.SubElement(element, u"title").text = unicode(username) + u": " + unicode(category.get('label', category.get('code', ''))) - etree.SubElement(element, u"abstract").text = unicode(category.get('label', category.get('code', ''))) - - tags_node = etree.SubElement(element, u"tags") - etree.SubElement(tags_node,u"tag").text = category.get('code', '') - - meta_element = etree.SubElement(element, u'meta') - - polemics_element = etree.Element(u'polemics') - etree.SubElement(polemics_element, u'polemic').text = category.get('code', '') - meta_element.append(polemics_element) - - etree.SubElement(meta_element, u"source", attrib={"url":annotation_url + "/" + annot['uuid'], "mimetype":u"application/json"}).text = etree.CDATA(json.dumps(annot)) - - # sort by tc in - if options.merge : - # remove all elements and put them in a array - # sort them with tc - #put them back - elements[:] = sorted(elements,key=lambda n: int(n.get('begin'))) - - #add to display node - if display_content_node is not None: - display_dec = None - for dec in display_content_node.iterchildren(tag=u"decoupage"): - if dec.get('idens','') == ensemble_id and dec.get('id', '') == decoupage_id: - display_dec = dec - break - if display_dec is None and ensemble_id and decoupage_id: - etree.SubElement(display_content_node, u"decoupage", attrib={'idens': ensemble_id, 'id': decoupage_id, 'tagsSelect':''}) - - output_data = etree.tostring(root, encoding="utf-8", method="xml", pretty_print=False, xml_declaration=True) - - if content_file_write and content_file_write.find("http") == 0: - - project["ldt"] = output_data - project['owner'] = project['owner'].replace('%7E','~') - project['contents'] = [c_url.replace('%7E','~') for c_url in project['contents']] - - post_param = {} - if options.post_param: - post_param = json.loads(options.post_param) - - get_logger().debug("write http " + content_file_write) #@UndefinedVariable - get_logger().debug("write http " + repr(post_param)) #@UndefinedVariable - get_logger().debug("write http " + repr(project)) #@UndefinedVariable - r = requests.put(content_file_write, data=json.dumps(project), headers={'content-type':'application/json'}, params=post_param); - get_logger().debug("write http " + repr(r) + " content " + r.text) #@UndefinedVariable - if r.status_code != requests.codes.ok: # @UndefinedVariable - r.raise_for_status() - else: - if content_file_write and os.path.exists(content_file_write): - dest_file_name = content_file_write - else: - dest_file_name = options.filename - - get_logger().debug("WRITE : " + dest_file_name) #@UndefinedVariable - output = open(dest_file_name, "w") - output.write(output_data) - output.flush() - output.close() + sync = utils.AnnotationsSynchronizer(**sync_args) + sync.export_annotations()