# HG changeset patch # User cavaliet # Date 1352741589 -3600 # Node ID a227bbd1d2c7430150d057d6ef9e1d85e865c5df # Parent 3a47d47ae9522b2a61e2c9cdf3d94ae1ac5462e4# Parent 7d0b9d12bf5aba6ba6189cdc6e91532e83a4e3a3 Merge with 7d0b9d12bf5aba6ba6189cdc6e91532e83a4e3a3 diff -r 7d0b9d12bf5a -r a227bbd1d2c7 .hgignore --- a/.hgignore Mon Nov 12 16:37:57 2012 +0100 +++ b/.hgignore Mon Nov 12 18:33:09 2012 +0100 @@ -33,3 +33,6 @@ syntax: regexp ^virtualenv/sync/project-boot\.py$ relre:^.metadata + +syntax: regexp +^web/\.htusers$ \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 .hgtags --- a/.hgtags Mon Nov 12 16:37:57 2012 +0100 +++ b/.hgtags Mon Nov 12 18:33:09 2012 +0100 @@ -112,3 +112,13 @@ 36d8d68fda811a9020dfcb4794ad42403ab7806b V01.26 3eae57bb42b3d5c8daf3e94331f9ef30cf444693 V01.26 2151b47c58c3291f82a2e190c474e7f616266742 V01.27 +ba611074ecb2c86907386b1c76c87f38e35aa6f0 V01.28 +ba611074ecb2c86907386b1c76c87f38e35aa6f0 V01.28 +be9868b4b13dbbf18bb82c5cad5008e5a540f43b V01.28 +be9868b4b13dbbf18bb82c5cad5008e5a540f43b V01.28 +ba43c842648c3d997157fae50693af90680e9875 V01.28 +ba43c842648c3d997157fae50693af90680e9875 V01.28 +7be50692abb4ad436eb668f20732dd79449c199f V01.28 +7be50692abb4ad436eb668f20732dd79449c199f V01.28 +e20fa653364745615a79f887eb6b77dfd58b171d V01.28 +d42fbe878062cf9765b8f968c5aad2161c479cd5 V01.29 diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/README --- a/src/ldt/README Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/README Mon Nov 12 18:33:09 2012 +0100 @@ -5,7 +5,6 @@ The ldt platform, is a django module allowing the annotation, indexation, consultation of temporal content (audio, video) - trick: compile messages for js python ../../../web/ldtplatform/manage.py makemessages -a -d djangojs @@ -62,3 +61,16 @@ swfobject.js ZeroClipboard tiny_mce + + +============== +Unit Test +============== + +In a terminal : + python manage.py test %test_api% +Will launch all the tests of the api + +For example : + python manage.py test ldt_utils +Will launch all the test defined in /ldt_utils/tests \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/__init__.py --- a/src/ldt/ldt/__init__.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/__init__.py Mon Nov 12 18:33:09 2012 +0100 @@ -1,4 +1,4 @@ -VERSION = (1, 27, 0, "final", 0) +VERSION = (1, 29, 0, "final", 0) def get_version(): diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/authentication.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/api/ldt/authentication.py Mon Nov 12 18:33:09 2012 +0100 @@ -0,0 +1,58 @@ +from django.conf import settings +from django.middleware.csrf import _sanitize_token, constant_time_compare +from django.utils.http import same_origin +from tastypie.authentication import Authentication + +# imported from tastypie's next version 0.9.12 +class SessionAuthentication(Authentication): + """ + An authentication mechanism that piggy-backs on Django sessions. + + This is useful when the API is talking to Javascript on the same site. + Relies on the user being logged in through the standard Django login + setup. + + Requires a valid CSRF token. + """ + def is_authenticated(self, request, **kwargs): + """ + Checks to make sure the user is logged in & has a Django session. + """ + # Cargo-culted from Django 1.3/1.4's ``django/middleware/csrf.py``. + # We can't just use what's there, since the return values will be + # wrong. + # We also can't risk accessing ``request.POST``, which will break with + # the serialized bodies. + if request.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'): + return request.user.is_authenticated() + + if getattr(request, '_dont_enforce_csrf_checks', False): + return request.user.is_authenticated() + + csrf_token = _sanitize_token(request.COOKIES.get(settings.CSRF_COOKIE_NAME, '')) + + if request.is_secure(): + referer = request.META.get('HTTP_REFERER') + + if referer is None: + return False + + good_referer = 'https://%s/' % request.get_host() + + if not same_origin(referer, good_referer): + return False + + request_csrf_token = request.META.get('HTTP_X_CSRFTOKEN', '') + + if not constant_time_compare(request_csrf_token, csrf_token): + return False + + return request.user.is_authenticated() + + def get_identifier(self, request): + """ + Provides a unique string identifier for the requestor. + + This implementation returns the user's username. + """ + return request.user.username \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/resources/__init__.py --- a/src/ldt/ldt/api/ldt/resources/__init__.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/api/ldt/resources/__init__.py Mon Nov 12 18:33:09 2012 +0100 @@ -2,5 +2,6 @@ from content import ContentResource from project import ProjectResource from segment import SegmentResource +from user import UserResource -__all__ = ["AnnotationResource", "ContentResource", "ProjectResource", "SegmentResource"] +__all__ = ["AnnotationResource", "ContentResource", "ProjectResource", "SegmentResource", "UserResource"] diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/resources/annotation.py --- a/src/ldt/ldt/api/ldt/resources/annotation.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/api/ldt/resources/annotation.py Mon Nov 12 18:33:09 2012 +0100 @@ -10,17 +10,17 @@ class AnnotationObject(object): - # For the moment, these attributes are useless. We just need to define AnnotationObject - id = "" - project = "" - type = "" - type_title = "" - media = "" - begin = "" - end = "" - content = {"title":"", "description":""} - tags = [] - meta = {"creator":"","created":""} + def __init__(self, id="", project = "", type = "", type_title = "", media = "", begin = 0, end = 0, content = {"title":"", "description":""}, tags = [], meta = {"creator":"","created":""}): + self.id = id + self.project = project + self.type = type + self.type_title = type_title + self.media = media + self.begin = begin + self.end = end + self.content = content + self.tags = tags + self.meta = meta class AnnotationResource(Resource): # For the moment, these attributes are useless. We just prepare relations to AnnotationObject @@ -36,23 +36,24 @@ meta = fields.DictField(attribute = 'meta') class Meta: - allowed_methods = ['put'] + allowed_methods = ['post'] resource_name = 'annotations' object_class = AnnotationObject authorization = Authorization() # always_return_data = True because we want the api returns the data with the updated ids always_return_data = True + include_resource_uri = False def obj_delete_list(self, request=None, **kwargs): return True def obj_create(self, bundle, request=None, **kwargs): - """ - """ - logging.debug("ICI 0-1 bundle.data = " + repr(bundle.data)) + #logging.debug("ICI 0-1 bundle.data = " + repr(bundle.data)) + # Here the a has the datas for only one annotation. Tastypie's post allows only one resource addition + a = bundle.data project_id = "" - if bundle.data.has_key('project') : - project_id = bundle.data["project"] + if a.has_key('project') : + project_id = a["project"] if project_id and project_id != "" : try: project = Project.objects.get(ldt_id=project_id) @@ -60,18 +61,17 @@ raise NotFound("Project not found. project_id = " + project_id) else : # If the project's is not defined, we get or create the content's front project. - iri_id = bundle.data["media"] + iri_id = a["media"] try: content = Content.objects.get(iri_id=iri_id) except Content.DoesNotExist: raise NotFound("Content not found. iri_id = " + iri_id) project = content.get_or_create_front_project() - bundle.data["project"] = project.ldt_id + a[u"project"] = project.ldt_id adder = LdtAnnotation(project) unprotect_models() # Allows anonymous user to modify models in this request only - # Here the bundle.data has the datas for only one annotation. The others api's functions parse the "objects" from the request's json. - a = bundle.data + dur = str(a['end'] - a['begin']) begin = str(a['begin']) # We test if the annotation has audio node @@ -95,16 +95,19 @@ add_annotation_to_stat(content, a['begin'], a['end']) # We update the ids - bundle.data['type'] = type_id - bundle.data['id'] = new_id - if not bundle.data['content'].has_key('audio') : - bundle.data['content']['audio'] = {'src':audio_src, 'href':audio_href} - # We reinject the datas in the bundle's response - bundle = self.full_hydrate(bundle) + a['type'] = type_id + a['id'] = new_id + if not a['content'].has_key('audio') : + a['content']['audio'] = {'src':audio_src, 'href':audio_href} # We save the added annotation and reprotect the contents and projects adder.save() protect_models() + # We update the AnnotationObject for the returned datas to be correct. + bundle.obj = AnnotationObject(id = a["id"], project = a["project"], type = a["type"], type_title = a["type_title"], media = a["media"], begin = a["begin"], end = a["end"], content = a['content'], tags = a['tags'], meta = a['meta']) return bundle + + def get_resource_uri(self, bundle_or_obj): + return '' \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/resources/content.py --- a/src/ldt/ldt/api/ldt/resources/content.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/api/ldt/resources/content.py Mon Nov 12 18:33:09 2012 +0100 @@ -1,6 +1,8 @@ from django.conf.urls.defaults import url -from ldt.ldt_utils.models import Content -from tastypie.resources import ModelResource +from ldt.ldt_utils.models import Content, Segment +from tastypie.resources import Bundle, ModelResource +from ldt.indexation import get_results_list +from itertools import groupby class ContentResource(ModelResource): @@ -9,10 +11,49 @@ resource_name = 'contents' queryset = Content.objects.all() excludes = ['media_obj'] + + def get_object_list(self, request): + return Content.safe_objects.all() def override_urls(self): # WARNING : in tastypie <= 1.0, override_urls is used instead of prepend_urls. From 1.0.0, prepend_urls will be prefered and override_urls deprecated return [ - url(r"^(?P%s)/(?P[\w\d_.-]+)/$" % self._meta.resource_name, self.wrap_view('dispatch_detail'), name="api_dispatch_detail"), + url(r"^(?P%s)/recommended/$" % self._meta.resource_name, self.wrap_view('get_recommended'), name="api_contents_recommended"), + url(r"^(?P%s)/(?P[\w\d_.-]+)/$" % self._meta.resource_name, self.wrap_view('dispatch_detail'), name="api_dispatch_detail"), ] + def get_resource_uri(self, bundle_or_obj): + kwargs = { + 'resource_name': self._meta.resource_name, + 'api_name': self._meta.api_name + } + if isinstance(bundle_or_obj, Bundle): + kwargs['iri_id'] = bundle_or_obj.obj.iri_id + else: + kwargs['iri_id'] = bundle_or_obj.iri_id + return self._build_reverse_url("api_dispatch_detail", kwargs=kwargs) + + def get_recommended(self, request, **kwargs): + self.method_check(request, allowed=['get']) + + keywords = request.GET.get('keywords','') + keywords_search = " OR ".join(keywords.split(',')) + field = request.GET.get('field','all') + + result_list = get_results_list(field, keywords_search) + score_dict = dict([(k,sum([e.score for e in i])) for k,i in groupby(result_list, lambda e: e.iri_id)]) + + res = [self.full_dehydrate(self.build_bundle(obj=c, request=request)) for c in Content.safe_objects.filter(iri_id__in = score_dict.keys())] + + def add_score(b,s): + b.data['score'] = s + return b + + object_list = { + 'objects': sorted([add_score(b, score_dict.get(b.data['iri_id'],0)) for b in res], key=lambda b: b.data['score'], reverse=True), + } + + self.log_throttled_access(request) + + return self.create_response(request, object_list) + \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/resources/project.py --- a/src/ldt/ldt/api/ldt/resources/project.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/api/ldt/resources/project.py Mon Nov 12 18:33:09 2012 +0100 @@ -1,23 +1,51 @@ from django.conf.urls.defaults import url from ldt.ldt_utils.models import Project +from ldt.api.ldt.authentication import SessionAuthentication +from ldt.api.ldt.serializers.cinelabserializer import CinelabSerializer +from ldt.api.ldt.resources import ContentResource +from ldt.api.ldt.resources.user import UserResource from tastypie.authorization import Authorization -from tastypie.resources import ModelResource - +from tastypie.resources import Bundle, ModelResource +from tastypie import fields +from ldt.security import protect_models, unprotect_models class ProjectResource(ModelResource): + contents = fields.ManyToManyField(ContentResource, 'contents') + owner = fields.ForeignKey(UserResource, 'owner') class Meta: - allowed_methods = ['get', 'put'] - authorization= Authorization() # BE CAREFUL WITH THAT, it's unsecure + allowed_methods = ['get', 'post'] + authorization = Authorization() # BE CAREFUL WITH THAT, it's unsecure + authentication = SessionAuthentication() resource_name = 'projects' queryset = Project.objects.all() + serializer = CinelabSerializer() + # In the future version : + # detail_uri_name = 'ldt_id' -# # WARNING : this project API will only accepts and returns json format, no matter format get parameter. -# def determine_format(self, request): -# return "application/json" + + def get_object_list(self, request): + return Project.safe_objects.all() def override_urls(self): # WARNING : in tastypie <= 1.0, override_urls is used instead of prepend_urls. From 1.0.0, prepend_urls will be prefered and override_urls deprecated return [ url(r"^(?P%s)/(?P[\w\d_.-]+)/$" % self._meta.resource_name, self.wrap_view('dispatch_detail'), name="api_dispatch_detail"), ] + + def get_resource_uri(self, bundle_or_obj): + kwargs = { + 'resource_name': self._meta.resource_name, + 'api_name': self._meta.api_name + } + if isinstance(bundle_or_obj, Bundle): + kwargs['ldt_id'] = bundle_or_obj.obj.ldt_id + else: + kwargs['ldt_id'] = bundle_or_obj.ldt_id + return self._build_reverse_url("api_dispatch_detail", kwargs=kwargs) + + # TEMPORARY (used before authentication/authorization, because saving a project modifies a Content (via ContentStat)) + def save_m2m(self, bundle): + unprotect_models() + super(ProjectResource, self).save_m2m(bundle) + protect_models() \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/resources/user.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/api/ldt/resources/user.py Mon Nov 12 18:33:09 2012 +0100 @@ -0,0 +1,26 @@ +from django.contrib.auth.models import User +from django.conf.urls.defaults import url +from tastypie.resources import Bundle, ModelResource + +class UserResource(ModelResource): + class Meta: + allowed_methods = ['get'] + queryset = User.objects.all() + resource_name = 'users' + + def override_urls(self): + # WARNING : in tastypie <= 1.0, override_urls is used instead of prepend_urls. From 1.0.0, prepend_urls will be prefered and override_urls deprecated + return [ + url(r"^(?P%s)/(?P[\w\d_.-]+)/$" % self._meta.resource_name, self.wrap_view('dispatch_detail'), name="api_dispatch_detail"), + ] + + def get_resource_uri(self, bundle_or_obj): + kwargs = { + 'resource_name': self._meta.resource_name, + 'api_name': self._meta.api_name + } + if isinstance(bundle_or_obj, Bundle): + kwargs['username'] = bundle_or_obj.obj.username + else: + kwargs['username'] = bundle_or_obj.username + return self._build_reverse_url("api_dispatch_detail", kwargs=kwargs) \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/serializers/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/api/ldt/serializers/__init__.py Mon Nov 12 18:33:09 2012 +0100 @@ -0,0 +1,1 @@ + diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/serializers/cinelabserializer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/api/ldt/serializers/cinelabserializer.py Mon Nov 12 18:33:09 2012 +0100 @@ -0,0 +1,296 @@ +from django.conf import settings +from django.core.serializers import json +from django.core.urlresolvers import reverse +from django.utils import simplejson +from ldt.ldt_utils.models import Content, Project +from ldt.ldt_utils.projectserializer import ProjectJsonSerializer +from ldt.ldt_utils.utils import generate_uuid +from tastypie.serializers import Serializer +from tastypie.exceptions import NotFound +import math +import lxml.etree +import logging + + +class CinelabSerializer(Serializer): + # Thanks to the this serializer, the api will be able to serialize and deserialize a project in cinelab json format + # See http://liris.cnrs.fr/advene/cinelab/ for more information + formats = ['json', 'jsonp', 'xml', 'yaml', 'html', 'plist', 'cinelab'] + content_types = { + 'cinelab': 'application/cinelab', + 'json': 'application/json', + 'jsonp': 'text/javascript', + 'xml': 'application/xml', + 'yaml': 'text/yaml', + 'html': 'text/html', + 'plist': 'application/x-plist', + } + json_indent = 2 + + def to_cinelab(self, data, options=None): + """ + Given some Python data, produces Cinelab/JSON output. + N.B. : for the moment, this serializer works with api_dispatch_detail only. + It does not work with api_dispatch_list. Only from_cinelab deals with api_dispatch_list's multiple objects like {"objects":[...]} + """ + options = options or {} + if hasattr(data, 'obj') : + if isinstance(data.obj, Project) : + ps = ProjectJsonSerializer(data.obj) + data = ps.serialize_to_cinelab() + return simplejson.dumps(data, cls=json.DjangoJSONEncoder, sort_keys=True, indent=self.json_indent) + raise NotFound("Project not found") + + + def from_cinelab(self, content): + """ + This function takes cinelab/json and transform it into a regular REST API json project data. + All fields are (* required at rebuilt json) : + "changed_by": "admin", + "created_by": "admin", + "creation_date": "2012-02-11T01:18:48.028534", + "description": "", + "image": "/pf/static/media/thumbnails/contents/content_default_icon.png", + "ldt*": "...............", + "ldt_id*": "594dc612-544e-11e1-96e5-00145ea49a02", + "modification_date": "2012-02-11T01:19:40.203322", + "state": 2, + "title*": "front project : test DailyMotion" + "contents*": ["IRI_ID_1","IRI_ID_2"] + "owner*": "user_id" + """ + logging.debug("FROM cinelab content = " + content) + + cinelab = simplejson.loads(content) + meta = cinelab["meta"] + ldt_id = generate_uuid() + # default state = (1, 'edition') OR (2, 'published') + state = 2 + contents = [reverse("api_dispatch_detail", kwargs={"api_name":"1.0", "resource_name":"contents", "iri_id":c["id"]}) for c in cinelab["medias"]] + owner_uri = reverse("api_dispatch_detail", kwargs={"api_name":"1.0", "resource_name":"users", "username":meta["dc:creator"]}) + + ldt = lxml.etree.tostring(self.cinelab_to_ldt(cinelab), pretty_print=True) + + s = {"description": meta["dc:description"], "ldt_id": ldt_id, "title": meta["dc:title"], + "created_by": meta["dc:creator"], "changed_by": meta["dc:contributor"], "creation_date": meta["dc:created"], "modification_date": meta["dc:modified"], + "contents": contents, "owner": owner_uri, "state":state, "ldt":ldt} + + #s = '{"description": "", "ldt": "", "000ldt_id": "gen_by_tc","title": "aaa GEN BY TC"}' + #s = '{"description": "", "ldt": "", "title": "aaaGEN BY TC"}' + #return simplejson.loads(simplejson.dumps(s)) + return s + + + def cinelab_to_ldt(self, cinelab, ldt_id=None): + # Start xml + meta = cinelab["meta"] + annotation_types = cinelab["annotation-types"] + annotations = cinelab["annotations"] + # If the ldt_id is not set, we get in the cinelab meta + if not ldt_id: + ldt_id = meta["id"] + + # Before creating the xml, we build a dict for tag {"id":"label"} + # Careful : works with python >= 2.7 + tag_dict = {t["id"]:t["meta"]["dc:title"] for t in cinelab["tags"]} + # We'll also build a annotation-type to media/ensemble dict to simplify the views node building + at_media_dict = {} + # We'll also build a dict from annotation id to media/ensemble/decoupage dict to simplify the edits/mashup node building + annot_ids_dict = {} + # We'll also build a dict from media id to media url dict to simplify the edits/mashup node building + media_url_dict = {} + + # create a dom + iri = lxml.etree.Element('iri') + + # Node project + projectNode = lxml.etree.SubElement(iri, 'project') + projectNode.set('abstract', meta["dc:description"]) + projectNode.set('title', meta["dc:title"]) + projectNode.set('user', meta["dc:creator"]) + projectNode.set('id', ldt_id) + + # Node medias and node annotations + mediasNode = lxml.etree.SubElement(iri, 'medias') + annotationsNode = lxml.etree.SubElement(iri, 'annotations') + for c in cinelab["medias"]: + # We add the content to the medias node + content = Content.objects.get(iri_id=c["id"]) + iri_id = content.iri_id + mediaNode = lxml.etree.SubElement(mediasNode, 'media') + mediaNode.set('id', iri_id) + mediaNode.set('src', content.iri_url()) + if content.videopath != None : + mediaNode.set('video', content.videopath) + media_url_dict[iri_id] = {'url':content.videopath + content.src, 'pos':None} + else: + mediaNode.set('video', settings.STREAM_URL) + media_url_dict[iri_id] = {'url':settings.STREAM_URL + content.src, 'pos':None} + mediaNode.set('pict', "") + mediaNode.set('extra', "") + # We add the annotations + contentNode = lxml.etree.SubElement(annotationsNode, 'content') + contentNode.set('id', iri_id) + # we search the cinelab lists if they represent a content's list of annotation-type + for l in cinelab["lists"]: + if l["meta"].has_key("id-ref"): + if l["meta"]["id-ref"]==iri_id: + # We build the ensemble node + ensembleNode = lxml.etree.SubElement(contentNode, 'ensemble') + ensembleNode.set('id', l["id"]) + ensembleNode.set('idProject', ldt_id) + ensembleNode.set('title', l["meta"]["dc:title"]) + ensembleNode.set('author', l["meta"]["dc:creator"]) + ensembleNode.set('abstract', l["meta"]["dc:description"]) + # We build the decoupage node, equivalent to an annotation-type + for at_ref in l["items"]: + at_id = at_ref["id-ref"] + # We get the annotation-type datas + for at in annotation_types: + if at["id"]==at_id: + at_media_dict[at_id] = (iri_id, l["id"]) + decoupageNode = lxml.etree.SubElement(ensembleNode, 'decoupage') + decoupageNode.set('id', at_id) + decoupageNode.set('author', at["dc:creator"]) + titleDec = lxml.etree.SubElement(decoupageNode, 'title') + titleDec.text = at["dc:title"] + abstractDec = lxml.etree.SubElement(decoupageNode, 'abstract') + abstractDec.text = at["dc:description"] + elementsNode = lxml.etree.SubElement(decoupageNode, 'elements') + # We get all the annotations for this media and this annotation-type + for a in annotations: + if a["media"]==iri_id and a["meta"]["id-ref"]==at_id: + annot_ids_dict[a["id"]] = (iri_id, l["id"], at_id, a["begin"], a["end"], a["color"]) + elementNode = lxml.etree.SubElement(elementsNode, 'element') + elementNode.set('id', a["id"]) + elementNode.set('begin', str(a["begin"])) + elementNode.set('dur', str(a["end"]-a["begin"])) + elementNode.set('author', a["meta"]["dc:creator"]) + elementNode.set('date', a["meta"]["dc:created"]) + elementNode.set('color', a["color"]) + img_src = "" + if a["content"].has_key("img"): + if a["content"]["img"].has_key("src"): + img_src = a["content"]["img"]["src"] + elementNode.set('src', img_src) + titleElm = lxml.etree.SubElement(elementNode, 'title') + titleElm.text = a["content"]["title"] + abstractElm = lxml.etree.SubElement(elementNode, 'abstract') + abstractElm.text = a["content"]["description"] + # Audio node, if the dict has the audio key + audioElm = lxml.etree.SubElement(elementNode, 'audio') + audioElm.set('source', "") + if a["content"].has_key("audio"): + audioElm.set('source', a["content"]["audio"]["src"]) + audioElm.text = a["content"]["audio"]["href"] + # The tags dict has been set before, we just get the labels + tagsNode = lxml.etree.SubElement(elementNode, 'tags') + if a["tags"]: + for t in a["tags"]: + if tag_dict.has_key(t["id-ref"]): + tagNode = lxml.etree.SubElement(tagsNode, 'tag') + tagNode.text = tag_dict[t["id-ref"]] + # Last element's node + lxml.etree.SubElement(elementNode, 'meta') + + # Now all medias and annotation-types and annotations are the xml + # We can set the views/displays node + displaysNode = lxml.etree.SubElement(iri, 'displays') + id_sel = None + i = 1 + for v in cinelab["views"]: + if "stat" not in v["id"] and v.has_key("annotation_types"): + displayNode = lxml.etree.SubElement(displaysNode, 'display') + displayNode.set('id', v["id"]) + displayNode.set('title', "View " + str(i)) + i += 1 + displayNode.set('tc', "0") + displayNode.set('zoom', "0") + displayNode.set('scroll', "0") + audioDis = lxml.etree.SubElement(displayNode, 'audio') + audioDis.set('source', "") + last_iri_id = "" + last_content_node = None + for at_id in v["annotation_types"]: + iri_id, ens_id = at_media_dict[at_id] + if iri_id != last_iri_id: + last_iri_id = iri_id + last_content_node = lxml.etree.SubElement(displayNode, 'content') + last_content_node.set('id', iri_id) + if last_content_node is not None: + decoupageNode = lxml.etree.SubElement(last_content_node, 'decoupage') + decoupageNode.set('idens', ens_id) + decoupageNode.set('id', at_id) + decoupageNode.set('tagsSelect', "") + if not id_sel: + id_sel = iri_id + if not id_sel: + id_sel = "" + displayNode.set('idsel', id_sel) + + # Now we build the edit node + editsNode = lxml.etree.SubElement(iri, 'edits') + i = 0 + for l in cinelab["lists"]: + if l["meta"].has_key("listtype"): + if l["meta"]["listtype"]=="mashup": + editingNode = lxml.etree.SubElement(editsNode, 'editing') + editingNode.set('id', str(i)) + i += 1 + editingNode.set('tags', "") + titleEd = lxml.etree.SubElement(editingNode, 'title') + titleEd.text = l["meta"]["dc:title"] + abstractEd = lxml.etree.SubElement(editingNode, 'abstract') + abstractEd.text = l["meta"]["dc:description"] + editNode = lxml.etree.SubElement(editingNode, 'edit') + editNode.set('id', "edit1") + editNode.set('tags', "") + # We build the 4 nodes (2 are used in reality : edit list and media list + eListNode = lxml.etree.SubElement(editNode, 'eList') + lxml.etree.SubElement(editNode, 'caption') + lxml.etree.SubElement(editNode, 'audio') + mListNode = lxml.etree.SubElement(editNode, 'mList') + media_pos = 0 + edit_fulltime = 0 + for a_id in l["items"]: + # the key is the annotation's id + iri_id, ens_id, at_id, begin, end, color = annot_ids_dict[a_id] + # We check the media's position in media list. If it was not set, we create the mList node + if media_url_dict[iri_id]["pos"] is None: + media_url_dict[iri_id]["pos"] = media_pos + mNode = lxml.etree.SubElement(mListNode, 'm') + mNode.set('ref', iri_id) + mNode.set('id', str(media_pos)) + media_pos += 1 + mNode.set('t', "v") + mNode.set('c', color) + contentEd = lxml.etree.SubElement(mNode, 'content') + contentEd.text = media_url_dict[iri_id]["url"] + # We add the annotation/instruction to the eList + instNode = lxml.etree.SubElement(eListNode, 'inst') + instNode.set('ref', iri_id + "|;|" + ens_id + "|;|" + at_id + "|;||;||;|" + a_id) + b = int(math.floor(begin/1000)) + instNode.set('begin', str(b)) + e = int(math.floor(end/1000)) + instNode.set('end', str(e)) + instNode.set('m', str(media_url_dict[iri_id]["pos"])) + instNode.set('v', "100") + instNode.set('eBegin', str(edit_fulltime)) + edit_fulltime = edit_fulltime + e - b + instNode.set('eEnd', str(edit_fulltime)) + instNode.set('trId', "0") + instNode.set('trIc', "0") + instNode.set('trOd', "0") + instNode.set('trOc', "0") + # The second empty edit + edit2Node = lxml.etree.SubElement(editingNode, 'edit') + edit2Node.set('id', "edit2") + edit2Node.set('tags', "") + lxml.etree.SubElement(edit2Node, 'eList') + lxml.etree.SubElement(edit2Node, 'caption') + lxml.etree.SubElement(edit2Node, 'audio') + lxml.etree.SubElement(edit2Node, 'mList') + + # This is the end + return iri + \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/api/ldt/urls.py --- a/src/ldt/ldt/api/ldt/urls.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/api/ldt/urls.py Mon Nov 12 18:33:09 2012 +0100 @@ -1,5 +1,5 @@ from django.conf.urls.defaults import patterns, include -from ldt.api.ldt.resources import ProjectResource, ContentResource, SegmentResource, AnnotationResource +from ldt.api.ldt.resources import ProjectResource, ContentResource, SegmentResource, AnnotationResource, UserResource from tastypie.api import Api v1_api = Api(api_name='1.0') @@ -7,6 +7,7 @@ v1_api.register(ContentResource()) v1_api.register(SegmentResource()) v1_api.register(AnnotationResource()) +v1_api.register(UserResource()) urlpatterns = patterns('', (r'', include(v1_api.urls)), diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/contentindexer.py --- a/src/ldt/ldt/ldt_utils/contentindexer.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/ldt_utils/contentindexer.py Mon Nov 12 18:33:09 2012 +0100 @@ -8,7 +8,8 @@ import logging import lxml.etree #@UnresolvedImport import tagging.utils -import urllib #@UnresolvedImport +from ldt.utils.url import request_with_auth +from StringIO import StringIO logger = logging.getLogger(__name__) @@ -129,8 +130,8 @@ def index_content(self, content): url = content.iri_url() - filepath = urllib.urlopen(url) - doc = lxml.etree.parse(filepath) #@UndefinedVariable + _, file_content = request_with_auth(url) + doc = lxml.etree.parse(StringIO(file_content)) #@UndefinedVariable Segment.objects.filter(iri_id=content.iri_id).delete() #@UndefinedVariable diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/fileimport.py --- a/src/ldt/ldt/ldt_utils/fileimport.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/ldt_utils/fileimport.py Mon Nov 12 18:33:09 2012 +0100 @@ -1,11 +1,13 @@ +from StringIO import StringIO from copy import deepcopy #@UnresolvedImport from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from ldt.utils import zipfileext +from ldt.utils.url import request_with_auth from models import Content, Media import fnmatch -import lxml.etree +import lxml.etree #@UnresolvedImport import mimetypes #@UnresolvedImport import os.path import shutil #@UnresolvedImport @@ -25,8 +27,8 @@ class IriInfo(object): - def __init__(self, id, order, titledesc, basepath="", videopath=settings.STREAM_URL, decoupage_blacklist=settings.DECOUPAGE_BLACKLIST, flatten=True): - self.id = id + def __init__(self, iri_id, order, titledesc, basepath="", videopath=settings.STREAM_URL, decoupage_blacklist=settings.DECOUPAGE_BLACKLIST, flatten=True): + self.id = iri_id self.basepath = basepath self.order = order self.src = "" @@ -48,14 +50,15 @@ def process_iri(self): # for just import a file ldt and get the title for every media if 'http' in self.src: - #url = urllib.urlopen(self.src) path = self.src - #doc = xml.dom.minidom.parse(url) #for import a zip, get title and copy file .iri in the media directory else: path = os.path.join(self.basepath, self.src) - #doc = xml.dom.minidom.parse(path) - + + if 'http' in path: + _,content = request_with_auth(path) + path = StringIO(content) + doc = lxml.etree.parse(path) #@UndefinedVariable @@ -78,9 +81,9 @@ #if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.tagName == "ensemble": if node.tag == "ensemble": #id = node.getAttributeNS(None,u"id") - id = node.attrib["id"] - if id not in ensembleids: - ensembleids.append(id) + ens_id = node.attrib["id"] + if ens_id not in ensembleids: + ensembleids.append(ens_id) if self.annotations is not None: newEnsemble = None @@ -304,43 +307,41 @@ for i, medianode in enumerate(result): # get iri file's id from file ldt - #id = medianode.attributes['id'].value - id = medianode.attrib['id'] + iri_id = medianode.attrib['id'] if self.check_existing_media: try: - Content.objects.get(iri_id=id) + Content.objects.get(iri_id=iri_id) do_pass = True except ObjectDoesNotExist: #Content.DoesNotExist do_pass = False else: do_pass = False if not do_pass: - if not (contents.has_key(id)): + if not (contents.has_key(iri_id)): # Create instance iriInfo(id, order, titledesc, basepath="", videopath=settings.STREAM_URL) if ldtpath: - contents[id] = IriInfo(id, i, "", os.path.dirname(ldtpath), flatten=self.flatten) + contents[iri_id] = IriInfo(iri_id, i, "", os.path.dirname(ldtpath), flatten=self.flatten) else: - contents[id] = IriInfo(id, i, "", flatten=self.flatten) + contents[iri_id] = IriInfo(iri_id, i, "", flatten=self.flatten) # Get iri file's url from ldt. This url can be relative path or absolute path. - #contents[id].src = medianode.attributes['src'].value - contents[id].src = medianode.attrib['src'] + contents[iri_id].src = medianode.attrib['src'] if medianode.attrib['video'] != "": - contents[id].videopath = medianode.attrib['video'] + contents[iri_id].videopath = medianode.attrib['video'] elif self.videopath != "" or self.videopath: - contents[id].videopath = self.videopath + contents[iri_id].videopath = self.videopath else: - contents[id].videopath = settings.STREAM_URL + contents[iri_id].videopath = settings.STREAM_URL #get annotation of file ldt result = doc.xpath("/iri/annotations/content") for contentnode in result: - id = contentnode.attrib['id'] - if contents.has_key(id): + content_id = contentnode.attrib['id'] + if contents.has_key(content_id): if self.author: contentnode.set("author", unicode(self.author)) - contents[id].annotations = contentnode + contents[content_id].annotations = contentnode #go throught values for iriinfo in contents.values(): diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/fixtures/base_data.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/ldt_utils/fixtures/base_data.json Mon Nov 12 18:33:09 2012 +0100 @@ -0,0 +1,36 @@ +[ + { + "pk": 6, + "model": "text.annotation", + "fields": { + "update_date": "2010-11-16 17:30:50", + "description": "texte de description", + "title": "titre de l'annotation", + "color": "#DDDDDD", + "text": "texte selectionne lors de la creation de l'annotation", + "creator": "wakimd", + "uri": "", + "creation_date": "2010-11-16 17:01:41", + "contributor": "oaubert", + "tags_field": "tag3,tag1,", + "external_id": "z2c1d1fa-629d-4520-a3d2-955b4f2582c0" + } + }, + { + "pk": 7, + "model": "text.annotation", + "fields": { + "update_date": "2010-11-16 17:30:50", + "description": "texte de description", + "title": "titre de l'annotation", + "color": "#DDDDDD", + "text": "texte selectionne lors de la creation de l'annotation", + "creator": "wakimd", + "uri": "http://www.leezam.com/pub/epub/123456!/OPS/chapter2.xhtml#pos=56,168", + "creation_date": "2010-11-16 17:01:41", + "contributor": "oaubert", + "tags_field": "tag3,tag1,", + "external_id": "mypersonnalid2" + } + } +] diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/fixtures/user_data.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/ldt/ldt/ldt_utils/fixtures/user_data.json Mon Nov 12 18:33:09 2012 +0100 @@ -0,0 +1,67 @@ +[ + { + "pk": 2, + "model": "auth.user", + "fields": { + "username": "admin", + "first_name": "", + "last_name": "", + "is_active": true, + "is_superuser": true, + "is_staff": false, + "last_login": "2010-12-12 00:04:07", + "groups": [], "user_permissions": [], + "password": "", + "email": "admin@example.com", + "date_joined": "2010-12-12 00:04:07" + } + }, + { + "pk": 1, + "model": "oauth_provider.resource", + "fields": { + "url": "/api/1.0/text/delete/", + "name": "delete", + "is_readonly": true + } + }, + { + "pk": 2, + "model": "oauth_provider.resource", + "fields": { + "url": "/api/1.0/text/create/", + "name": "create", + "is_readonly": true + } + }, + { + "pk": 3, + "model": "oauth_provider.resource", + "fields": { + "url": "/api/1.0/text/update/", + "name": "update", + "is_readonly": true + } + }, + { + "pk": 4, + "model": "oauth_provider.resource", + "fields": { + "url": "", + "name": "all", + "is_readonly": true + } + }, + { + "pk": 1, + "model": "oauth_provider.consumer", + "fields": { + "status": 1, + "name": "example.com", + "secret": "kd94hf93k423kf44", + "user": 2, + "key": "dpf43f3p2l4k3l03", + "description": "" + } + } +] \ No newline at end of file diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/models.py --- a/src/ldt/ldt/ldt_utils/models.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/ldt_utils/models.py Mon Nov 12 18:33:09 2012 +0100 @@ -8,13 +8,14 @@ get_current_user) from ldt.security.manager import SafeManager from ldt.security.models import SafeModel +from ldt.utils import url as url_utils from sorl.thumbnail import ImageField from tagging.models import Tag from utils import (create_ldt, copy_ldt, create_empty_iri, update_iri, generate_uuid) from ldt.utils import generate_hash import datetime -import lxml.etree +import lxml.etree #@UnresolvedImport import mimetypes import os.path import re @@ -54,7 +55,6 @@ description = models.TextField(null=True, blank=True, verbose_name=_('description')) title = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('title')) src = models.CharField(max_length=1024, verbose_name=_('media.src')) - #TODO: use a fixed length char field src_hash = models.CharField(max_length=128, unique=True, verbose_name=_('media.src_hash'), blank=True) mimetype_field = models.CharField(max_length=512, null=True, blank=True, verbose_name=_('media.mimetype')) @@ -178,10 +178,10 @@ iri_file_path = self.iri_file_path() thumbnail = os.path.join(settings.MEDIA_ROOT, unicode(self.image)) if os.path.exists(iri_file_path): - dir = os.path.dirname(iri_file_path) - temp = os.path.join(os.path.join(os.path.dirname(dir), "temp"), self.iri_id) + iri_dir = os.path.dirname(iri_file_path) + temp = os.path.join(os.path.join(os.path.dirname(iri_dir), "temp"), self.iri_id) try: - move(dir, temp) + move(iri_dir, temp) except Exception, e: raise e if os.path.exists(thumbnail): @@ -197,8 +197,8 @@ #del .iri, and .png from temp directory def commit(self): iri_file_path=self.iri_file_path() - dir = os.path.dirname(iri_file_path) - temp = os.path.join(os.path.join(os.path.dirname(dir), "temp"), self.iri_id) + iri_dir = os.path.dirname(iri_file_path) + temp = os.path.join(os.path.join(os.path.dirname(iri_dir), "temp"), self.iri_id) thumbnail = os.path.join(settings.MEDIA_ROOT, unicode(self.image)) temp_thumbnail = os.path.join(os.path.dirname(thumbnail), "temp") if os.path.exists(temp): @@ -211,12 +211,12 @@ #move .iri, and .png to there original directory def rollback(self): iri_file_path=self.iri_file_path() - dir = os.path.dirname(iri_file_path) - temp = os.path.join(os.path.join(os.path.dirname(dir), "temp"), self.iri_id) + iri_dir = os.path.dirname(iri_file_path) + temp = os.path.join(os.path.join(os.path.dirname(iri_dir), "temp"), self.iri_id) thumbnail = os.path.join(settings.MEDIA_ROOT, unicode(self.image)) temp_thumbnail = os.path.join(os.path.dirname(thumbnail), "temp") if os.path.exists(temp): - move(temp, dir) + move(temp, iri_dir) os.rmdir(os.path.dirname(temp)) if os.path.exists(temp_thumbnail): move(os.path.join(temp_thumbnail, os.path.basename(thumbnail)), os.path.dirname(thumbnail)) @@ -260,12 +260,12 @@ try: iri_file_path = self.iri_file_path() if not os.path.exists(iri_file_path): - dir = os.path.dirname(iri_file_path) - if not os.path.exists(dir): - os.makedirs(dir) + iri_dir = os.path.dirname(iri_file_path) + if not os.path.exists(iri_dir): + os.makedirs(iri_dir) created = True - file = open(iri_file_path, "w") - create_empty_iri(file, self, "IRI") + iri_file = open(iri_file_path, "w") + create_empty_iri(iri_file, self, "IRI") else: created = False update_iri(iri_file_path, self, "IRI") @@ -301,10 +301,13 @@ return str(self.id) + ": " + self.iri_id def iri_url(self, web_url=settings.WEB_URL): - if 'http' in self.iriurl or 'https' in self.iriurl: + if url_utils.is_absolute(self.iriurl): return self.iriurl else: - return unicode(web_url) + unicode(settings.MEDIA_URL) + u"ldt/" + unicode(self.iriurl) + res_url = unicode(settings.MEDIA_URL) + u"ldt/" + unicode(self.iriurl) + if not url_utils.is_absolute(res_url): + res_url = unicode(web_url) + res_url + return res_url def iri_file_path(self): return os.path.join(os.path.join(os.path.join(settings.MEDIA_ROOT, "ldt"), self.iri_id), os.path.basename(self.iriurl)) diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/projectserializer.py --- a/src/ldt/ldt/ldt_utils/projectserializer.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/ldt_utils/projectserializer.py Mon Nov 12 18:33:09 2012 +0100 @@ -16,7 +16,7 @@ """ Serialize a project object to a cinelab compatible array """ -class ProjectSerializer: +class ProjectJsonSerializer: def __init__(self, project, from_contents=True, from_display=True, first_cutting=None, only_one_cutting=False): self.project = project @@ -239,10 +239,10 @@ if not element_tags: element_tags = None - + new_annotation = { - "begin": int(element_begin), - "end": int(element_begin) + int(element_duration), + "begin": int(float(element_begin)), + "end": int(float(element_begin)) + int(float(element_duration)), "id": element_id, "media": element_media, "color": element_color, @@ -320,7 +320,7 @@ ensemble_node = cutting_node.xpath('..')[0] content_node = ensemble_node.xpath('..')[0] iri_id = content_node.get("id") - content = Content.objects.get(iri_id=iri_id) + content = Content.objects.get(iri_id=iri_id) self.__parse_ensemble(ensemble_node, content, cutting_only=[cutting_node]) @@ -455,7 +455,7 @@ self.views_dict['test'] = new_display - if self.serialize_contents: + if self.serialize_contents: res = doc.xpath("/iri/body/ensembles/ensemble") for ensemble_node in res: self.__parse_ensemble(ensemble_node, content) diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/searchutils.py --- a/src/ldt/ldt/ldt_utils/searchutils.py Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/ldt_utils/searchutils.py Mon Nov 12 18:33:09 2012 +0100 @@ -1,4 +1,3 @@ -from django.conf import settings from ldt.indexation import SimpleSearch from ldt.ldt_utils.models import Content, Project from ldt.ldt_utils.utils import LdtUtils diff -r 7d0b9d12bf5a -r a227bbd1d2c7 src/ldt/ldt/ldt_utils/templates/front/front_base.html --- a/src/ldt/ldt/ldt_utils/templates/front/front_base.html Mon Nov 12 16:37:57 2012 +0100 +++ b/src/ldt/ldt/ldt_utils/templates/front/front_base.html Mon Nov 12 18:33:09 2012 +0100 @@ -78,7 +78,7 @@