# HG changeset patch # User cavaliet # Date 1396257581 -7200 # Node ID 7913cdea80e7dac4fe5aa1710b8a9233a0ce42b8 # Parent a5500ded7c87cb62580a2f5e9e2c3804e7810051 import data now recursive diff -r a5500ded7c87 -r 7913cdea80e7 src/spel/__init__.py --- a/src/spel/__init__.py Fri Mar 28 17:36:35 2014 +0100 +++ b/src/spel/__init__.py Mon Mar 31 11:19:41 2014 +0200 @@ -1,4 +1,4 @@ -VERSION = (0, 4, 3, "final", 0) +VERSION = (0, 4, 4, "final", 0) VERSION_STR = unicode(".".join(map(lambda i:"%02d" % (i,), VERSION[:2]))) diff -r a5500ded7c87 -r 7913cdea80e7 src/spel/management/commands/loadspeldata.py --- a/src/spel/management/commands/loadspeldata.py Fri Mar 28 17:36:35 2014 +0100 +++ b/src/spel/management/commands/loadspeldata.py Mon Mar 31 11:19:41 2014 +0200 @@ -70,149 +70,149 @@ Type 'y' to continue, or 'n' to quit: """) do_import = (confirm == "y") - # Continue if do_import: print("Parsing folder %s ..." % path) - for f in listdir(path): - json_path = join(path,f,"cinelab_iri.json") - if isfile(json_path): - print("Parsing json file %s ..." % json_path) - json_data = False - try: - file_data = open(json_path) - json_data = json.load(file_data) - file_data.close() - except: - print("Error while parsing json file.") - if json_data: - # Save media and content - media, _ = Media.objects.get_or_create(src=f+"/original.mp4", duration=json_data["medias"][0]["meta"]["duration"]) - media.is_public = True - ctt_id = f - content = Content.objects.create(iri_id=ctt_id, - iriurl=ctt_id+u"/"+ctt_id+u".iri", - media_obj=media, - title=json_data["meta"]["dc:title"], - duration=int(json_data["medias"][0]["meta"]["duration"]), - content_creation_date = json_data["meta"]["dc:created"]) - content.is_public = True - # Get content front projet - proj = content.front_project - username = proj.owner.username - now = datetime.utcnow().isoformat() - # Start data to send to api - proj_data = {} - proj_data["meta"] = {} - proj_data["meta"]["id"] = proj.ldt_id - proj_data["meta"]["dc:title"] = proj.title - proj_data["meta"]["dc:creator"] = username - proj_data["meta"]["dc:description"] = "description added" - proj_data["meta"]["dc:created"] = json_data["meta"]["dc:created"] - proj_data["meta"]["dc:modified"] = json_data["meta"]["dc:modified"] - proj_data["meta"]["dc:contributor"] = username - proj_data["medias"] = [] - proj_data["medias"].append({"id": content.iri_id}) - # The tags and annotations (main part) - proj_data["tags"] = [] - proj_data["annotations"] = [] - tags_id_label = {} - tags_label_id = {} - for a in json_data["annotations"]: - # "content": { "data": { "modalites_sceniques": "costumes,décors",... } } - if "content" in a and "data" in a["content"] and type(a["content"]["data"]) == type(dict()): - annot_tags = [] - desc = "" - title = "" - # Build tags - for k,v in a["content"]["data"].iteritems(): - if k!="commentaire" and k!="description" and k!="titre": - v = v.split(",") - for val in v: - val = val.strip() - tag_label = k + u": " + val - if val!="": - if not tag_label in tags_label_id: - tags_label_id[tag_label] = generate_uuid() - tags_id_label[tags_label_id[tag_label]] = tag_label - #logger.debug("CREATED") - #logger.debug(tags_label_id[tag_label] + " = " + tags_id_label[tags_label_id[tag_label]]) - proj_data["tags"].append({"meta": { "dc:title": tag_label }, "id": tags_label_id[tag_label] }) - annot_tags.append({"id-ref": tags_label_id[tag_label] }) - elif k=="commentaire" or k=="description": - desc = v - elif k=="titre": - title = v - # Build annotation with needed fields - proj_data["annotations"].append({ - "content": { - "mimetype": "application/x-ldt-structured", - "description": desc, -# "img": { -# "src": "" -# }, - "title": title, -# "polemics": [ ], - }, - "begin": a["begin"], - "meta": { -# "dc:contributor": "admin", - "id-ref": a["type"], - "dc:created": now, -# "dc:modified": "2014-03-04T16:40:23.609971", - "dc:creator": username - }, - "end": a["end"], - "tags": annot_tags, - "color": "16763904", - "media": ctt_id, - "id": a["id"] + for dirpath, dirnames, filenames in os.walk(path): + #print("Parsing walk %s, %s, %s" % (dirpath, dirnames, filenames)) + for filename in filenames: + if filename == "cinelab_iri.json": + json_path = os.path.join(dirpath, filename) + print("Parsing json file %s ..." % json_path) + json_data = False + try: + file_data = open(json_path) + json_data = json.load(file_data) + file_data.close() + except: + print("Error while parsing json file.") + if json_data: + ctt_id = os.path.basename(dirpath) + # Save media and content + media, _ = Media.objects.get_or_create(src=ctt_id+"/original.mp4", duration=json_data["medias"][0]["meta"]["duration"]) + media.is_public = True + content = Content.objects.create(iri_id=ctt_id, + iriurl=ctt_id+u"/"+ctt_id+u".iri", + media_obj=media, + title=json_data["meta"]["dc:title"], + duration=int(json_data["medias"][0]["meta"]["duration"]), + content_creation_date = json_data["meta"]["dc:created"]) + content.is_public = True + # Get content front projet + proj = content.front_project + username = proj.owner.username + now = datetime.utcnow().isoformat() + # Start data to send to api + proj_data = {} + proj_data["meta"] = {} + proj_data["meta"]["id"] = proj.ldt_id + proj_data["meta"]["dc:title"] = proj.title + proj_data["meta"]["dc:creator"] = username + proj_data["meta"]["dc:description"] = "description added" + proj_data["meta"]["dc:created"] = json_data["meta"]["dc:created"] + proj_data["meta"]["dc:modified"] = json_data["meta"]["dc:modified"] + proj_data["meta"]["dc:contributor"] = username + proj_data["medias"] = [] + proj_data["medias"].append({"id": content.iri_id}) + # The tags and annotations (main part) + proj_data["tags"] = [] + proj_data["annotations"] = [] + tags_id_label = {} + tags_label_id = {} + for a in json_data["annotations"]: + # "content": { "data": { "modalites_sceniques": "costumes,décors",... } } + if "content" in a and "data" in a["content"] and type(a["content"]["data"]) == type(dict()): + annot_tags = [] + desc = "" + title = "" + # Build tags + for k,v in a["content"]["data"].iteritems(): + if k!="commentaire" and k!="description" and k!="titre": + v = v.split(",") + for val in v: + val = val.strip() + tag_label = k + u": " + val + if val!="": + if not tag_label in tags_label_id: + tags_label_id[tag_label] = generate_uuid() + tags_id_label[tags_label_id[tag_label]] = tag_label + #logger.debug("CREATED") + #logger.debug(tags_label_id[tag_label] + " = " + tags_id_label[tags_label_id[tag_label]]) + proj_data["tags"].append({"meta": { "dc:title": tag_label }, "id": tags_label_id[tag_label] }) + annot_tags.append({"id-ref": tags_label_id[tag_label] }) + elif k=="commentaire" or k=="description": + desc = v + elif k=="titre": + title = v + # Build annotation with needed fields + proj_data["annotations"].append({ + "content": { + "mimetype": "application/x-ldt-structured", + "description": desc, + # "img": { + # "src": "" + # }, + "title": title, + # "polemics": [ ], + }, + "begin": a["begin"], + "meta": { + # "dc:contributor": "admin", + "id-ref": a["type"], + "dc:created": now, + # "dc:modified": "2014-03-04T16:40:23.609971", + "dc:creator": username + }, + "end": a["end"], + "tags": annot_tags, + "color": "16763904", + "media": ctt_id, + "id": a["id"] + }) + + # The annotation-types + proj_data["annotation-types"] = [] + at_ids = [] + for at in json_data["annotation_types"]: + proj_data["annotation-types"].append({ + # dc:contributor: "admin", + "dc:creator": username, + "dc:title": at["id"], + "id": at["id"], + # dc:created: "2014-03-04T14:51:13.907674", + "dc:description": "" + # dc:modified: "2014-03-04T14:51:13.907674" }) - - # The annotation-types - proj_data["annotation-types"] = [] - at_ids = [] - for at in json_data["annotation_types"]: - proj_data["annotation-types"].append({ -# dc:contributor: "admin", - "dc:creator": username, - "dc:title": at["id"], - "id": at["id"], -# dc:created: "2014-03-04T14:51:13.907674", - "dc:description": "" -# dc:modified: "2014-03-04T14:51:13.907674" - }) - at_ids.append({ "id-ref": at["id"] }) - # The list of annotation-types - list_id = generate_uuid() - proj_data["lists"] = [{ - "items": at_ids, - "meta": { - "dc:creator": username, - "id-ref": ctt_id, - "dc:title": "SPEL", - "dc:description": "" - }, - "id": list_id - }] - # The views for default display - proj_data["views"] = [{ - "id": generate_uuid(), - "contents": [ ctt_id ], - "annotation_types": [atid["id-ref"] for atid in at_ids] - }] - - serializr = CinelabSerializer() - serializr.validate_cinelab_json(proj_data) - ldt_xml = serializr.cinelab_to_ldt(proj_data) - proj.ldt = lxml.etree.tostring(ldt_xml, pretty_print=True) - #logger.debug(proj.ldt) - proj.save() - update_stat_content(content) - else: - print("Ignoring or not exist %s ..." % json_path) + at_ids.append({ "id-ref": at["id"] }) + # The list of annotation-types + list_id = generate_uuid() + proj_data["lists"] = [{ + "items": at_ids, + "meta": { + "dc:creator": username, + "id-ref": ctt_id, + "dc:title": "SPEL", + "dc:description": "" + }, + "id": list_id + }] + # The views for default display + proj_data["views"] = [{ + "id": generate_uuid(), + "contents": [ ctt_id ], + "annotation_types": [atid["id-ref"] for atid in at_ids] + }] + + serializr = CinelabSerializer() + serializr.validate_cinelab_json(proj_data) + ldt_xml = serializr.cinelab_to_ldt(proj_data) + proj.ldt = lxml.etree.tostring(ldt_xml, pretty_print=True) + #logger.debug(proj.ldt) + proj.save() + update_stat_content(content) + else: + print("Ignoring or not exist.") # This is the end print("This is the end. DO NOT FORGET TO RUN THE COMMAND 'REINDEX -P' TO GENERATE SEGMENTS AND TAGS.") - \ No newline at end of file