# HG changeset patch # User durandn # Date 1467303017 -7200 # Node ID 8125ce36415c4f991ad7159ec0f574ff8deb479c # Parent aec0f3381736cabc861aed266423bc32d50f17a4 start on dbpedia requests diff -r aec0f3381736 -r 8125ce36415c src/iconolab/models.py --- a/src/iconolab/models.py Thu Jun 30 14:24:57 2016 +0200 +++ b/src/iconolab/models.py Thu Jun 30 18:10:17 2016 +0200 @@ -4,285 +4,324 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django_comments_xtd.models import XtdComment -import uuid, json +from django.utils.text import slugify +import uuid, json, re, requests, urllib class Tag(models.Model): - label = models.SlugField(blank=True, null=True) - link = models.URLField(unique=True) - description = models.CharField(max_length=255, blank=True, null=True) - collection = models.ForeignKey('Collection', blank=True, null=True) - - def is_internal(self): - return self.link.startswith(settings.BASE_URL) + label = models.SlugField(blank=True, null=True) + link = models.URLField(unique=True) + description = models.CharField(max_length=255, blank=True, null=True) + collection = models.ForeignKey('Collection', blank=True, null=True) + + def is_internal(self): + return self.link.startswith(settings.INTERNAL_TAGS_URL) class TaggingInfo(models.Model): - revision = models.ForeignKey('AnnotationRevision', on_delete=models.CASCADE) - tag = models.ForeignKey('Tag', on_delete=models.CASCADE) - accuracy = models.IntegerField() - relevancy = models.IntegerField() + revision = models.ForeignKey('AnnotationRevision', on_delete=models.CASCADE) + tag = models.ForeignKey('Tag', on_delete=models.CASCADE) + accuracy = models.IntegerField() + relevancy = models.IntegerField() class Collection(models.Model): - name = models.CharField(max_length=50, unique=True) - description = models.CharField(max_length=255) + name = models.CharField(max_length=50, unique=True) + description = models.CharField(max_length=255) - def __str__(self): - return self.name + def __str__(self): + return self.name class Item(models.Model): - collection = models.ForeignKey(Collection, related_name="items") + collection = models.ForeignKey(Collection, related_name="items") - + class ItemMetadata(models.Model): - item = models.OneToOneField('Item', related_name='metadatas') - joconde_ref = models.CharField(max_length=20, null=False, blank=False, unique=True) - domain = models.CharField(max_length=255) - title = models.CharField(max_length=255) - description = models.CharField(max_length=255) + item = models.OneToOneField('Item', related_name='metadatas') + joconde_ref = models.CharField(max_length=20, null=False, blank=False, unique=True) + domain = models.CharField(max_length=255) + title = models.CharField(max_length=255) + description = models.CharField(max_length=255) class ImageStats(models.Model): - image = models.OneToOneField('Image', related_name='stats', blank=False, null=False) - views_count = models.IntegerField(blank=True, null=True, default=0) - annotations_count = models.IntegerField(blank=True, null=True, default=0) - submitted_revisions_count = models.IntegerField(blank=True, null=True, default=0) - comments_count = models.IntegerField(blank=True, null=True, default=0) - folders_inclusion_count = models.IntegerField(blank=True, null=True, default=0) - tag_count = models.IntegerField(blank=True, null=True, default=0) + image = models.OneToOneField('Image', related_name='stats', blank=False, null=False) + views_count = models.IntegerField(blank=True, null=True, default=0) + annotations_count = models.IntegerField(blank=True, null=True, default=0) + submitted_revisions_count = models.IntegerField(blank=True, null=True, default=0) + comments_count = models.IntegerField(blank=True, null=True, default=0) + folders_inclusion_count = models.IntegerField(blank=True, null=True, default=0) + tag_count = models.IntegerField(blank=True, null=True, default=0) class Image(models.Model): - image_guid = models.UUIDField(default=uuid.uuid4, editable=False) - name = models.CharField(max_length=200) - media = models.ImageField(upload_to='uploads/', height_field='height', width_field='width') - item = models.ForeignKey('Item', related_name='images', null=True, blank=True) - height = models.IntegerField(null=False, blank=False) - width = models.IntegerField(null=False, blank=False) - created = models.DateTimeField(auto_now_add=True, null=True) - - def __str__(self): - return self.name + image_guid = models.UUIDField(default=uuid.uuid4, editable=False) + name = models.CharField(max_length=200) + media = models.ImageField(upload_to='uploads/', height_field='height', width_field='width') + item = models.ForeignKey('Item', related_name='images', null=True, blank=True) + height = models.IntegerField(null=False, blank=False) + width = models.IntegerField(null=False, blank=False) + created = models.DateTimeField(auto_now_add=True, null=True) + + def __str__(self): + return self.name # # Folders # class Folder(models.Model): -# label = models.CharField(max_length=255) -# owner = models.ForeignKey(User) -# images = models.ManyToManyField(Image) +# label = models.CharField(max_length=255) +# owner = models.ForeignKey(User) +# images = models.ManyToManyField(Image) # -# def __str__(self): -# return label +# def __str__(self): +# return label class AnnotationManager(models.Manager): - - # Call Annotation.objects.create_annotation to initialize a new Annotation with its associated AnnotationStats and initial AnnotationRevision - @transaction.atomic - def create_annotation(self, author, image, title='', description='', fragment='', tags_json='[]'): - # Create annotation object - new_annotation = Annotation( - image=image, - author=author - ) - new_annotation.save() - - # Create initial revision - initial_revision = AnnotationRevision( - annotation=new_annotation, - author=author, - title=title, - description=description, - fragment=fragment, - state=AnnotationRevision.ACCEPTED - ) - initial_revision.save() - initial_revision.set_tags(tags_json) - - new_annotation.current_revision = initial_revision - new_annotation.save() - - # Create stats object - new_annotation_stats = AnnotationStats(annotation=new_annotation) - new_annotation_stats.save() - new_annotation.stats = new_annotation_stats - new_annotation.save() - return new_annotation + + # Call Annotation.objects.create_annotation to initialize a new Annotation with its associated AnnotationStats and initial AnnotationRevision + @transaction.atomic + def create_annotation(self, author, image, title='', description='', fragment='', tags_json='[]'): + # Create annotation object + new_annotation = Annotation( + image=image, + author=author + ) + new_annotation.save() + + # Create initial revision + initial_revision = AnnotationRevision( + annotation=new_annotation, + author=author, + title=title, + description=description, + fragment=fragment, + state=AnnotationRevision.ACCEPTED + ) + initial_revision.save() + initial_revision.set_tags(tags_json) + + new_annotation.current_revision = initial_revision + new_annotation.save() + + # Create stats object + new_annotation_stats = AnnotationStats(annotation=new_annotation) + new_annotation_stats.save() + new_annotation.stats = new_annotation_stats + new_annotation.save() + return new_annotation class AnnotationStats(models.Model): - annotation = models.OneToOneField('Annotation', related_name='stats', blank=False, null=False) - submitted_revisions_count = models.IntegerField(blank=True, null=True, default=1) - accepted_revisions_count = models.IntegerField(blank=True, null=True, default=1) - contributors_count = models.IntegerField(blank=True, null=True, default=1) - views_count = models.IntegerField(blank=True, null=True, default=0) - comments_count = models.IntegerField(blank=True, null=True, default=0) - tag_count = models.IntegerField(blank=True, null=True, default=0) - - def contributors(self): - # returns users that submitted an accepted revision - return + annotation = models.OneToOneField('Annotation', related_name='stats', blank=False, null=False) + submitted_revisions_count = models.IntegerField(blank=True, null=True, default=1) + accepted_revisions_count = models.IntegerField(blank=True, null=True, default=1) + contributors_count = models.IntegerField(blank=True, null=True, default=1) + views_count = models.IntegerField(blank=True, null=True, default=0) + comments_count = models.IntegerField(blank=True, null=True, default=0) + tag_count = models.IntegerField(blank=True, null=True, default=0) + + def contributors(self): + # returns users that submitted an accepted revision + return class Annotation(models.Model): - annotation_guid = models.UUIDField(default=uuid.uuid4, editable=False) - image = models.ForeignKey('Image', related_name='annotations', on_delete=models.CASCADE) - source_revision = models.ForeignKey('AnnotationRevision', related_name='source_related_annotation', blank=True, null=True) - current_revision = models.OneToOneField('AnnotationRevision', related_name='current_for_annotation', blank=True, null=True) - author = models.ForeignKey(User, null=True) - created = models.DateTimeField(auto_now_add=True, null=True) - - objects = AnnotationManager() - - def update_stats(self): - pass - - # Call to create a new revision, possibly from a merge - @transaction.atomic - def make_new_revision(self, author, title, description, fragment, tags_json): - if author == self.author: - # We're creating an automatically accepted revision - new_revision_state = AnnotationRevision.ACCEPTED - else: - # Revision will require validation - new_revision_state = AnnotationRevision.AWAITING - new_revision = AnnotationRevision( - annotation = self, - parent_revision=self.current_revision, - title=title, - description=description, - author=author, - fragment=fragment, - state=new_revision_state - ) - new_revision.save() - new_revision.set_tags(tags_json) - if new_revision.state == AnnotationRevision.ACCEPTED: - self.current_revision = new_revision - self.save() - return new_revision - - # Call when we're validating an awaiting revision whose parent is the current revision AS IT WAS CREATED - @transaction.atomic - def validate_existing_revision(self, revision_to_validate): - if revision_to_validate.parent_revision == current_revision: - self.current_revision = revision_to_validate - revision_to_validate.state = AnnotationRevision.ACCEPTED - revision_to_validate.save() - self.save() - - # Call when we're validating an awaiting revision whose parent isn't the current revision OR IF IT WAS CHANGED BY THE ANNOTATION AUTHOR - @transaction.atomic - def merge_existing_revision(self, title, description, fragment, tags, revision_to_merge): - merged_revision = self.make_new_revision(author=self.author, title=title, description=description, fragment=fragment, tags=tags) - merged_revision.merge_parent_revision = revision_to_merge - merged_revision.save() - self.current_revision=merged_revision - self.save() + annotation_guid = models.UUIDField(default=uuid.uuid4, editable=False) + image = models.ForeignKey('Image', related_name='annotations', on_delete=models.CASCADE) + source_revision = models.ForeignKey('AnnotationRevision', related_name='source_related_annotation', blank=True, null=True) + current_revision = models.OneToOneField('AnnotationRevision', related_name='current_for_annotation', blank=True, null=True) + author = models.ForeignKey(User, null=True) + created = models.DateTimeField(auto_now_add=True, null=True) + + objects = AnnotationManager() + + def update_stats(self): + pass + + # Call to create a new revision, possibly from a merge + @transaction.atomic + def make_new_revision(self, author, title, description, fragment, tags_json): + if author == self.author: + # We're creating an automatically accepted revision + new_revision_state = AnnotationRevision.ACCEPTED + else: + # Revision will require validation + new_revision_state = AnnotationRevision.AWAITING + new_revision = AnnotationRevision( + annotation = self, + parent_revision=self.current_revision, + title=title, + description=description, + author=author, + fragment=fragment, + state=new_revision_state + ) + new_revision.save() + new_revision.set_tags(tags_json) + if new_revision.state == AnnotationRevision.ACCEPTED: + self.current_revision = new_revision + self.save() + return new_revision + + # Call when we're validating an awaiting revision whose parent is the current revision AS IT WAS CREATED + @transaction.atomic + def validate_existing_revision(self, revision_to_validate): + if revision_to_validate.parent_revision == current_revision: + self.current_revision = revision_to_validate + revision_to_validate.state = AnnotationRevision.ACCEPTED + revision_to_validate.save() + self.save() + + # Call when we're validating an awaiting revision whose parent isn't the current revision OR IF IT WAS CHANGED BY THE ANNOTATION AUTHOR + @transaction.atomic + def merge_existing_revision(self, title, description, fragment, tags, revision_to_merge): + merged_revision = self.make_new_revision(author=self.author, title=title, description=description, fragment=fragment, tags=tags) + merged_revision.merge_parent_revision = revision_to_merge + merged_revision.save() + self.current_revision=merged_revision + self.save() class AnnotationRevision(models.Model): - - AWAITING = 0 - ACCEPTED = 1 - REJECTED = 2 - - REVISION_STATES = ( - (AWAITING, 'awaiting'), - (ACCEPTED, 'accepted'), - (REJECTED, 'rejected') - ) - - revision_guid = models.UUIDField(default=uuid.uuid4) - annotation = models.ForeignKey('Annotation', related_name='revisions', null=False, blank=False) - parent_revision = models.ForeignKey('AnnotationRevision', related_name='child_revisions', blank=True, null=True) - merge_parent_revision = models.ForeignKey('AnnotationRevision', related_name='child_revisions_merge', blank=True, null=True) - author = models.ForeignKey(User, null=True) - title = models.CharField(max_length=255) - description = models.TextField(null=True) - fragment = models.TextField() - tags = models.ManyToManyField('Tag', through='TaggingInfo', through_fields=('revision', 'tag')) - state = models.IntegerField(choices=REVISION_STATES, default=AWAITING) - created = models.DateTimeField(auto_now_add=True, null=True) + + AWAITING = 0 + ACCEPTED = 1 + REJECTED = 2 + + REVISION_STATES = ( + (AWAITING, 'awaiting'), + (ACCEPTED, 'accepted'), + (REJECTED, 'rejected') + ) + + revision_guid = models.UUIDField(default=uuid.uuid4) + annotation = models.ForeignKey('Annotation', related_name='revisions', null=False, blank=False) + parent_revision = models.ForeignKey('AnnotationRevision', related_name='child_revisions', blank=True, null=True) + merge_parent_revision = models.ForeignKey('AnnotationRevision', related_name='child_revisions_merge', blank=True, null=True) + author = models.ForeignKey(User, null=True) + title = models.CharField(max_length=255) + description = models.TextField(null=True) + fragment = models.TextField() + tags = models.ManyToManyField('Tag', through='TaggingInfo', through_fields=('revision', 'tag')) + state = models.IntegerField(choices=REVISION_STATES, default=AWAITING) + created = models.DateTimeField(auto_now_add=True, null=True) - def set_tags(self, tags_json_string): - try: - tags_dict = json.loads(tags_json_string) - except ValueError: - pass - for tag_data in tags_dict: - tag_string = tag_data.get("tag_input") - tag_accuracy = tag_data.get("accuracy", 0) - tag_relevancy = tag_data.get("relevancy", 0) - - if tag_string.startswith("http://") or tag_string.startswith("https://"): #check if url - if Tag.objects.filter(link=tag_string).exists(): #check if tag already exists - tag_obj = Tag.objects.get(link=tag_string) - else: - tag_obj = Tag.objects.create( - link = tag_string, - ) - else: - tag_obj = Tag.objects.create( - label = tag_string, - description = "", - link = settings.BASE_URL+'/'+tag_string, - collection = self.annotation.image.item.collection - ) - tag_info = TaggingInfo.objects.create( - tag=tag_obj, - revision=self, - accuracy = tag_accuracy, - relevancy = tag_relevancy - ) - - def get_tags_json(self): - final_list = [] - for tagging_info in self.tagginginfo_set.select_related("tag").all(): - if tagging_info.tag.is_internal(): - final_list.append({ - "tag_label": tagging_info.tag.label, - "tag_link": tagging_info.tag.link, - "accuracy": tagging_info.accuracy, - "relevancy": tagging_info.relevancy - }) - else: - #import label from dbpedia - final_list.append({ - "tag_label": "", - "tag_link": tagging_info.tag.link, - "accuracy": tagging_info.accuracy, - "relevancy": tagging_info.relevancy - }) - return json.dumps(final_list) - + def set_tags(self, tags_json_string): + try: + tags_dict = json.loads(tags_json_string) + except ValueError: + pass + for tag_data in tags_dict: + tag_string = tag_data.get("tag_input") + tag_accuracy = tag_data.get("accuracy", 0) + tag_relevancy = tag_data.get("relevancy", 0) + + if tag_string.startswith("http://") or tag_string.startswith("https://"): #check if url + if Tag.objects.filter(link=tag_string).exists(): #check if tag already exists + tag_obj = Tag.objects.get(link=tag_string) + else: + tag_obj = Tag.objects.create( + link = tag_string, + ) + else: + new_tag_link = settings.BASE_URL+'/'+slugify(tag_string) + if Tag.objects.filter(link=new_tag_link).exists(): + # Somehow we received a label for an existing tag + tag_obj = Tag.objects.get(link=new_tag_link) + else: + tag_obj = Tag.objects.create( + label = tag_string, + description = "", + link = settings.INTERNAL_TAGS_URL+'/'+slugify(tag_string), + collection = self.annotation.image.item.collection + ) + tag_info = TaggingInfo.objects.create( + tag=tag_obj, + revision=self, + accuracy = tag_accuracy, + relevancy = tag_relevancy + ) + + def get_tags_json(self): + + def fetch_from_dbpedia(uri, lang): +# sparql_template = 'PREFIX dbpedia-owl: ' \ +# 'select distinct * where { ' \ +# + 'OPTIONAL { <<%uri%>> rdfs:label ?l FILTER( langMatches( lang(?l), "<%lang%>" ) ) }. ' \ +# + 'OPTIONAL { <<%uri%>> dbpedia-owl:thumbnail ?t }. ' \ +# + 'OPTIONAL { <<%uri%>> dbpedia-owl:abstract ?a FILTER( langMatches( lang(?a), "<%lang%>" ) ) }. ' \ +# + 'OPTIONAL { <<%uri%>> dbpedia-owl:wikiPageRedirects ?r }. ' \ +# + 'OPTIONAL { ?r rdfs:label ?lr FILTER( langMatches( lang(?lr), "<%lang%>" ) ) }. ' \ +# + 'OPTIONAL { ?r dbpedia-owl:thumbnail ?tr }. ' \ +# + 'OPTIONAL { ?r dbpedia-owl:abstract ?ar FILTER( langMatches( lang(?ar), "<%lang%>" ) ) }. ' \ +# + '}' + sparql_template = 'select distinct * where { <<%uri%>> rdfs:label ?l FILTER( langMatches( lang(?l), "<%lang%>" ) ) }' + sparql_query = re.sub("<%uri%>", uri, re.sub("<%lang%>", lang, sparql_template)) + sparql_query_url = "http://dbpedia.org/sparql" + dbpedia_resp = requests.get( + sparql_query_url, + params={ + "query": sparql_query, + "format": "json" + } + ) + print(dbpedia_resp.status_code) + print(dbpedia_resp.text) + return "wow" + + test = fetch_from_dbpedia("http://dbpedia.org/resource/Haiti", "fr") + final_list = [] + for tagging_info in self.tagginginfo_set.select_related("tag").all(): + if tagging_info.tag.is_internal(): + final_list.append({ + "tag_label": tagging_info.tag.label, + "tag_link": tagging_info.tag.link, + "accuracy": tagging_info.accuracy, + "relevancy": tagging_info.relevancy + }) + else: + tag_link = tagging_info.tag.link + #import label from external + externaL_repos_fetch_dict = { + "http://dbpedia.org/": fetch_from_dbpedia, + } + fetch_label = next(handling_function for base_uri, handling_function in externaL_repos_fetch_dict.items() if tag_link.startswith(base_uri)) + tag_label = fetch_label(tag_link, "fr") + final_list.append({ + "tag_label": tag_label, + "tag_link": tag_link, + "accuracy": tagging_info.accuracy, + "relevancy": tagging_info.relevancy + }) + return json.dumps(final_list) + class IconolabComment(XtdComment): - revision = models.ForeignKey('AnnotationRevision', related_name='creation_comment', null=True, blank=True) - metacategories = models.ManyToManyField('MetaCategory', through='MetaCategoryInfo', through_fields=('comment', 'metacategory')) + revision = models.ForeignKey('AnnotationRevision', related_name='creation_comment', null=True, blank=True) + metacategories = models.ManyToManyField('MetaCategory', through='MetaCategoryInfo', through_fields=('comment', 'metacategory')) - + class MetaCategory(models.Model): - collection = models.ForeignKey(Collection) - label = models.CharField(max_length=255) + collection = models.ForeignKey(Collection) + label = models.CharField(max_length=255) - def __str__(self): - return self.label - + def __str__(self): + return self.label + class MetaCategoryInfo(models.Model): - comment = models.ForeignKey('IconolabComment', on_delete=models.CASCADE) - metacategory = models.ForeignKey('MetaCategory', on_delete=models.CASCADE) + comment = models.ForeignKey('IconolabComment', on_delete=models.CASCADE) + metacategory = models.ForeignKey('MetaCategory', on_delete=models.CASCADE) class CommentAttachement(models.Model): - - LINK = 0 - IMAGE = 1 - PDF = 2 - COMMENT_CHOICES = ( - (LINK, 'link'), - (IMAGE, 'image'), - (PDF, 'pdf') - ) - - comment = models.ForeignKey('IconolabComment', related_name='attachments', on_delete=models.CASCADE) - attachment_type = models.IntegerField(choices=COMMENT_CHOICES, default=0) - data = models.TextField(blank=False) \ No newline at end of file + + LINK = 0 + IMAGE = 1 + PDF = 2 + COMMENT_CHOICES = ( + (LINK, 'link'), + (IMAGE, 'image'), + (PDF, 'pdf') + ) + + comment = models.ForeignKey('IconolabComment', related_name='attachments', on_delete=models.CASCADE) + attachment_type = models.IntegerField(choices=COMMENT_CHOICES, default=0) + data = models.TextField(blank=False) \ No newline at end of file diff -r aec0f3381736 -r 8125ce36415c src/iconolab/templates/iconolab/detail_annotation.html --- a/src/iconolab/templates/iconolab/detail_annotation.html Thu Jun 30 14:24:57 2016 +0200 +++ b/src/iconolab/templates/iconolab/detail_annotation.html Thu Jun 30 18:10:17 2016 +0200 @@ -50,7 +50,7 @@

{{ comment.comment }}

{{ comment.submit_date }} -  {{ comment.name }} {% if comment.allow_thread %} -  {% trans "Reply" %}{% endif %}
- {% if comment.revision %}Voir révision{% endif %} + {% if comment.revision %}View revision{% endif %} {% for metacategory in comment.metacategories.all %} {{metacategory.label}} {% endfor %}