# HG changeset patch # User ymh # Date 1310081942 -7200 # Node ID 289ded09831619e98470049a7dae167fe4e14f4d # Parent 953ee4c9c7c3417f4fb72561777d691688143c5e add revision link cf bug #12 diff -r 953ee4c9c7c3 -r 289ded098316 sbin/sync/sync_hdabo --- a/sbin/sync/sync_hdabo Tue Jul 05 17:16:58 2011 +0200 +++ b/sbin/sync/sync_hdabo Fri Jul 08 01:39:02 2011 +0200 @@ -28,5 +28,5 @@ rm -fr ~/tmp/hdabo_V$1; fi -ssh iri@web.iri.centrepompidou.fr "export PYTHONPATH=/iridata/www/exp/hdabo && source /srv/virtualenv/hdabo/bin/activate && cd /iridata/www/exp/hdabo/hdabo && python manage.py collectstatic" +ssh iri@web.iri.centrepompidou.fr "export PYTHONPATH=/iridata/www/exp/hdabo && source /srv/virtualenv/hdabo/bin/activate && cd /iridata/www/exp/hdabo/hdabo && python manage.py collectstatic --noinput" ssh iri@web.iri.centrepompidou.fr sudo apache2ctl restart diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/fixtures/datasheet_347.yaml.bz2 Binary file web/hdabo/fixtures/datasheet_347.yaml.bz2 has changed diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/management/commands/order_tags.py --- a/web/hdabo/management/commands/order_tags.py Tue Jul 05 17:16:58 2011 +0200 +++ b/web/hdabo/management/commands/order_tags.py Fri Jul 08 01:39:02 2011 +0200 @@ -89,8 +89,8 @@ ts.save() ts_list.append(ts) ts_list.sort(key=lambda t: (-t.index_note, t.order)) - for i, ts in enumerate(ts_list): - ts.order = i + 1 + for k, ts in enumerate(ts_list): + ts.order = k + 1 ts.save() if ds.manual_order: ds.manual_order = False diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/models.py --- a/web/hdabo/models.py Tue Jul 05 17:16:58 2011 +0200 +++ b/web/hdabo/models.py Fri Jul 08 01:39:02 2011 +0200 @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from django.conf import settings from django.contrib.auth.models import User from django.db import models from hdabo.fields import SortedManyToManyField @@ -276,5 +277,13 @@ original_order = models.IntegerField(null=False, blank=False, default=0) order = models.IntegerField(null=False, blank=False, default=0, db_index=True) index_note = models.FloatField(null=False, blank=False, default=0.0, db_index=True) + wikipedia_revision_id = models.BigIntegerField(unique=False, blank=True, null=True) + + @Property + def wikipedia_verion_permalink(): #@NoSelf + def fget(self): + return settings.WIKIPEDIA_VERSION_PERMALINK_TEMPLATE % (unicode(self.wikipedia_revision_id)) + + return locals() diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/settings.py --- a/web/hdabo/settings.py Tue Jul 05 17:16:58 2011 +0200 +++ b/web/hdabo/settings.py Fri Jul 08 01:39:02 2011 +0200 @@ -160,7 +160,7 @@ HAYSTACK_SITECONF = 'hdabo.search.sites' WIKIPEDIA_API_URL = "http://fr.wikipedia.org/w/api.php" - +WIKIPEDIA_VERSION_PERMALINK_TEMPLATE = "http://fr.wikipedia.org/w/index.php?oldid=%s" DBPEDIA_URI_TEMPLATE = "http://dbpedia.org/resource/%s" SEARCH_STAR_CHARACTER = "*" diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/static/hdabo/img/clock_arrow.png Binary file web/hdabo/static/hdabo/img/clock_arrow.png has changed diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/templates/partial/tag_table.html --- a/web/hdabo/templates/partial/tag_table.html Tue Jul 05 17:16:58 2011 +0200 +++ b/web/hdabo/templates/partial/tag_table.html Fri Jul 08 01:39:02 2011 +0200 @@ -8,6 +8,7 @@ label {% comment %}original_label{% endcomment %} Lien W + pLien W Lien D Catégorie {% if ordered_tags %}Supprimer
le lien W{% endif %} @@ -32,6 +33,13 @@ {% endif %} + {% if t.wikipedia_revision_id %} + + {% else %} +   + {% endif %} + + {% if t.tag.dbpedia_uri and t.tag.dbpedia_uri != "" %} {% else %} @@ -58,6 +66,13 @@ {% endif %} + {% if t.wikipedia_revision_id %} + + {% else %} +   + {% endif %} + + {% if t.dbpedia_uri and t.dbpedia_uri != "" %} {% else %} diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/views.py --- a/web/hdabo/views.py Tue Jul 05 17:16:58 2011 +0200 +++ b/web/hdabo/views.py Fri Jul 08 01:39:02 2011 +0200 @@ -192,7 +192,7 @@ def get_current_page(num_page=None, nb_by_page=None, searched=None): - base_queryset = Tag.objects.all() + base_queryset = Tag.objects if searched and searched != "" : searched = searched.strip() @@ -257,17 +257,22 @@ tag.label = tag_label site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable - wp_res = query_wikipedia_title(site, tag_label) - status, url, pageid, dbpedia_uri = (wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"]) + wp_res = query_wikipedia_title(site, label=tag_label) + status, url, pageid, dbpedia_uri, revision_id = (wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"], wp_res['revision_id']) if status is not None: tag.url_status = status + old_pageid = tag.wikipedia_pageid + tag.wikipedia_url = url tag.wikipedia_pageid = pageid tag.dbpedia_uri = dbpedia_uri tag.save() + + if old_pageid != pageid: + TaggedSheet.objects.filter(tag=tag).update(wikipedia_revision_id=revision_id) return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], searched=request.POST["searched"]) @@ -288,10 +293,11 @@ if tag_label.lower() in [t.label.lower() for t in ds.tags.all()]: return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s existe déjà pour cette fiche." % (tag_label)}), mimetype="application/json") - tag, created = get_or_create_tag(tag_label) #@UnusedVariable + tag, revision_id, created = get_or_create_tag(tag_label) #@UnusedVariable ts = TaggedSheet.objects.get(tag=tag_id, datasheet=ds_id) ts.tag = tag + ts.wikipedia_revision_id = revision_id kwargs = {DJANGO_ID + "__exact": unicode(ds_id)} results = SearchQuerySet().filter(title=tag_label).filter_or(description=tag_label).filter(**kwargs) if len(results) > 0: @@ -322,7 +328,9 @@ tag.wikipedia_url = None tag.wikipedia_pageid = None - process_tag(site, tag, 0) + process_tag(site, tag) + + if u"datasheet_id" in request.POST : return get_tag_table(request=request, ds_id=request.POST["datasheet_id"], valid=0) @@ -341,14 +349,14 @@ return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s existe déjà pour cette fiche." % (tag_label)}), mimetype="application/json") - tag, created = get_or_create_tag(tag_label) + tag, revision_id, created = get_or_create_tag(tag_label) # We put the tag at the bottom of the datasheet's tag list # if the tag is created or if the tag is not in the list list_ts = TaggedSheet.objects.filter(datasheet=ds) - if created or len(list_ts.filter(tag=tag)) == 0 : + if created or list_ts.filter(tag=tag).count() == 0 : new_order = list_ts.aggregate(Max('order'))['order__max'] + 1 - ts = TaggedSheet.objects.create(datasheet=ds, tag=tag, original_order=new_order, order=new_order) + ts = TaggedSheet.objects.create(datasheet=ds, tag=tag, original_order=new_order, order=new_order, wikipedia_revision_id=revision_id) ts.save() ds.manual_order = True ds.save() diff -r 953ee4c9c7c3 -r 289ded098316 web/hdabo/wp_utils.py --- a/web/hdabo/wp_utils.py Tue Jul 05 17:16:58 2011 +0200 +++ b/web/hdabo/wp_utils.py Fri Jul 08 01:39:02 2011 +0200 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from django.conf import settings -from hdabo.models import Tag +from hdabo.models import Tag, TaggedSheet from wikitools import api, wiki from django.utils.http import urlquote @@ -13,7 +13,7 @@ tag = tag[0].upper() + tag[1:] return tag -def urlize_for_wkipedia(label): +def urlize_for_wikipedia(label): return urlquote(label.replace(" ","_")) @@ -23,24 +23,32 @@ return True return False -def query_wikipedia_title(site, label): +def query_wikipedia_title(site, label=None, pageid=None): - params = {'action':'query', 'titles': label, 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500'} + params = {'action':'query', 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500', 'rvprop':'ids'} + + if label: + params['titles'] = label + else: + params['pageids'] = pageid wpquery = api.APIRequest(site, params) #@UndefinedVariable response = wpquery.query() original_response = response + def return_null_result(): + return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'revision_id': None,'response': response } + query_dict = response['query'] # get page if multiple pages or none -> return Tag.null_result pages = query_dict.get("pages", {}) if len(pages) > 1 or len(pages) == 0: - return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'response': response } + return return_null_result() page = pages.values()[0] if u"invalid" in page or u"missing" in page: - return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'response': response } + return return_null_result() url = page.get(u'fullurl', None) pageid = page.get(u'pageid', None) @@ -54,17 +62,18 @@ status = Tag.TAG_URL_STATUS_DICT["match"] if status == Tag.TAG_URL_STATUS_DICT["redirection"]: - params = {'action':'query', 'titles': label, 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500', 'redirects':True} + params['redirects'] = True wpquery = api.APIRequest(site, params) #@UndefinedVariable response = wpquery.query() query_dict = response['query'] pages = query_dict.get("pages", {}) #we know that we have at least one answer if len(pages) > 1 or len(pages) == 0: - return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'response': response } + return return_null_result() page = pages.values()[0] + revision_id = page.get('lastrevid', None) #process language to extract the english label english_label = None @@ -77,11 +86,12 @@ break if english_label and "#" not in english_label: - dbpedia_uri = settings.DBPEDIA_URI_TEMPLATE % (urlize_for_wkipedia(english_label)) + dbpedia_uri = settings.DBPEDIA_URI_TEMPLATE % (urlize_for_wikipedia(english_label)) else: dbpedia_uri = None + - return { 'new_label': new_label, 'status': status, 'wikipedia_url': url, 'pageid': pageid, 'dbpedia_uri': dbpedia_uri, 'response': original_response } + return { 'new_label': new_label, 'status': status, 'wikipedia_url': url, 'pageid': pageid, 'dbpedia_uri': dbpedia_uri, 'revision_id': revision_id, 'response': original_response } @@ -93,10 +103,11 @@ tag, created = Tag.objects.get_or_create(label__iexact=tag_label_normalized, defaults={'label':tag_label_normalized, 'original_label':tag_label}) + site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable + if created: - site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable - wp_res = query_wikipedia_title(site, tag_label_normalized) #@UnusedVariable - new_label, status, url, pageid, dbpedia_uri = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"] + wp_res = query_wikipedia_title(site, label=tag_label_normalized) + new_label, status, url, pageid, dbpedia_uri, wikipedia_revision_id = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"], wp_res['revision_id'] # We save the datas @@ -110,17 +121,26 @@ tag.save() - return tag, created + elif tag.wikipedia_pageid: + wp_res = query_wikipedia_title(site, pageid=tag.wikipedia_pageid) + wikipedia_revision_id = wp_res['revision_id'] + else: + wikipedia_revision_id = None + + + return tag, wikipedia_revision_id, created -def process_tag(site, tag, verbosity): +def process_tag(site, tag, verbosity=0): - wp_res = query_wikipedia_title(site, tag.label) - new_label, status, url, pageid, response, dbpedia_uri = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res['response'], wp_res["dbpedia_uri"] + wp_res = query_wikipedia_title(site, label=tag.label) + new_label, status, url, pageid, response, dbpedia_uri, revision_id = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res['response'], wp_res["dbpedia_uri"], wp_res["revision_id"] if verbosity >= 2 : print "response from query to %s with parameters %s :" % (site.apibase, repr(new_label)) print repr(response) + prev_wikipedia_pageid = tag.wikipedia_pageid + if new_label is not None: tag.label = new_label if status is not None: @@ -130,6 +150,9 @@ tag.dbpedia_uri = dbpedia_uri tag.save() - + + if prev_wikipedia_pageid != pageid: + TaggedSheet.objects.filter(tag=tag).update(wikipedia_revision_id=revision_id) +