--- a/sbin/sync/sync_hdabo Tue Jul 05 17:16:58 2011 +0200
+++ b/sbin/sync/sync_hdabo Fri Jul 08 01:39:02 2011 +0200
@@ -28,5 +28,5 @@
rm -fr ~/tmp/hdabo_V$1;
fi
-ssh iri@web.iri.centrepompidou.fr "export PYTHONPATH=/iridata/www/exp/hdabo && source /srv/virtualenv/hdabo/bin/activate && cd /iridata/www/exp/hdabo/hdabo && python manage.py collectstatic"
+ssh iri@web.iri.centrepompidou.fr "export PYTHONPATH=/iridata/www/exp/hdabo && source /srv/virtualenv/hdabo/bin/activate && cd /iridata/www/exp/hdabo/hdabo && python manage.py collectstatic --noinput"
ssh iri@web.iri.centrepompidou.fr sudo apache2ctl restart
Binary file web/hdabo/fixtures/datasheet_347.yaml.bz2 has changed
--- a/web/hdabo/management/commands/order_tags.py Tue Jul 05 17:16:58 2011 +0200
+++ b/web/hdabo/management/commands/order_tags.py Fri Jul 08 01:39:02 2011 +0200
@@ -89,8 +89,8 @@
ts.save()
ts_list.append(ts)
ts_list.sort(key=lambda t: (-t.index_note, t.order))
- for i, ts in enumerate(ts_list):
- ts.order = i + 1
+ for k, ts in enumerate(ts_list):
+ ts.order = k + 1
ts.save()
if ds.manual_order:
ds.manual_order = False
--- a/web/hdabo/models.py Tue Jul 05 17:16:58 2011 +0200
+++ b/web/hdabo/models.py Fri Jul 08 01:39:02 2011 +0200
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
+from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from hdabo.fields import SortedManyToManyField
@@ -276,5 +277,13 @@
original_order = models.IntegerField(null=False, blank=False, default=0)
order = models.IntegerField(null=False, blank=False, default=0, db_index=True)
index_note = models.FloatField(null=False, blank=False, default=0.0, db_index=True)
+ wikipedia_revision_id = models.BigIntegerField(unique=False, blank=True, null=True)
+
+ @Property
+ def wikipedia_verion_permalink(): #@NoSelf
+ def fget(self):
+ return settings.WIKIPEDIA_VERSION_PERMALINK_TEMPLATE % (unicode(self.wikipedia_revision_id))
+
+ return locals()
--- a/web/hdabo/settings.py Tue Jul 05 17:16:58 2011 +0200
+++ b/web/hdabo/settings.py Fri Jul 08 01:39:02 2011 +0200
@@ -160,7 +160,7 @@
HAYSTACK_SITECONF = 'hdabo.search.sites'
WIKIPEDIA_API_URL = "http://fr.wikipedia.org/w/api.php"
-
+WIKIPEDIA_VERSION_PERMALINK_TEMPLATE = "http://fr.wikipedia.org/w/index.php?oldid=%s"
DBPEDIA_URI_TEMPLATE = "http://dbpedia.org/resource/%s"
SEARCH_STAR_CHARACTER = "*"
Binary file web/hdabo/static/hdabo/img/clock_arrow.png has changed
--- a/web/hdabo/templates/partial/tag_table.html Tue Jul 05 17:16:58 2011 +0200
+++ b/web/hdabo/templates/partial/tag_table.html Fri Jul 08 01:39:02 2011 +0200
@@ -8,6 +8,7 @@
<th>label</th>
{% comment %}<th>original_label</th>{% endcomment %}
<th class="text_centered">Lien W</th>
+ <th class="text_centered">pLien W</th>
<th class="text_centered">Lien D</th>
<th>Catégorie</th>
{% if ordered_tags %}<th class="large_25 text_centered">Supprimer<br/>le lien W</th>{% endif %}
@@ -32,6 +33,13 @@
{% endif %}
</td>
<td class="text_centered">
+ {% if t.wikipedia_revision_id %}
+ <a href="{{t.wikipedia_verion_permalink}}" target="_blank"><img src="{{STATIC_URL}}hdabo/img/clock_arrow.png" ></a>
+ {% else %}
+
+ {% endif %}
+ </td>
+ <td class="text_centered">
{% if t.tag.dbpedia_uri and t.tag.dbpedia_uri != "" %}
<a href="{{t.tag.dbpedia_uri}}" target="_blank"><img src="{{STATIC_URL}}hdabo/img/arrow_green_right.png" ></a>
{% else %}
@@ -58,6 +66,13 @@
{% endif %}
</td>
<td class="text_centered">
+ {% if t.wikipedia_revision_id %}
+ <a href="{{t.wikipedia_verion_permalink}}" target="_blank"><img src="{{STATIC_URL}}hdabo/img/clock_arrow.png" ></a>
+ {% else %}
+
+ {% endif %}
+ </td>
+ <td class="text_centered">
{% if t.dbpedia_uri and t.dbpedia_uri != "" %}
<a href="{{t.dbpedia_uri}}" target="_blank"><img src="{{STATIC_URL}}hdabo/img/arrow_green_right.png" ></a>
{% else %}
--- a/web/hdabo/views.py Tue Jul 05 17:16:58 2011 +0200
+++ b/web/hdabo/views.py Fri Jul 08 01:39:02 2011 +0200
@@ -192,7 +192,7 @@
def get_current_page(num_page=None, nb_by_page=None, searched=None):
- base_queryset = Tag.objects.all()
+ base_queryset = Tag.objects
if searched and searched != "" :
searched = searched.strip()
@@ -257,17 +257,22 @@
tag.label = tag_label
site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable
- wp_res = query_wikipedia_title(site, tag_label)
- status, url, pageid, dbpedia_uri = (wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"])
+ wp_res = query_wikipedia_title(site, label=tag_label)
+ status, url, pageid, dbpedia_uri, revision_id = (wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"], wp_res['revision_id'])
if status is not None:
tag.url_status = status
+ old_pageid = tag.wikipedia_pageid
+
tag.wikipedia_url = url
tag.wikipedia_pageid = pageid
tag.dbpedia_uri = dbpedia_uri
tag.save()
+
+ if old_pageid != pageid:
+ TaggedSheet.objects.filter(tag=tag).update(wikipedia_revision_id=revision_id)
return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], searched=request.POST["searched"])
@@ -288,10 +293,11 @@
if tag_label.lower() in [t.label.lower() for t in ds.tags.all()]:
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s existe déjà pour cette fiche." % (tag_label)}), mimetype="application/json")
- tag, created = get_or_create_tag(tag_label) #@UnusedVariable
+ tag, revision_id, created = get_or_create_tag(tag_label) #@UnusedVariable
ts = TaggedSheet.objects.get(tag=tag_id, datasheet=ds_id)
ts.tag = tag
+ ts.wikipedia_revision_id = revision_id
kwargs = {DJANGO_ID + "__exact": unicode(ds_id)}
results = SearchQuerySet().filter(title=tag_label).filter_or(description=tag_label).filter(**kwargs)
if len(results) > 0:
@@ -322,7 +328,9 @@
tag.wikipedia_url = None
tag.wikipedia_pageid = None
- process_tag(site, tag, 0)
+ process_tag(site, tag)
+
+
if u"datasheet_id" in request.POST :
return get_tag_table(request=request, ds_id=request.POST["datasheet_id"], valid=0)
@@ -341,14 +349,14 @@
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s existe déjà pour cette fiche." % (tag_label)}), mimetype="application/json")
- tag, created = get_or_create_tag(tag_label)
+ tag, revision_id, created = get_or_create_tag(tag_label)
# We put the tag at the bottom of the datasheet's tag list
# if the tag is created or if the tag is not in the list
list_ts = TaggedSheet.objects.filter(datasheet=ds)
- if created or len(list_ts.filter(tag=tag)) == 0 :
+ if created or list_ts.filter(tag=tag).count() == 0 :
new_order = list_ts.aggregate(Max('order'))['order__max'] + 1
- ts = TaggedSheet.objects.create(datasheet=ds, tag=tag, original_order=new_order, order=new_order)
+ ts = TaggedSheet.objects.create(datasheet=ds, tag=tag, original_order=new_order, order=new_order, wikipedia_revision_id=revision_id)
ts.save()
ds.manual_order = True
ds.save()
--- a/web/hdabo/wp_utils.py Tue Jul 05 17:16:58 2011 +0200
+++ b/web/hdabo/wp_utils.py Fri Jul 08 01:39:02 2011 +0200
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
from django.conf import settings
-from hdabo.models import Tag
+from hdabo.models import Tag, TaggedSheet
from wikitools import api, wiki
from django.utils.http import urlquote
@@ -13,7 +13,7 @@
tag = tag[0].upper() + tag[1:]
return tag
-def urlize_for_wkipedia(label):
+def urlize_for_wikipedia(label):
return urlquote(label.replace(" ","_"))
@@ -23,24 +23,32 @@
return True
return False
-def query_wikipedia_title(site, label):
+def query_wikipedia_title(site, label=None, pageid=None):
- params = {'action':'query', 'titles': label, 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500'}
+ params = {'action':'query', 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500', 'rvprop':'ids'}
+
+ if label:
+ params['titles'] = label
+ else:
+ params['pageids'] = pageid
wpquery = api.APIRequest(site, params) #@UndefinedVariable
response = wpquery.query()
original_response = response
+ def return_null_result():
+ return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'revision_id': None,'response': response }
+
query_dict = response['query']
# get page if multiple pages or none -> return Tag.null_result
pages = query_dict.get("pages", {})
if len(pages) > 1 or len(pages) == 0:
- return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'response': response }
+ return return_null_result()
page = pages.values()[0]
if u"invalid" in page or u"missing" in page:
- return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'response': response }
+ return return_null_result()
url = page.get(u'fullurl', None)
pageid = page.get(u'pageid', None)
@@ -54,17 +62,18 @@
status = Tag.TAG_URL_STATUS_DICT["match"]
if status == Tag.TAG_URL_STATUS_DICT["redirection"]:
- params = {'action':'query', 'titles': label, 'prop':'info|categories|langlinks', 'inprop':'url', 'lllimit':'500', 'cllimit':'500', 'redirects':True}
+ params['redirects'] = True
wpquery = api.APIRequest(site, params) #@UndefinedVariable
response = wpquery.query()
query_dict = response['query']
pages = query_dict.get("pages", {})
#we know that we have at least one answer
if len(pages) > 1 or len(pages) == 0:
- return { 'new_label': None, 'status': Tag.TAG_URL_STATUS_DICT["null_result"], 'wikipedia_url': None, 'pageid': None, 'dbpedia_uri': None, 'response': response }
+ return return_null_result()
page = pages.values()[0]
+ revision_id = page.get('lastrevid', None)
#process language to extract the english label
english_label = None
@@ -77,11 +86,12 @@
break
if english_label and "#" not in english_label:
- dbpedia_uri = settings.DBPEDIA_URI_TEMPLATE % (urlize_for_wkipedia(english_label))
+ dbpedia_uri = settings.DBPEDIA_URI_TEMPLATE % (urlize_for_wikipedia(english_label))
else:
dbpedia_uri = None
+
- return { 'new_label': new_label, 'status': status, 'wikipedia_url': url, 'pageid': pageid, 'dbpedia_uri': dbpedia_uri, 'response': original_response }
+ return { 'new_label': new_label, 'status': status, 'wikipedia_url': url, 'pageid': pageid, 'dbpedia_uri': dbpedia_uri, 'revision_id': revision_id, 'response': original_response }
@@ -93,10 +103,11 @@
tag, created = Tag.objects.get_or_create(label__iexact=tag_label_normalized, defaults={'label':tag_label_normalized, 'original_label':tag_label})
+ site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable
+
if created:
- site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable
- wp_res = query_wikipedia_title(site, tag_label_normalized) #@UnusedVariable
- new_label, status, url, pageid, dbpedia_uri = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"]
+ wp_res = query_wikipedia_title(site, label=tag_label_normalized)
+ new_label, status, url, pageid, dbpedia_uri, wikipedia_revision_id = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"], wp_res['revision_id']
# We save the datas
@@ -110,17 +121,26 @@
tag.save()
- return tag, created
+ elif tag.wikipedia_pageid:
+ wp_res = query_wikipedia_title(site, pageid=tag.wikipedia_pageid)
+ wikipedia_revision_id = wp_res['revision_id']
+ else:
+ wikipedia_revision_id = None
+
+
+ return tag, wikipedia_revision_id, created
-def process_tag(site, tag, verbosity):
+def process_tag(site, tag, verbosity=0):
- wp_res = query_wikipedia_title(site, tag.label)
- new_label, status, url, pageid, response, dbpedia_uri = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res['response'], wp_res["dbpedia_uri"]
+ wp_res = query_wikipedia_title(site, label=tag.label)
+ new_label, status, url, pageid, response, dbpedia_uri, revision_id = wp_res['new_label'], wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res['response'], wp_res["dbpedia_uri"], wp_res["revision_id"]
if verbosity >= 2 :
print "response from query to %s with parameters %s :" % (site.apibase, repr(new_label))
print repr(response)
+ prev_wikipedia_pageid = tag.wikipedia_pageid
+
if new_label is not None:
tag.label = new_label
if status is not None:
@@ -130,6 +150,9 @@
tag.dbpedia_uri = dbpedia_uri
tag.save()
-
+
+ if prev_wikipedia_pageid != pageid:
+ TaggedSheet.objects.filter(tag=tag).update(wikipedia_revision_id=revision_id)
+
