# -*- coding: utf-8 -*-
from django.conf import settings
from django.contrib.auth.decorators import login_required #@UnusedImport
from django.core.paginator import Paginator
from django.db import connection
from django.db.models import Max, Count, Min
from django.http import HttpResponseBadRequest
from django.shortcuts import render_to_response, redirect
from django.template import RequestContext
from django.utils.http import urlquote
from haystack.constants import DJANGO_ID
from haystack.query import SearchQuerySet
from hdabo.utils import OrderedDict, remove_accents, normalize
from hdabo.wp_utils import (normalize_tag, query_wikipedia_title,
get_or_create_tag, process_tag, reorder_datasheet_tags)
from models import Datasheet, Organisation, Tag, TagCategory, TaggedSheet
from wikitools import wiki
import django.utils.simplejson as json
import re
@login_required
def home(request):
orgas = Organisation.objects.all().order_by('name')
org_list = []
all_ds_mapping = dict([(res['organisation'], res['nb_all']) for res in Datasheet.objects.values("organisation").annotate(nb_all=Count("organisation"))])
validated_ds_mapping = dict([(res['organisation'], [res['nb_val'], res['first_id_val']]) for res in Datasheet.objects.filter(validated=True).values("organisation").annotate(nb_val=Count("organisation")).annotate(first_id_val=Min("id"))])
unvalidated_ds_mapping = dict([(res['organisation'], [res['nb_unval'], res['first_id_unval']]) for res in Datasheet.objects.filter(validated=False).values("organisation").annotate(nb_unval=Count("organisation")).annotate(first_id_unval=Min("id"))])
# We get the hda_id from the id
val_hda_ids = dict([(res.id, res.hda_id) for res in Datasheet.objects.filter(id__in=[v[1] for v in validated_ds_mapping.values()]).only("id", "hda_id")])
unval_hda_ids = dict([(res.id, res.hda_id) for res in Datasheet.objects.filter(id__in=[v[1] for v in unvalidated_ds_mapping.values()]).only("id", "hda_id")])
for orga in orgas :
nb_all = all_ds_mapping.get(orga.id, 0)
duo_val = validated_ds_mapping.get(orga.id, [0, None])
nb_val = duo_val[0]
first_id_val = val_hda_ids.get(duo_val[1], None)
duo_unval = unvalidated_ds_mapping.get(orga.id, [0, None])
nb_unval = duo_unval[0]
first_id_unval = unval_hda_ids.get(duo_unval[1], None)
org_list.append({'organisation':orga, 'nb_all':nb_all, 'nb_val':nb_val, 'first_id_val':first_id_val, 'nb_unval':nb_unval, 'first_id_unval':first_id_unval})
return render_to_response("organisation_list.html",
{'organisations':org_list},
context_instance=RequestContext(request))
@login_required
def display_datasheet(request, ds_id=None):
if ds_id :
ds = Datasheet.objects.get(hda_id=ds_id)
# In this function the context is given by the get parameters.
if "index" in request.GET :
try:
index = int(request.GET["index"])
except :
index = 0
else :
index = 0
if "nb_sheets" in request.GET :
try:
nb_sheets = int(request.GET["nb_sheets"])
except :
nb_sheets = None
else :
nb_sheets = None
# If there is tag parameter, it means that we display all the ds tagged by one particular tag
tag = None
if "tag" in request.GET :
tag = Tag.objects.get(id=int(request.GET["tag"]))
datasheets_qs = Datasheet.objects.filter(tags__in=[tag]).order_by("organisation__name", "original_creation_date").select_related("format")
# If tag is set and if ds_id is None, it means that we have to display the first ds
if not ds_id :
index = 0
ds = datasheets_qs[0]
else :
# The current list depends on the ds's validation state
datasheets_qs = Datasheet.objects.filter(organisation=ds.organisation).filter(validated=ds.validated).order_by("id").select_related("format")
# We calculate the number of sheets if necessary
if not nb_sheets :
nb_sheets = datasheets_qs.count()
# We get only the prev/current/next sheets
if nb_sheets > 1 :
if index == 0 :
select_qs = datasheets_qs[:2]
prev_index = 0
prev_id = select_qs[0].hda_id
next_index = 1
next_id = select_qs[1].hda_id
elif index == (nb_sheets - 1) :
select_qs = datasheets_qs[nb_sheets - 2:]
prev_index = nb_sheets - 2
prev_id = select_qs[0].hda_id
next_index = nb_sheets - 1
next_id = select_qs[1].hda_id
else :
select_qs = datasheets_qs[index - 1:index + 2]
prev_index = index - 1
prev_id = select_qs[0].hda_id
next_index = index + 1
next_id = select_qs[2].hda_id
else :
prev_index = next_index = 0
prev_id = next_id = datasheets_qs[0].hda_id
# We get the ORDERED tags if we display one sheet (case valid = 0 and 1)
ordered_tags = TaggedSheet.objects.filter(datasheet=ds).select_related("tag").order_by('order')
displayed_index = index + 1;
zero_id = datasheets_qs[0].hda_id
last_index = max(nb_sheets - 1, 0)
last_id = datasheets_qs[last_index].hda_id
return render_to_response("generic_sheet.html",
{'ds':ds, 'orga_name':ds.organisation.name,
'nb_sheets':nb_sheets, 'ordered_tags':ordered_tags,
'zero_id':zero_id, 'prev_index':prev_index, 'prev_id':prev_id,
'next_index':next_index, 'next_id':next_id,
'last_index':last_index, 'last_id':last_id,
'displayed_index':displayed_index, 'tag':tag, 'valid':ds.validated,
'categories':json.dumps(get_categories())},
context_instance=RequestContext(request))
@login_required
def list_for_orga(request, orga_id=None, valid=None, start_index=None):
orga = Organisation.objects.get(id=orga_id)
orga_name = orga.name
if start_index :
try:
start_index = int(start_index)
except :
start_index = 0
else :
start_index = 0
# If valid = 0, we search unvalidated sheets
# If valid = 1, we search validated sheets
# If valid = 2, we search AND DISPLAY all sheets
datasheets_qs = Datasheet.objects.filter(organisation=orga).order_by("id").select_related("format")
if valid == "1" :
# We count all the validated sheets
datasheets_qs = datasheets_qs.filter(validated=True)
# And select the current one
datasheets = [datasheets_qs[start_index]]
elif valid != "2":
valid = "0"
# We count all the validated sheets
datasheets_qs = datasheets_qs.filter(validated=False)
# And select the current one
datasheets = [datasheets_qs[start_index]]
else:
datasheets = datasheets_qs
nb_sheets = datasheets_qs.count()
# We get the ORDERED tags if we display one sheet (case valid = 0 and 1)
ordered_tags = None
if valid != "2" :
ordered_tags = TaggedSheet.objects.filter(datasheet=datasheets[0]).select_related("tag").order_by('order')
displayed_index = start_index + 1;
prev_index = max(start_index - 1, 0);
next_index = min(nb_sheets - 1, start_index + 1);
last_index = max(nb_sheets - 1, 0);
return render_to_response("list_for_orga.html",
{'datasheets':datasheets, 'orga_name':orga_name,
'nb_sheets':nb_sheets, 'orga_id':orga_id, 'ordered_tags':ordered_tags,
'prev_index':prev_index, 'next_index':next_index, 'last_index':last_index,
'start_index':start_index, 'displayed_index':displayed_index, 'valid':valid,
'categories':json.dumps(get_categories())},
context_instance=RequestContext(request))
@login_required
def all_tags(request, num_page=None, nb_by_page=None, sort="+pop", searched=None):
# If the view is asked after a form sent with post vars, it means that searched is a post var.
if u"searched" in request.POST :
searched = request.POST["searched"]
nb_by_page = settings.PAGINATION_DEFAULT_NB_BY_PAGE
num_page = 1
# Get paginator and current page
current_page, p, num_page, nb_by_page = get_current_page(num_page, nb_by_page, sort, searched)
prev_page = max(num_page - 1, 1)
next_page = min(num_page + 1, p.num_pages)
last_page = p.num_pages
cursor = connection.cursor() #@UndefinedVariable
fl_list = []
try:
cursor.execute("select upper(substring(normalized_label from 1 for 1)) as fl from hdabo_tag group by fl order by fl;")
for row in cursor:
new_char = remove_accents(row[0])
if new_char not in fl_list:
fl_list.append(new_char)
finally:
cursor.close()
search_def = tuple([(c, urlquote(c + settings.SEARCH_STAR_CHARACTER)) for c in fl_list])
reverse_sort = ("-" if sort[0] == "+" else "+") + sort[1:]
return render_to_response("all_tags.html",
{'nb_total':p.count, 'tags':current_page.object_list, 'current_page':current_page,
'prev_page':prev_page, 'next_page':next_page, 'last_page':last_page,
'num_page':num_page, 'nb_by_page':nb_by_page, 'searched':searched,
'categories':json.dumps(get_categories()),
'search_def':search_def, 'sort':sort, 'reverse_sort': reverse_sort},
context_instance=RequestContext(request))
@login_required
def tag_up_down(request):
ds_id = request.POST["datasheet_id"]
# post vars new_order and old_order indicate the position (from 1) of the tag in the list.
# NB : it is different from the TagSheet.order in the database.
new_order = int(request.POST["new_order"]) - 1
old_order = int(request.POST["old_order"]) - 1
# First we get the datasheet's TaggedSheets (list to force evaluation)
ordered_tags = list(TaggedSheet.objects.filter(datasheet__hda_id=ds_id).order_by('order'))
# We change the moved TaggedSheets's order
new_ts_order = ordered_tags[new_order].order
moved_ts = ordered_tags[old_order]
moved_ts.order = new_ts_order
moved_ts.save()
# We move the TaggedSheets's order
if new_order > old_order :
# And we decrease the other ones
for i in range(old_order + 1, new_order + 1) :
ts = ordered_tags[i]
ts.order = ts.order - 1
ts.save()
else :
# And we increase the other ones
for i in range(new_order, old_order) :
ts = ordered_tags[i]
ts.order = ts.order + 1
ts.save()
ds = Datasheet.objects.get(hda_id=ds_id)
ds.manual_order = True
ds.save()
return get_tag_table(request=request, ds_id=ds_id, valid=0)
@login_required
def get_tag_table(request=None, ds_id=None, valid=None):
ordered_tags = TaggedSheet.objects.filter(datasheet__hda_id=ds_id).order_by('order')
return render_to_response("partial/tag_table.html",
{'ordered_tags':ordered_tags, 'valid':valid},
context_instance=RequestContext(request))
@login_required
def get_all_tags_table(request, num_page=None, nb_by_page=None, sort="+pop", searched=None):
current_page, p, num_page, nb_by_page = get_current_page(num_page, nb_by_page, sort, searched) #@UnusedVariable
reverse_sort = ("-" if sort[0] == "+" else "+") + sort[1:]
return render_to_response("partial/all_tags_table.html",
{'tags':current_page.object_list, 'nb_by_page':nb_by_page, 'searched':searched, 'sort': sort, 'reverse_sort': reverse_sort},
context_instance=RequestContext(request))
def get_current_page(num_page=None, nb_by_page=None, sort="+pop", searched=None):
base_queryset = Tag.objects
if searched and searched != "" :
searched = normalize(searched.strip())
regex = "^%s$" % (re.escape(searched).replace(re.escape(settings.SEARCH_STAR_CHARACTER), ".*"))
base_queryset = base_queryset.filter(normalized_label__iregex=regex)
alltags = base_queryset.annotate(num_ds=Count('datasheet'))
alltags = {
"+lab" : alltags.order_by('normalized_label', 'label'),
"-lab" : alltags.order_by('-normalized_label', '-label'),
"+pop" : alltags.order_by('-popularity', '-num_ds', 'normalized_label', 'label'),
"-pop" : alltags.order_by('popularity', 'num_ds', '-normalized_label', '-label'),
}.get(sort, alltags.order_by('-popularity', '-num_ds', 'normalized_label', 'label'))
# We build the paginator for the requested list
if nb_by_page :
try:
nb_by_page = int(nb_by_page)
except :
nb_by_page = settings.PAGINATION_DEFAULT_NB_BY_PAGE
else :
nb_by_page = settings.PAGINATION_DEFAULT_NB_BY_PAGE
if num_page :
try:
num_page = int(num_page)
except :
num_page = 1
else :
num_page = 1
p = Paginator(alltags, nb_by_page)
#use the much simpler base query to have the count
p._count = base_queryset.count()
current_page = p.page(num_page)
return current_page, p, num_page, nb_by_page
@login_required
def remove_tag_from_list(request=None):
ds_id = request.POST["datasheet_id"]
tag_id = request.POST["tag_id"]
# First we get the datasheet's TaggedSheets
ds_tags = TaggedSheet.objects.filter(datasheet__hda_id=ds_id)
# We get the current TaggedSheet and we delete it
ts = ds_tags.filter(tag=Tag.objects.filter(id=tag_id))[0]
ts.delete()
ds = Datasheet.objects.get(hda_id=ds_id)
ds.manual_order = True
ds.save()
return get_tag_table(request=request, ds_id=ds_id, valid=0)
@login_required
def modify_tag(request):
tag_id = request.POST["id"]
tag_label = request.POST["value"]
tag = Tag.objects.get(id=tag_id)
if tag.label != tag_label:
tag.label = tag_label
site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable
wp_res = query_wikipedia_title(site, label=tag_label)
status, url, pageid, dbpedia_uri, revision_id = (wp_res['status'], wp_res['wikipedia_url'], wp_res['pageid'], wp_res["dbpedia_uri"], wp_res['revision_id'])
if status is not None:
tag.url_status = status
old_pageid = tag.wikipedia_pageid
tag.wikipedia_url = url
tag.wikipedia_pageid = pageid
tag.dbpedia_uri = dbpedia_uri
try:
tag.save()
except:
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s (%s) existe déjà." % (tag_label, tag.original_label)}), mimetype="application/json")
if old_pageid != pageid:
TaggedSheet.objects.filter(tag=tag).update(wikipedia_revision_id=revision_id)
return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], sort=request.POST["sort"], searched=request.POST["searched"])
@login_required
def modify_tag_datasheet(request):
tag_id = request.POST["id"]
tag_label = request.POST["value"]
ds_id = request.POST["datasheet_id"]
tag = Tag.objects.get(id=tag_id)
ds = Datasheet.objects.get(hda_id=ds_id)
if tag.label != tag_label:
if tag_label.lower() in [t.label.lower() for t in ds.tags.all()]:
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s existe déjà pour cette fiche." % (tag_label)}), mimetype="application/json")
tag, revision_id, created = get_or_create_tag(tag_label) #@UnusedVariable
ts = TaggedSheet.objects.get(tag=tag_id, datasheet__hda_id=ds_id)
ts.tag = tag
ts.wikipedia_revision_id = revision_id
kwargs = {DJANGO_ID + "__exact": unicode(ds_id)}
results = SearchQuerySet().filter(title=tag_label).filter_or(description=tag_label).filter(**kwargs)
if len(results) > 0:
ts.index_note = results[0].score
ts.save()
ds.manual_order = True
ds.save()
return get_tag_table(request=request, ds_id=ds_id, valid=0)
@login_required
def reset_wikipedia_info(request):
# 2 cases :
# - ordered tag for one datasheet : POST["datasheet_id"] is not null
# - all tags list : POST["datasheet_id"] is null and POST["num_page"] and POST["nb_by_page"] are not null
tag_id = request.POST["tag_id"]
tag = Tag.objects.get(id=tag_id)
site = wiki.Wiki(settings.WIKIPEDIA_API_URL) #@UndefinedVariable
tag.label = normalize_tag(tag.original_label)
#reset tag info
tag.wikipedia_url = None
tag.wikipedia_pageid = None
try:
process_tag(site, tag)
except:
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"La version sémantisée du tag %s (%s) existe déjà." % (tag.label, tag.original_label)}), mimetype="application/json")
if u"datasheet_id" in request.POST :
return get_tag_table(request=request, ds_id=request.POST["datasheet_id"], valid=0)
else :
return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], sort=request.POST["sort"], searched=request.POST["searched"])
@login_required
def add_tag(request=None):
ds_id = request.POST["datasheet_id"]
tag_label = request.POST["value"]
ds = Datasheet.objects.get(hda_id=ds_id)
if tag_label.lower() in [t.label.lower() for t in ds.tags.all()]:
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"Le tag %s existe déjà pour cette fiche." % (tag_label)}), mimetype="application/json")
tag, revision_id, created = get_or_create_tag(tag_label)
# We put the tag at the bottom of the datasheet's tag list
# if the tag is created or if the tag is not in the list
list_ts = TaggedSheet.objects.filter(datasheet=ds)
if created or list_ts.filter(tag=tag).count() == 0 :
new_order = list_ts.aggregate(Max('order'))['order__max'] + 1
ts = TaggedSheet.objects.create(datasheet=ds, tag=tag, original_order=new_order, order=new_order, wikipedia_revision_id=revision_id)
ts.save()
ds.manual_order = True
ds.save()
return get_tag_table(request=request, ds_id=ds_id, valid=0)
@login_required
def remove_wp_link(request=None):
# 2 cases :
# - ordered tag for one datasheet : POST["datasheet_id"] is not null
# - all tags list : POST["datasheet_id"] is null and POST["num_page"] and POST["nb_by_page"] are not null
tag_id = request.POST["tag_id"]
tag = Tag.objects.get(id=tag_id)
if u"datasheet_id" in request.POST :
#create another tag almost identical, without the W info
new_tag, created = Tag.objects.get_or_create(label=tag.label, original_label=tag.original_label, url_status=Tag.TAG_URL_STATUS_DICT['null_result'], #@UnusedVariable
defaults={'label':tag.label, 'original_label': tag.original_label, 'url_status':Tag.TAG_URL_STATUS_DICT['null_result'], 'wikipedia_url': None, 'wikipedia_pageid': None, 'dbpedia_uri': None, 'category':tag.category, 'alias':tag.alias, 'popularity':tag.popularity})
TaggedSheet.objects.filter(tag=tag, datasheet__hda_id=request.POST["datasheet_id"]).update(tag=new_tag, wikipedia_revision_id=None)
else:
if Tag.objects.filter(label=tag.label, original_label=tag.original_label, url_status=Tag.TAG_URL_STATUS_DICT['null_result']).count() > 0:
return HttpResponseBadRequest(json.dumps({'error': 'duplicate_tag', 'message': u"La version désémantisée du tag %s (%s) existe déjà." % (tag.label, tag.original_label)}), mimetype="application/json")
tag.wikipedia_url = None
tag.wikipedia_pageid = None
tag.dbpedia_uri = None
tag.url_status = Tag.TAG_URL_STATUS_DICT['null_result']
tag.save()
TaggedSheet.objects.filter(tag=tag).update(wikipedia_revision_id=None)
if u"datasheet_id" in request.POST :
return get_tag_table(request=request, ds_id=request.POST["datasheet_id"], valid=0)
else :
return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], sort=request.POST["sort"], searched=request.POST["searched"])
@login_required
def validate_datasheet(request=None, ds_id=None, valid=None):
# We set if valid is true of false, function of the url parameters
if valid == "1" or valid == "true" or not valid :
valid = True
else :
valid = False
# We validate or unvalidate the requester datasheet
if request.user.is_authenticated():
user = request.user
else:
user = None
ds = Datasheet.objects.get(hda_id=ds_id)
if valid :
ds.validate(user)
else :
ds.unvalidate()
ds.save()
# If there are still some unvalidated/validated ds for the ds's orga, we display the first one
same_organisation_ds = Datasheet.objects.filter(organisation=ds.organisation).filter(validated=not valid)
if len(same_organisation_ds) > 0 :
return redirect('display_datasheet', ds_id=same_organisation_ds[0].hda_id)
else :
return redirect('home')
@login_required
def update_tag_alias(request):
# 2 cases :
# - ordered tag for one datasheet : POST["datasheet_id"] is not null
# - all tags list : POST["datasheet_id"] is null and POST["num_page"] and POST["nb_by_page"] are not null
tag_id = request.POST["id"]
tag_alias = request.POST["value"]
tag = Tag.objects.get(id=tag_id)
tag.alias = tag_alias
tag.save()
if u"datasheet_id" in request.POST :
return get_tag_table(request=request, ds_id=request.POST["datasheet_id"], valid=0)
else :
return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], sort=request.POST["sort"], searched=request.POST["searched"])
def get_categories():
# List of categories in an OrderedDict
categories = OrderedDict({"":""})
for c in TagCategory.objects.order_by('label') :
categories.update({c.label:c.label})
return categories
@login_required
def update_tag_category(request):
tag_id = request.POST["id"]
cat = request.POST["value"]
tag = Tag.objects.get(id=tag_id)
if cat and cat != "" :
tag.category = TagCategory.objects.get(label=cat)
else :
tag.category = None
tag.save()
if u"datasheet_id" in request.POST :
return get_tag_table(request=request, ds_id=request.POST["datasheet_id"], valid=0)
else :
return get_all_tags_table(request=request, num_page=request.POST["num_page"], nb_by_page=request.POST["nb_by_page"], sort=request.POST["sort"], searched=request.POST["searched"])
@login_required
def reorder_tag_datasheet(request):
ds_id = request.REQUEST['datasheet_id']
ds = Datasheet.objects.get(hda_id=ds_id)
reorder_datasheet_tags(ds)
return get_tag_table(request=request, ds_id=ds_id, valid=0)