# -*- coding: utf-8 -*-
'''
Created on Jan 31, 2012
@author: ymh
'''
from django.conf import settings
from django.core.cache import cache
from django.db.models import Q, Count
from django.http import HttpResponse
from hdabo.models import Tag, Datasheet, TaggedSheet
from hdalab.models import (TagLinks, HdaSession, Country, TagYears,
DatasheetExtras)
from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields
import django.utils.simplejson as json
import hmac
import itertools
import uuid
from hdalab.utils import fix_cache_key
def taginfo(request):
label = request.GET.get('label', None)
resobj = {'requested_label' : label}
resobj["content_count"] = Datasheet.objects.filter(taggedsheet__tag__label__iexact = label).distinct().count()
res = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = label).order_by('-dbpedia_uri')[0:1]
if len(res) == 1:
restag = res.get()
resobj["dbpedia_uri"] = restag.dbpedia_uri
if resobj["dbpedia_uri"] is not None and restag.dbpedia_fields is not None:
dbfield = restag.dbpedia_fields
resobj["abstract"] = dbfield.abstract
resobj["dbpedia_label"] = dbfield.label
resobj["thumbnail"] = dbfield.thumbnail
transqs = DbpediaFieldsTranslation.objects.filter(master=dbfield, language_code=request.LANGUAGE_CODE)[0:1]
if transqs:
trans = transqs.get()
resobj['translated_abstract'] = trans.abstract
resobj['translated_label'] = trans.label
else:
resobj['translated_abstract'] = dbfield.abstract
resobj['translated_label'] = dbfield.label
#res = Tag.objects.filter(label__iexact = label).order_by('-wikipedia_url')[0:1]
#if len(res) == 1:
# resobj["wikipedia_url"] = res.get().wikipedia_url
if 'translated_label' in resobj:
wikipedia_label = resobj['translated_label']
else:
wikipedia_label = label
wikipedia_label = wikipedia_label[0].capitalize() + wikipedia_label[1:]
resobj["wikipedia_url"] = "http://%s.wikipedia.org/wiki/%s" % (request.LANGUAGE_CODE,wikipedia_label.replace(' ', '_'))
# We don't use links at the moment, so I'll comment this line to speed up requests
# resobj["links"] = [{'subject':tl.subject.label, 'object':tl.object.label} for tl in TagLinks.objects.select_related().filter(Q(subject__label__iexact = label) | Q(object__label__iexact = label))]
return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
def tagtranslation(request):
labels = request.GET.get('labels',None)
if not labels:
return HttpResponse(content=json.dumps({}), mimetype='application/json')
labelslist = [lbl.strip() for lbl in labels.split(",")]
masters = []
for lbl in labelslist:
labelqs = Tag.objects.select_related('dbpedia_fields').get(label__iexact = lbl)[0:1]
if len(labelqs) > 0:
tag = labelqs.get()
if tag.dbpedia_fields:
masters.append(tag.dbpedia_fields)
translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)
translations = dict([(t.master.label, t.label) for t in translationqs])
return HttpResponse(content=json.dumps(translations), mimetype='application/json')
def sessioninfo(request):
data = json.loads(request.GET.get('data', "{}"))
write = False
if 'sessionid' in request.GET:
request.session['sessionid'] = request.GET['sessionid']
if 'sessionkey' in request.GET:
request.session['sessionkey'] = request.GET['sessionkey']
if 'sessionid' in request.session:
sessionid = request.session['sessionid']
if HdaSession.objects.filter(sessionid=sessionid).count() == 1:
sessionkey = request.session.get('sessionkey',None)
hm = hmac.new(settings.SECRET_KEY, sessionid)
if hm.hexdigest() == sessionkey:
write = True
else:
del request.session['sessionid']
if 'sessionid' not in request.session:
sessionid = unicode(uuid.uuid1())
HdaSession.objects.create(sessionid=sessionid, data=json.dumps({}))
write = True
request.session['sessionid'] = sessionid
request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest()
if write and data:
HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data))
else:
data = HdaSession.objects.get(sessionid=sessionid).data
data = json.loads(data) if data else {}
resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid }
if write:
resobj['sessionkey'] = request.session['sessionkey']
return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
def tagsearch(request):
q = request.GET.get('term',None)
if q:
lq = q.lower()
qs = Tag.objects.filter( Q(label__icontains = q ) | Q(dbpedia_fields__translations__label__icontains = q, dbpedia_fields__translations__language_code=request.LANGUAGE_CODE)) if q else Tag.objects.all()
qs = qs.annotate(nb=Count('datasheet')).order_by('-nb')[:20]
qslist = list(qs)
transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=request.LANGUAGE_CODE).select_related("master")
translations = dict([(tr.master.tag_id, tr.label) for tr in transqs])
res = []
for t in qslist:
resobj = {'value':t.label,'nb':t.nb}
if t.id in translations:
resobj['label'] = translations[t.id]
else:
resobj['label'] = t.label
if q is None or resobj['label'].lower().find(lq) != -1:
res.append(resobj)
return HttpResponse(content=json.dumps(res), mimetype='application/json')
def filter(request):
periode = request.GET.get('period',None)
label = request.GET.get('label', None)
country = request.GET.get('country', None)
contentlist = request.GET.get('contentlist', None)
max_tag_order = request.GET.get('mto', 12)
content_count = request.GET.get('contentcount', 12)
tag_count = request.GET.get('tagcount', 30)
cache_key = fix_cache_key("-".join(("filter",request.LANGUAGE_CODE,str(periode),str(label),str(country),str(contentlist),str(max_tag_order),str(content_count),str(tag_count))))
outputstr = cache.get(cache_key)
if outputstr is None:
matchtagids = []
tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique'])
countryqs = Country.objects
discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields')
yearqs = TagYears.objects
contentqs = Datasheet.objects.filter(validated=True)
labeltranslations = []
if label or periode or country or contentlist :
if periode:
years = periode.split(",")
start_year = int(years[0])
end_year = int(years[0:2][-1])
delta = max(1, (end_year-start_year)/2)
minstart = start_year - delta
maxend = end_year + delta
matchtagqs = Tag.objects.filter(years__end_year__gte = start_year,
years__start_year__lte = end_year,
years__end_year__lte = maxend,
years__start_year__gte = minstart)
matchtagids += [t.id for t in matchtagqs]
contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
taggedsheet__order__lte = max_tag_order)
if label:
masters = []
for txtlbl in label.split(","):
matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip())
for t in matchtagqs:
if t.id not in matchtagids:
matchtagids.append(t.id)
if t.dbpedia_fields:
masters.append(t.dbpedia_fields)
contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
taggedsheet__order__lte = max_tag_order)
translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)
labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs]
if country:
for country_uri in country.split(","):
matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri)
matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids]
contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
taggedsheet__order__lte = max_tag_order)
if contentlist:
contentqs = contentqs.filter(id__in = contentlist.split(","))
tagqs = tagqs.filter(datasheet__in = contentqs)
countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs)
discqs = discqs.filter(datasheet__in = contentqs)
yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs)
if contentlist is None:
contentqs.order_by('?')
cont_count = contentqs.count()
contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]])
contentids = contenus.keys()
qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids)
for dse in qs:
contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude}
qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order'))
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE)
translations = dict([(trans.master_id,trans.label) for trans in transqs])
for ts in qs:
match_tag = ts.tag.id in matchtagids
contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label})
if match_tag:
contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order
if contentlist is None:
contenus = sorted(contenus.values(),key=lambda e: -e['score'])
else:
contenus = contenus.values()
#tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count]
tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count]
#.select_related('dbpedia_fields')
# hack to add only necessary fields in the group by
# contournement bug https://code.djangoproject.com/ticket/17144
tagqs.query.clear_select_fields()
tagqs.query.add_fields(['id','label'], False)
tagqs.query.set_group_by()
tagqslist = list(tagqs)
dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)])
transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE)
translations = dict([(trans.master_id,trans.label) for trans in transqs])
tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist]
countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet'))
countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs])
discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10])
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE)
translations = dict([(trans.master_id,trans.label) for trans in transqs])
disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist]
years = {}
yearqs = yearqs.annotate(nb=Count('tag__taggedsheet'))
for ty in yearqs:
for year in range(ty.start_year, ty.end_year):
years[year] = ty.nb + (years[year] if year in years else 0)
yearchange = []
for year in sorted(years.keys()):
score = years[year]
if year < 2011:
if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score):
yearchange.append({'year': year, 'score': score})
if year+1 not in years and year != -1 and score != 0:
yearchange.append({'year': year+1, 'score': 0})
tag_translations = {}
for t in itertools.chain(labeltranslations,disciplines,tags):
tag_translations[t['label']] = t['translated_label']
for c in contenus:
for t in c['tags']:
tag_translations[t['label']] = t['translated_label']
output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations}
outputstr = json.dumps(output)
cache.set(cache_key, outputstr)
return HttpResponse(content=outputstr, mimetype='application/json')