--- a/web/hdalab/views/ajax.py Tue Mar 13 11:33:47 2012 +0100
+++ b/web/hdalab/views/ajax.py Tue Mar 13 13:28:26 2012 +0100
@@ -5,6 +5,7 @@
@author: ymh
'''
from django.conf import settings
+from django.core.cache import cache
from django.db.models import Q, Count
from django.http import HttpResponse
from hdabo.models import Tag, Datasheet, TaggedSheet
@@ -15,6 +16,7 @@
import hmac
import itertools
import uuid
+from hdalab.utils import fix_cache_key
def taginfo(request):
label = request.GET.get('label', None)
@@ -151,150 +153,158 @@
def filter(request):
- periode = request.GET.get('period',None)
- label = request.GET.get('label', None)
- country = request.GET.get('country', None)
- contentlist = request.GET.get('contentlist', None)
- max_tag_order = request.GET.get('mto', 12)
- content_count = request.GET.get('contentcount', 12)
- tag_count = request.GET.get('tagcount', 30)
- matchtagids = []
+ cache_key = fix_cache_key(request.get_full_path())
+
+ outputstr = cache.get(cache_key)
- tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique'])
- countryqs = Country.objects
- discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields')
- yearqs = TagYears.objects
-
- contentqs = Datasheet.objects.filter(validated=True)
- labeltranslations = []
-
- if label or periode or country or contentlist :
+ if outputstr is None:
+ periode = request.GET.get('period',None)
+ label = request.GET.get('label', None)
+ country = request.GET.get('country', None)
+ contentlist = request.GET.get('contentlist', None)
+ max_tag_order = request.GET.get('mto', 12)
+ content_count = request.GET.get('contentcount', 12)
+ tag_count = request.GET.get('tagcount', 30)
+
+ matchtagids = []
- if periode:
- years = periode.split(",")
- start_year = int(years[0])
- end_year = int(years[0:2][-1])
- delta = max(1, (end_year-start_year)/2)
- minstart = start_year - delta
- maxend = end_year + delta
- matchtagqs = Tag.objects.filter(years__end_year__gte = start_year,
- years__start_year__lte = end_year,
- years__end_year__lte = maxend,
- years__start_year__gte = minstart)
- matchtagids += [t.id for t in matchtagqs]
- contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
- taggedsheet__order__lte = max_tag_order)
+ tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique'])
+ countryqs = Country.objects
+ discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields')
+ yearqs = TagYears.objects
+
+ contentqs = Datasheet.objects.filter(validated=True)
+ labeltranslations = []
+
+ if label or periode or country or contentlist :
- if label:
- masters = []
- for txtlbl in label.split(","):
- matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip())
- for t in matchtagqs:
- if t.id not in matchtagids:
- matchtagids.append(t.id)
- if t.dbpedia_fields:
- masters.append(t.dbpedia_fields)
-
- contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
- taggedsheet__order__lte = max_tag_order)
- translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)
- labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs]
-
- if country:
- for country_uri in country.split(","):
- matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri)
- matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids]
+ if periode:
+ years = periode.split(",")
+ start_year = int(years[0])
+ end_year = int(years[0:2][-1])
+ delta = max(1, (end_year-start_year)/2)
+ minstart = start_year - delta
+ maxend = end_year + delta
+ matchtagqs = Tag.objects.filter(years__end_year__gte = start_year,
+ years__start_year__lte = end_year,
+ years__end_year__lte = maxend,
+ years__start_year__gte = minstart)
+ matchtagids += [t.id for t in matchtagqs]
contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
taggedsheet__order__lte = max_tag_order)
- if contentlist:
- contentqs = contentqs.filter(id__in = contentlist.split(","))
-
- tagqs = tagqs.filter(datasheet__in = contentqs)
- countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs)
- discqs = discqs.filter(datasheet__in = contentqs)
- yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs)
-
- if contentlist is None:
- contentqs.order_by('?')
+
+ if label:
+ masters = []
+ for txtlbl in label.split(","):
+ matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip())
+ for t in matchtagqs:
+ if t.id not in matchtagids:
+ matchtagids.append(t.id)
+ if t.dbpedia_fields:
+ masters.append(t.dbpedia_fields)
+
+ contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
+ taggedsheet__order__lte = max_tag_order)
+ translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)
+ labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs]
+
+ if country:
+ for country_uri in country.split(","):
+ matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri)
+ matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids]
+ contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs,
+ taggedsheet__order__lte = max_tag_order)
+ if contentlist:
+ contentqs = contentqs.filter(id__in = contentlist.split(","))
+
+ tagqs = tagqs.filter(datasheet__in = contentqs)
+ countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs)
+ discqs = discqs.filter(datasheet__in = contentqs)
+ yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs)
- cont_count = contentqs.count()
-
- contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]])
- contentids = contenus.keys()
-
- qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids)
- for dse in qs:
- contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude}
-
- qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order'))
-
- transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE)
- translations = dict([(trans.master_id,trans.label) for trans in transqs])
-
- for ts in qs:
- match_tag = ts.tag.id in matchtagids
- contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label})
+ if contentlist is None:
+ contentqs.order_by('?')
+
+ cont_count = contentqs.count()
+
+ contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]])
+ contentids = contenus.keys()
- if match_tag:
- contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order
+ qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids)
+ for dse in qs:
+ contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude}
+
+ qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order'))
+
+ transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE)
+ translations = dict([(trans.master_id,trans.label) for trans in transqs])
- if contentlist is None:
- contenus = sorted(contenus.values(),key=lambda e: -e['score'])
- else:
- contenus = contenus.values()
-
- #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count]
- tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count]
- #.select_related('dbpedia_fields')
- # hack to add only necessary fields in the group by
- # contournement bug https://code.djangoproject.com/ticket/17144
- tagqs.query.clear_select_fields()
- tagqs.query.add_fields(['id','label'], False)
- tagqs.query.set_group_by()
-
- tagqslist = list(tagqs)
+ for ts in qs:
+ match_tag = ts.tag.id in matchtagids
+ contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label})
+
+ if match_tag:
+ contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order
+
+ if contentlist is None:
+ contenus = sorted(contenus.values(),key=lambda e: -e['score'])
+ else:
+ contenus = contenus.values()
- dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)])
-
- transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE)
- translations = dict([(trans.master_id,trans.label) for trans in transqs])
-
- tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist]
-
- countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet'))
- countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs])
-
- discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10])
+ #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count]
+ tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count]
+ #.select_related('dbpedia_fields')
+ # hack to add only necessary fields in the group by
+ # contournement bug https://code.djangoproject.com/ticket/17144
+ tagqs.query.clear_select_fields()
+ tagqs.query.add_fields(['id','label'], False)
+ tagqs.query.set_group_by()
+
+ tagqslist = list(tagqs)
- transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE)
- translations = dict([(trans.master_id,trans.label) for trans in transqs])
-
+ dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)])
- disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist]
+ transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE)
+ translations = dict([(trans.master_id,trans.label) for trans in transqs])
- years = {}
- yearqs = yearqs.annotate(nb=Count('tag__taggedsheet'))
- for ty in yearqs:
- for year in range(ty.start_year, ty.end_year):
- years[year] = ty.nb + (years[year] if year in years else 0)
+ tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist]
+
+ countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet'))
+ countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs])
+
+ discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10])
- yearchange = []
- for year in sorted(years.keys()):
- score = years[year]
- if year < 2011:
- if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score):
- yearchange.append({'year': year, 'score': score})
- if year+1 not in years and year != -1 and score != 0:
- yearchange.append({'year': year+1, 'score': 0})
-
- tag_translations = {}
- for t in itertools.chain(labeltranslations,disciplines,tags):
- tag_translations[t['label']] = t['translated_label']
- for c in contenus:
- for t in c['tags']:
+ transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE)
+ translations = dict([(trans.master_id,trans.label) for trans in transqs])
+
+
+ disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist]
+
+ years = {}
+ yearqs = yearqs.annotate(nb=Count('tag__taggedsheet'))
+ for ty in yearqs:
+ for year in range(ty.start_year, ty.end_year):
+ years[year] = ty.nb + (years[year] if year in years else 0)
+
+ yearchange = []
+ for year in sorted(years.keys()):
+ score = years[year]
+ if year < 2011:
+ if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score):
+ yearchange.append({'year': year, 'score': score})
+ if year+1 not in years and year != -1 and score != 0:
+ yearchange.append({'year': year+1, 'score': 0})
+
+ tag_translations = {}
+ for t in itertools.chain(labeltranslations,disciplines,tags):
tag_translations[t['label']] = t['translated_label']
-
- output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations}
-
- return HttpResponse(content=json.dumps(output), mimetype='application/json')
+ for c in contenus:
+ for t in c['tags']:
+ tag_translations[t['label']] = t['translated_label']
+
+ output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations}
+ outputstr = json.dumps(output)
+ cache.set(cache_key, outputstr)
+
+ return HttpResponse(content=outputstr, mimetype='application/json')