diff -r 0a082ab236ec -r 8527c5a3ddb7 web/hdalab/views/ajax.py --- a/web/hdalab/views/ajax.py Tue Mar 13 11:33:47 2012 +0100 +++ b/web/hdalab/views/ajax.py Tue Mar 13 13:28:26 2012 +0100 @@ -5,6 +5,7 @@ @author: ymh ''' from django.conf import settings +from django.core.cache import cache from django.db.models import Q, Count from django.http import HttpResponse from hdabo.models import Tag, Datasheet, TaggedSheet @@ -15,6 +16,7 @@ import hmac import itertools import uuid +from hdalab.utils import fix_cache_key def taginfo(request): label = request.GET.get('label', None) @@ -151,150 +153,158 @@ def filter(request): - periode = request.GET.get('period',None) - label = request.GET.get('label', None) - country = request.GET.get('country', None) - contentlist = request.GET.get('contentlist', None) - max_tag_order = request.GET.get('mto', 12) - content_count = request.GET.get('contentcount', 12) - tag_count = request.GET.get('tagcount', 30) - matchtagids = [] + cache_key = fix_cache_key(request.get_full_path()) + + outputstr = cache.get(cache_key) - tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']) - countryqs = Country.objects - discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields') - yearqs = TagYears.objects - - contentqs = Datasheet.objects.filter(validated=True) - labeltranslations = [] - - if label or periode or country or contentlist : + if outputstr is None: + periode = request.GET.get('period',None) + label = request.GET.get('label', None) + country = request.GET.get('country', None) + contentlist = request.GET.get('contentlist', None) + max_tag_order = request.GET.get('mto', 12) + content_count = request.GET.get('contentcount', 12) + tag_count = request.GET.get('tagcount', 30) + + matchtagids = [] - if periode: - years = periode.split(",") - start_year = int(years[0]) - end_year = int(years[0:2][-1]) - delta = max(1, (end_year-start_year)/2) - minstart = start_year - delta - maxend = end_year + delta - matchtagqs = Tag.objects.filter(years__end_year__gte = start_year, - years__start_year__lte = end_year, - years__end_year__lte = maxend, - years__start_year__gte = minstart) - matchtagids += [t.id for t in matchtagqs] - contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, - taggedsheet__order__lte = max_tag_order) + tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']) + countryqs = Country.objects + discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields') + yearqs = TagYears.objects + + contentqs = Datasheet.objects.filter(validated=True) + labeltranslations = [] + + if label or periode or country or contentlist : - if label: - masters = [] - for txtlbl in label.split(","): - matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip()) - for t in matchtagqs: - if t.id not in matchtagids: - matchtagids.append(t.id) - if t.dbpedia_fields: - masters.append(t.dbpedia_fields) - - contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, - taggedsheet__order__lte = max_tag_order) - translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE) - labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] - - if country: - for country_uri in country.split(","): - matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri) - matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids] + if periode: + years = periode.split(",") + start_year = int(years[0]) + end_year = int(years[0:2][-1]) + delta = max(1, (end_year-start_year)/2) + minstart = start_year - delta + maxend = end_year + delta + matchtagqs = Tag.objects.filter(years__end_year__gte = start_year, + years__start_year__lte = end_year, + years__end_year__lte = maxend, + years__start_year__gte = minstart) + matchtagids += [t.id for t in matchtagqs] contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, taggedsheet__order__lte = max_tag_order) - if contentlist: - contentqs = contentqs.filter(id__in = contentlist.split(",")) - - tagqs = tagqs.filter(datasheet__in = contentqs) - countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) - discqs = discqs.filter(datasheet__in = contentqs) - yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) - - if contentlist is None: - contentqs.order_by('?') + + if label: + masters = [] + for txtlbl in label.split(","): + matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip()) + for t in matchtagqs: + if t.id not in matchtagids: + matchtagids.append(t.id) + if t.dbpedia_fields: + masters.append(t.dbpedia_fields) + + contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, + taggedsheet__order__lte = max_tag_order) + translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE) + labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] + + if country: + for country_uri in country.split(","): + matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri) + matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids] + contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, + taggedsheet__order__lte = max_tag_order) + if contentlist: + contentqs = contentqs.filter(id__in = contentlist.split(",")) + + tagqs = tagqs.filter(datasheet__in = contentqs) + countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) + discqs = discqs.filter(datasheet__in = contentqs) + yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) - cont_count = contentqs.count() - - contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) - contentids = contenus.keys() - - qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) - for dse in qs: - contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} - - qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) - - transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE) - translations = dict([(trans.master_id,trans.label) for trans in transqs]) - - for ts in qs: - match_tag = ts.tag.id in matchtagids - contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label}) + if contentlist is None: + contentqs.order_by('?') + + cont_count = contentqs.count() + + contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) + contentids = contenus.keys() - if match_tag: - contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order + qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) + for dse in qs: + contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} + + qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) + + transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE) + translations = dict([(trans.master_id,trans.label) for trans in transqs]) - if contentlist is None: - contenus = sorted(contenus.values(),key=lambda e: -e['score']) - else: - contenus = contenus.values() - - #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] - tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] - #.select_related('dbpedia_fields') - # hack to add only necessary fields in the group by - # contournement bug https://code.djangoproject.com/ticket/17144 - tagqs.query.clear_select_fields() - tagqs.query.add_fields(['id','label'], False) - tagqs.query.set_group_by() - - tagqslist = list(tagqs) + for ts in qs: + match_tag = ts.tag.id in matchtagids + contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label}) + + if match_tag: + contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order + + if contentlist is None: + contenus = sorted(contenus.values(),key=lambda e: -e['score']) + else: + contenus = contenus.values() - dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) - - transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE) - translations = dict([(trans.master_id,trans.label) for trans in transqs]) - - tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist] - - countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) - countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) - - discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) + #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] + tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] + #.select_related('dbpedia_fields') + # hack to add only necessary fields in the group by + # contournement bug https://code.djangoproject.com/ticket/17144 + tagqs.query.clear_select_fields() + tagqs.query.add_fields(['id','label'], False) + tagqs.query.set_group_by() + + tagqslist = list(tagqs) - transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE) - translations = dict([(trans.master_id,trans.label) for trans in transqs]) - + dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) - disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] + transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE) + translations = dict([(trans.master_id,trans.label) for trans in transqs]) - years = {} - yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) - for ty in yearqs: - for year in range(ty.start_year, ty.end_year): - years[year] = ty.nb + (years[year] if year in years else 0) + tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist] + + countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) + countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) + + discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) - yearchange = [] - for year in sorted(years.keys()): - score = years[year] - if year < 2011: - if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score): - yearchange.append({'year': year, 'score': score}) - if year+1 not in years and year != -1 and score != 0: - yearchange.append({'year': year+1, 'score': 0}) - - tag_translations = {} - for t in itertools.chain(labeltranslations,disciplines,tags): - tag_translations[t['label']] = t['translated_label'] - for c in contenus: - for t in c['tags']: + transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE) + translations = dict([(trans.master_id,trans.label) for trans in transqs]) + + + disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] + + years = {} + yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) + for ty in yearqs: + for year in range(ty.start_year, ty.end_year): + years[year] = ty.nb + (years[year] if year in years else 0) + + yearchange = [] + for year in sorted(years.keys()): + score = years[year] + if year < 2011: + if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score): + yearchange.append({'year': year, 'score': score}) + if year+1 not in years and year != -1 and score != 0: + yearchange.append({'year': year+1, 'score': 0}) + + tag_translations = {} + for t in itertools.chain(labeltranslations,disciplines,tags): tag_translations[t['label']] = t['translated_label'] - - output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} - - return HttpResponse(content=json.dumps(output), mimetype='application/json') + for c in contenus: + for t in c['tags']: + tag_translations[t['label']] = t['translated_label'] + + output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} + outputstr = json.dumps(output) + cache.set(cache_key, outputstr) + + return HttpResponse(content=outputstr, mimetype='application/json')