diff -r 2695537badbe -r f45d7494332e src/hdalab/views/ajax.py --- a/src/hdalab/views/ajax.py Fri Apr 03 18:41:06 2015 +0200 +++ b/src/hdalab/views/ajax.py Mon Apr 27 17:29:51 2015 +0200 @@ -23,27 +23,27 @@ logger = logging.getLogger(__name__) def tagtranslation(request): - + lang = request.GET.get('lang',request.LANGUAGE_CODE) labels = request.GET.get('labels',None) if not labels: return HttpResponse(content=json.dumps({}), content_type='application/json') - + labelslist = [lbl.strip() for lbl in labels.split(",")] masters = [] - + for lbl in labelslist: labelqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = lbl)[0:1] if len(labelqs) > 0: tag = labelqs.get() if tag.dbpedia_fields: masters.append(tag.dbpedia_fields) - + translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang) - + translations = dict([(t.master.label, t.label) for t in translationqs]) - + return HttpResponse(content=json.dumps(translations), content_type='application/json') def subcat(category, globtags, level, max_level ): @@ -63,7 +63,7 @@ tag_in_list = {'label' : label, 'contents': []} resobj['themes'].append(tag_in_list) globtags[label] = {'level': (level + 1), 'access': tag_in_list } - + if level < max_level: subcats = WpCategory.objects.filter(parent_categories__parent_category = category) resobj['themes'] += [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))] @@ -98,7 +98,7 @@ master_category = WpCategory.objects.filter(label__iexact=label)[0:1] if len(master_category): resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL ) - + #datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct() datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__iexact = label, taggedsheet__order__lte = ROOT_MAX_TAG_ORDER).select_related('organisation').distinct() for datasheet in datasheets: @@ -143,52 +143,52 @@ 'contents': [] } - + return HttpResponse(content=json.dumps(resobj), content_type='application/json') def sessioninfo(request): - + data = json.loads(request.GET.get('data', "{}")) write = False - + if 'sessionid' in request.GET: request.session['sessionid'] = request.GET['sessionid'] if 'sessionkey' in request.GET: request.session['sessionkey'] = request.GET['sessionkey'] - + if 'sessionid' in request.session: sessionid = request.session['sessionid'] - + if HdaSession.objects.filter(sessionid=sessionid).count() == 1: sessionkey = request.session.get('sessionkey',None) hm = hmac.new(settings.SECRET_KEY, sessionid) if hm.hexdigest() == sessionkey: - write = True + write = True else: del request.session['sessionid'] - + if 'sessionid' not in request.session: sessionid = unicode(uuid.uuid1()) HdaSession.objects.create(sessionid=sessionid, data=json.dumps({})) write = True request.session['sessionid'] = sessionid request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest() - + if write and data: HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data)) else: data = HdaSession.objects.get(sessionid=sessionid).data data = json.loads(data) if data else {} - + resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid } if write: resobj['sessionkey'] = request.session['sessionkey'] - + return HttpResponse(content=json.dumps(resobj), content_type='application/json') def tagsearch(request): - + q = request.GET.get('term',None) maxcount = int(request.GET.get('count','40')) lang = request.GET.get('lang',request.LANGUAGE_CODE) @@ -196,11 +196,11 @@ count_notices_bool = True if count_notices_str: count_notices_bool = {'true': True, 'false': False, "0": False, "1": True}.get(count_notices_str.lower()) - + stemming_langs = [ 'fr', 'en', 'de', 'it' ] # For Japanese, there are no word boundaries, we should not use the regexp in that case no_translate_langs = [ 'fr' ] - + if q: lq = q.lower() qs = Tag.objects.select_related('dbpedia_fields').filter(datasheet__validated=True) @@ -217,22 +217,22 @@ qs = qs.filter(dbpedia_fields__translations__label__icontains=q, dbpedia_fields__translations__language_code=lang, dbpedia_fields__translations__is_label_translated = True) else: qs = Tag.objects.filter(~Q(dbpedia_uri = None)) - + if count_notices_bool: qs = qs.annotate(nb=Count('datasheet',distinct=True)).order_by('-nb')[:maxcount] else: qs = qs.distinct()[:maxcount] - + qslist = list(qs) - + if lang in no_translate_langs: translations = {} else: transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=lang, is_label_translated=True).select_related("master") translations = dict([(tr.master.tag_id, {'label':tr.label,'abstract':tr.abstract, 'is_label_translated': tr.is_label_translated}) for tr in transqs]) - + res = [] - + for t in qslist: if hasattr(t, 'dbpedia_fields'): dbfields = t.dbpedia_fields @@ -240,7 +240,7 @@ if count_notices_bool: resobj['nb'] = t.nb resobj['thumbnail'] = dbfields.thumbnail if dbfields is not None else None - + if t.id in translations: resobj['value'] = translations[t.id]['label'] resobj['abstract'] = translations[t.id]['abstract'] @@ -249,27 +249,27 @@ resobj['abstract'] = dbfields.abstract if dbfields is not None else None if q is None or resobj['value'].lower().find(lq) != -1: res.append(resobj) - + return HttpResponse(content=json.dumps(res), content_type='application/json') def catsearch(request): - + q = request.GET.get('term',None) - + # On ne récupère que les catégories qui sont également des tags qrx = '(\\m|\\b)%s'%q qs = Tag.objects.filter(label__iregex=qrx) - + labels = [tag.label for tag in qs] - + qs = WpCategory.objects.annotate(nb=Count('child_categories__child_category__tags')).filter(label__in = labels, nb__gt=0) - + res = [{'value':t.label} for t in qs] - + return HttpResponse(content=json.dumps(res), content_type='application/json') def filter(request): - + lang = request.GET.get('lang',request.LANGUAGE_CODE) periode = request.GET.get('period',None) label = request.GET.get('label', None) @@ -278,38 +278,38 @@ max_tag_order = int(request.GET.get('mto', '12')) content_count = request.GET.get('contentcount', 8) tag_count = request.GET.get('tagcount', 30) - + outputstr = filter_generic(lang, periode, label, country, contentlist, max_tag_order, content_count, tag_count) - + return HttpResponse(content=outputstr, content_type='application/json') def filter_generic(lang="fr-fr", periode=None, label=None, country=None, contentlist=None, max_tag_order=12, content_count=8, tag_count=30): - + no_translate_langs = [ 'fr' ] - + key_parts = ("filter",lang,periode,label,country,contentlist,max_tag_order,content_count,tag_count) key_parts = [unicode(p).encode("ascii", "ignore") for p in key_parts] - + cache_key = fix_cache_key("-".join(key_parts)) - + outputstr = cache.get(cache_key) - + if outputstr is None: - + matchtagids = [] - + tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']).filter(~Q(dbpedia_uri = None)) countryqs = Country.objects discqs = Tag.objects.filter(~Q(dbpedia_uri = None), category__label = u'Discipline artistique').select_related('dbpedia_fields') yearqs = TagYears.objects - + contentqs = Datasheet.objects.filter(validated=True) labeltranslations = [] - + if label or periode or country or contentlist : matchtagqslist = [] - + if periode: years = periode.split(",") start_year = int(years[0]) @@ -318,18 +318,18 @@ minstart = start_year - delta maxend = end_year + delta matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None), - years__end_year__gte = start_year, + years__end_year__gte = start_year, years__start_year__lte = end_year, years__end_year__lte = maxend, years__start_year__gte = minstart, ) matchtagqslist.append(matchtagqs) - + if label: for txtlbl in label.split(","): matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = txtlbl.strip()) matchtagqslist.append(matchtagqs) - + if country: for country_uri in country.split(","): matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),locatedin__country__dbpedia_uri = country_uri) @@ -345,40 +345,40 @@ newcondid = Q(id__in = matchtagqs) tagcond = newcond if tagcond is None else (tagcond & newcond) tagcondid = newcondid if tagcondid is None else (tagcondid | newcondid) - + contentqs = contentqs.filter(tagcond).distinct() matchtagidsqs = list(Tag.objects.select_related("dbpedia_fields").only("id").filter(tagcondid)) matchtagids = [t.id for t in matchtagidsqs] - + if lang not in no_translate_langs: masters = [t.dbpedia_fields for t in matchtagidsqs if t.dbpedia_fields is not None] - - translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang) + + translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang) labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] - + tagqs = tagqs.filter(datasheet__in = contentqs) countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) discqs = discqs.filter(datasheet__in = contentqs) yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) - + if contentlist is None: contentqs.order_by('?') - + cont_count = contentqs.count() - + logger.debug("ajax filter SQL for contentqs %s", contentqs.query) - + contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'hda_id': content.hda_id, 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) contentids = contenus.keys() - + qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) for dse in qs: contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} - + qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) - + translations = {} - + if lang not in no_translate_langs: ts_list = [] for ts in qs: @@ -386,69 +386,69 @@ ts_list.append(ts.tag.dbpedia_fields) transqs = DbpediaFieldsTranslation.objects.filter(master__in = ts_list, language_code = lang) translations = dict([(trans.master_id,trans.label) for trans in transqs]) - + for ts in qs: if hasattr(ts, 'tag') and hasattr(ts.tag, 'dbpedia_fields') : match_tag = ts.tag.id in matchtagids - contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, - 'label': ts.tag.label, - 'order': ts.order, - 'match': match_tag, + contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, + 'label': ts.tag.label, + 'order': ts.order, + 'match': match_tag, 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label, 'url': ts.tag.dbpedia_uri, 'wkpd_url': ts.tag.wikipedia_url}) - + if match_tag: contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order - + if contentlist is None: contenus = sorted(contenus.values(),key=lambda e: -e['score']) else: contenus = contenus.values() - + #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] - tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] + tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label', 'dbpedia_uri', 'wikipedia_url')[:tag_count] #.select_related('dbpedia_fields') # hack to add only necessary fields in the group by # contournement bug https://code.djangoproject.com/ticket/17144 - tagqs.query.clear_select_fields() - tagqs.query.add_fields(['id','label'], False) - tagqs.query.set_group_by() - + #tagqs.query.clear_select_fields() + #tagqs.query.add_fields(['id','label'], False) + #tagqs.query.set_group_by() + tagqslist = list(tagqs) - + dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) if lang not in no_translate_langs: transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = lang) translations = dict([(trans.master_id,trans.label) for trans in transqs]) - - tags = [{'id': tag.id, + + tags = [{'id': tag.id, 'label': tag.label, - 'score': tag.nb, + 'score': tag.nb, 'thumbnail': dbpediafields[tag.id].thumbnail if tag.id in dbpediafields else None, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label, 'url': tag.dbpedia_uri, 'wkpd_url': tag.wikipedia_url} for tag in tagqslist] - + countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) - + discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) if lang not in no_translate_langs: list_dbpediafields = [tag.dbpedia_fields for tag in discqslist if tag.dbpedia_fields is not None] transqs = DbpediaFieldsTranslation.objects.filter(master__in = list_dbpediafields, language_code = lang) translations = dict([(trans.master_id,trans.label) for trans in transqs]) - + disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] - + years = {} yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) for ty in yearqs: for year in range(ty.start_year, ty.end_year): years[year] = ty.nb + (years[year] if year in years else 0) - + yearchange = [] for year in sorted(years.keys()): score = years[year] @@ -457,51 +457,51 @@ yearchange.append({'year': year, 'score': score}) if year+1 not in years and year != -1 and score != 0: yearchange.append({'year': year+1, 'score': 0}) - + tag_translations = {} for t in itertools.chain(labeltranslations,disciplines,tags): tag_translations[t['label']] = t['translated_label'] for c in contenus: for t in c['tags']: tag_translations[t['label']] = t['translated_label'] - + output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} outputstr = json.dumps(output) cache.set(cache_key, outputstr) - + return outputstr - + def subtree(tree): MAX_TAG_ORDER = 16 label = tree['label'] sub = tree.get('contents',[]) - + datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__iexact = label, taggedsheet__order__lte = MAX_TAG_ORDER).annotate(tagorder=Min('taggedsheet__order')).select_related('organisation').distinct() - + contents = [{ 'description': ds.description, 'title': ds.title, 'url': ds.url, 'score': int((MAX_TAG_ORDER - ds.tagorder)/2), 'id': ds.id, 'hda_id': ds.hda_id, 'organization': ds.organisation.name, 'organization_url': ds.organisation.website } for ds in datasheets] - + contents = sorted(contents, key=lambda e: -e['score']) - + res = { 'label': label } - + if len(contents): res['contents'] = contents - + if len(sub): subcats = [subtree(st) for st in sub] subcats = [sc for sc in subcats if len(sc.get('contents',[])) or len(sc.get('themes',[]))] res['themes'] = subcats - + return res def filltree(request): - + tree = request.GET.get('tree','{}') - + treeobj = json.loads(tree) - + res = subtree(treeobj) - + return HttpResponse(content=json.dumps(res), content_type='application/json')