| author | ymh <ymh.work@gmail.com> |
| Sun, 09 Sep 2012 21:35:10 +0200 | |
| changeset 243 | 1f2840354865 |
| parent 238 | f818b9430585 |
| child 248 | 91bc8521e3cb |
| permissions | -rw-r--r-- |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
1 |
# -*- coding: utf-8 -*- |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
2 |
''' |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
3 |
Created on Jan 31, 2012 |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
4 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
5 |
@author: ymh |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
6 |
''' |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
7 |
from django.conf import settings |
| 154 | 8 |
from django.core.cache import cache |
| 135 | 9 |
from django.db.models import Q, Count |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
10 |
from django.http import HttpResponse |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
11 |
from hdabo.models import Tag, Datasheet, TaggedSheet |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
12 |
from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras |
| 152 | 13 |
from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields |
| 204 | 14 |
from hdalab.models.categories import WpCategory, WpCategoryInclusion, TagWpCategory |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
15 |
from hdalab.utils import fix_cache_key |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
16 |
import copy |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
17 |
import django.utils.simplejson as json |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
18 |
import hmac |
| 135 | 19 |
import itertools |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
20 |
import uuid |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
21 |
|
| 135 | 22 |
def tagtranslation(request): |
23 |
||
| 235 | 24 |
lang = request.GET.get('lang',request.LANGUAGE_CODE) |
| 135 | 25 |
labels = request.GET.get('labels',None) |
26 |
||
27 |
if not labels: |
|
28 |
return HttpResponse(content=json.dumps({}), mimetype='application/json') |
|
29 |
||
30 |
labelslist = [lbl.strip() for lbl in labels.split(",")] |
|
31 |
masters = [] |
|
32 |
||
33 |
for lbl in labelslist: |
|
| 165 | 34 |
labelqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = lbl)[0:1] |
| 135 | 35 |
if len(labelqs) > 0: |
36 |
tag = labelqs.get() |
|
37 |
if tag.dbpedia_fields: |
|
38 |
masters.append(tag.dbpedia_fields) |
|
39 |
||
| 235 | 40 |
translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang) |
| 135 | 41 |
|
42 |
translations = dict([(t.master.label, t.label) for t in translationqs]) |
|
43 |
||
44 |
return HttpResponse(content=json.dumps(translations), mimetype='application/json') |
|
| 204 | 45 |
|
46 |
def subcat(category, globtags, level, max_level ): |
|
47 |
# recursive function used by cattree |
|
| 208 | 48 |
catlabel = category.label |
49 |
tags = Tag.objects.filter(wp_categories__wp_category = category).distinct() |
|
| 204 | 50 |
taglabels = [k for k in dict([(t.label,t.label) for t in tags])] |
51 |
resobj = { |
|
52 |
'category': category.label, |
|
53 |
'tags': [], |
|
54 |
'contents': [] |
|
55 |
} |
|
56 |
for label in taglabels: |
|
57 |
if label == catlabel: |
|
58 |
globtags[label] = {'level': level, 'access': resobj } |
|
59 |
else: |
|
60 |
tag_in_list = {'label' : label, 'contents': []} |
|
61 |
resobj['tags'].append(tag_in_list) |
|
62 |
globtags[label] = {'level': (level + 1), 'access': tag_in_list } |
|
63 |
||
64 |
if level < max_level: |
|
| 208 | 65 |
subcats = WpCategory.objects.filter(parent_categories__parent_category = category) |
| 204 | 66 |
resobj['sub_categories'] = [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))] |
67 |
return resobj |
|
68 |
||
69 |
def cleantags(category): |
|
70 |
if category.has_key('contents') and len(category['contents']) == 0: |
|
71 |
del category['contents'] |
|
| 208 | 72 |
if category.has_key('contents'): |
73 |
category['contents'] = sorted(category['contents'], key=lambda content: -content['score']) |
|
| 204 | 74 |
if category.has_key('tags'): |
| 208 | 75 |
category['tags'] = [cleantags(tag) for tag in category['tags'] if len(tag['contents'])] |
| 204 | 76 |
if len(category['tags']) == 0: |
77 |
del category['tags'] |
|
| 208 | 78 |
else: |
79 |
category['tags'] = sorted(category['tags'], key=lambda tag: tag['label']) |
|
| 204 | 80 |
if category.has_key('sub_categories'): |
81 |
sub_cats = [] |
|
82 |
for sub_cat in category['sub_categories']: |
|
83 |
cat = cleantags(sub_cat) |
|
84 |
if cat.has_key('tags') or cat.has_key('sub_categories') or cat.has_key('contents'): |
|
85 |
sub_cats.append(cat) |
|
| 208 | 86 |
category['sub_categories'] = sorted(sub_cats, key=lambda cat: cat['category']) |
| 204 | 87 |
if len(category['sub_categories']) == 0: |
88 |
del category['sub_categories'] |
|
89 |
return category |
|
90 |
||
91 |
def cattree(request): |
|
92 |
# Gets the category tree from a label |
|
| 232 | 93 |
ROOT_MAX_TAG_ORDER = 8 |
94 |
MAX_TAG_ORDER = 8 |
|
| 204 | 95 |
MAX_LEVEL = 3 |
| 212 | 96 |
LEVEL_COEFF = 5 |
| 204 | 97 |
label = request.GET.get('label', None) |
| 208 | 98 |
lowerlabel = label.lower() |
| 204 | 99 |
globtags = {} |
100 |
resobj = None |
|
101 |
master_category = WpCategory.objects.filter(label__iexact=label)[0:1] |
|
102 |
if len(master_category): |
|
103 |
resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL ) |
|
| 135 | 104 |
|
| 208 | 105 |
# tag_list = [k for k in globtags] |
| 204 | 106 |
|
| 208 | 107 |
# if len(tag_list): |
108 |
contents = [] |
|
109 |
# datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct() |
|
| 232 | 110 |
datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__iexact = label, taggedsheet__order__lte = ROOT_MAX_TAG_ORDER).select_related('organisation').distinct() |
| 208 | 111 |
for datasheet in datasheets: |
112 |
# Calculating where we add the datasheet in the tree |
|
113 |
maintag = None |
|
114 |
maintagscore = -5 |
|
115 |
dsscore = 0 |
|
116 |
rootscore = 0 |
|
117 |
for ts in TaggedSheet.objects.select_related('tag','datasheet').filter(datasheet__id=datasheet.id,order__lte=MAX_TAG_ORDER): |
|
118 |
label = ts.tag.label |
|
119 |
if globtags.has_key(label): |
|
| 212 | 120 |
score = LEVEL_COEFF * globtags[label]['level'] - ts.order |
| 208 | 121 |
if score > maintagscore: |
122 |
maintagscore = score |
|
123 |
maintag = label |
|
124 |
dsscore = (MAX_TAG_ORDER - ts.order) |
|
125 |
if label.lower() == lowerlabel: |
|
126 |
rootscore = (ROOT_MAX_TAG_ORDER - ts.order) |
|
127 |
if maintag is not None: |
|
| 212 | 128 |
globtags[maintag]['access']['contents'].append({ |
129 |
'id': datasheet.id, |
|
130 |
'title': datasheet.title, |
|
131 |
'url': datasheet.url, |
|
132 |
'description': datasheet.description, |
|
133 |
'hda_id': datasheet.hda_id, |
|
| 232 | 134 |
'organization': datasheet.organisation.name, |
135 |
'score': max(dsscore, rootscore) |
|
| 212 | 136 |
}) |
| 208 | 137 |
cleantags(resobj) |
| 204 | 138 |
|
139 |
# resobj['contents'] = [{'id': d.id, 'title': d.title, 'tags': [t.label for t in d.tags.filter(taggedsheet__order__lte=5)]} for d in datasheets] |
|
140 |
||
141 |
return HttpResponse(content=json.dumps(resobj), mimetype='application/json') |
|
| 135 | 142 |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
143 |
def sessioninfo(request): |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
144 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
145 |
data = json.loads(request.GET.get('data', "{}")) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
146 |
write = False |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
147 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
148 |
if 'sessionid' in request.GET: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
149 |
request.session['sessionid'] = request.GET['sessionid'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
150 |
if 'sessionkey' in request.GET: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
151 |
request.session['sessionkey'] = request.GET['sessionkey'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
152 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
153 |
if 'sessionid' in request.session: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
154 |
sessionid = request.session['sessionid'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
155 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
156 |
if HdaSession.objects.filter(sessionid=sessionid).count() == 1: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
157 |
sessionkey = request.session.get('sessionkey',None) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
158 |
hm = hmac.new(settings.SECRET_KEY, sessionid) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
159 |
if hm.hexdigest() == sessionkey: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
160 |
write = True |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
161 |
else: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
162 |
del request.session['sessionid'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
163 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
164 |
if 'sessionid' not in request.session: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
165 |
sessionid = unicode(uuid.uuid1()) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
166 |
HdaSession.objects.create(sessionid=sessionid, data=json.dumps({})) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
167 |
write = True |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
168 |
request.session['sessionid'] = sessionid |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
169 |
request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest() |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
170 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
171 |
if write and data: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
172 |
HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data)) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
173 |
else: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
174 |
data = HdaSession.objects.get(sessionid=sessionid).data |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
175 |
data = json.loads(data) if data else {} |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
176 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
177 |
resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid } |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
178 |
if write: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
179 |
resobj['sessionkey'] = request.session['sessionkey'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
180 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
181 |
return HttpResponse(content=json.dumps(resobj), mimetype='application/json') |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
182 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
183 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
184 |
def tagsearch(request): |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
185 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
186 |
q = request.GET.get('term',None) |
| 238 | 187 |
maxcount = int(request.GET.get('count','40')) |
| 235 | 188 |
lang = request.GET.get('lang',request.LANGUAGE_CODE) |
189 |
||
190 |
stemming_langs = [ 'fr', 'en', 'de', 'it' ] |
|
191 |
# For Japanese, there are no word boundaries, we should not use the regexp in that case |
|
192 |
no_translate_langs = [ 'fr' ] |
|
193 |
||
| 150 | 194 |
if q: |
195 |
lq = q.lower() |
|
| 235 | 196 |
qs = Tag.objects.select_related('dbpedia_fields').filter(datasheet__validated=True) |
197 |
qrx = '(\\m|\\b)%s'%q |
|
198 |
if lang in no_translate_langs: |
|
199 |
if lang in stemming_langs: |
|
200 |
qs = qs.filter( label__iregex = qrx ) |
|
201 |
else: |
|
202 |
qs = qs.filter( label__icontains = q ) |
|
203 |
else: |
|
204 |
if lang in stemming_langs: |
|
|
243
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
205 |
qs = qs.filter(dbpedia_fields__translations__label__iregex=qrx, dbpedia_fields__translations__language_code=lang, dbpedia_fields__translations__is_label_translated = True) |
| 235 | 206 |
else: |
|
243
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
207 |
qs = qs.filter(dbpedia_fields__translations__label__icontains=q, dbpedia_fields__translations__language_code=lang, dbpedia_fields__translations__is_label_translated = True) |
| 235 | 208 |
else: |
209 |
qs = Tag.objects.filter(~Q(dbpedia_uri = None)) |
|
210 |
||
| 238 | 211 |
qs = qs.annotate(nb=Count('datasheet',distinct=True)).order_by('-nb')[:maxcount] |
| 152 | 212 |
|
213 |
qslist = list(qs) |
|
214 |
||
| 235 | 215 |
if lang in no_translate_langs: |
216 |
translations = {} |
|
217 |
else: |
|
|
243
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
218 |
transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=lang, is_label_translated=True).select_related("master") |
|
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
219 |
translations = dict([(tr.master.tag_id, {'label':tr.label,'abstract':tr.abstract, 'is_label_translated': tr.is_label_translated}) for tr in transqs]) |
| 152 | 220 |
|
| 150 | 221 |
res = [] |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
222 |
|
| 152 | 223 |
for t in qslist: |
| 235 | 224 |
dbfields = t.dbpedia_fields |
225 |
resobj = {'original_label':t.label,'nb':t.nb} |
|
226 |
resobj['thumbnail'] = dbfields.thumbnail if dbfields is not None else None |
|
|
243
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
227 |
|
|
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
228 |
# if t.id in translations and not translations[t.id].get('is_label_translated', True): |
|
1f2840354865
correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents:
238
diff
changeset
|
229 |
# continue |
| 152 | 230 |
if t.id in translations: |
| 235 | 231 |
resobj['value'] = translations[t.id]['label'] |
232 |
resobj['abstract'] = translations[t.id]['abstract'] |
|
| 150 | 233 |
else: |
| 235 | 234 |
resobj['value'] = t.label |
235 |
resobj['abstract'] = dbfields.abstract if dbfields is not None else None |
|
236 |
if q is None or resobj['value'].lower().find(lq) != -1: |
|
| 150 | 237 |
res.append(resobj) |
238 |
||
239 |
return HttpResponse(content=json.dumps(res), mimetype='application/json') |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
240 |
|
| 205 | 241 |
def catsearch(request): |
242 |
||
243 |
q = request.GET.get('term',None) |
|
244 |
||
| 208 | 245 |
# On ne récupère que les catégories qui sont également des tags |
| 235 | 246 |
qrx = '(\\m|\\b)%s'%q |
247 |
qs = Tag.objects.filter(label__iregex=q) |
|
| 208 | 248 |
|
249 |
labels = [tag.label for tag in qs] |
|
250 |
||
| 235 | 251 |
qs = WpCategory.objects.annotate(nb=Count('child_categories__child_category__tags')).filter(label__in = labels, nb__gt=0) |
| 205 | 252 |
|
253 |
res = [{'value':t.label} for t in qs] |
|
254 |
||
255 |
return HttpResponse(content=json.dumps(res), mimetype='application/json') |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
256 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
257 |
def filter(request): |
| 127 | 258 |
|
| 235 | 259 |
lang = request.GET.get('lang',request.LANGUAGE_CODE) |
| 158 | 260 |
periode = request.GET.get('period',None) |
261 |
label = request.GET.get('label', None) |
|
262 |
country = request.GET.get('country', None) |
|
263 |
contentlist = request.GET.get('contentlist', None) |
|
| 238 | 264 |
max_tag_order = int(request.GET.get('mto', '12')) |
| 158 | 265 |
content_count = request.GET.get('contentcount', 12) |
266 |
tag_count = request.GET.get('tagcount', 30) |
|
267 |
||
| 235 | 268 |
key_parts = ("filter",lang,periode,label,country,contentlist,max_tag_order,content_count,tag_count) |
| 163 | 269 |
key_parts = [unicode(p).encode("utf-8") for p in key_parts] |
270 |
||
271 |
cache_key = fix_cache_key("-".join(key_parts)) |
|
| 154 | 272 |
|
273 |
outputstr = cache.get(cache_key) |
|
| 127 | 274 |
|
| 154 | 275 |
if outputstr is None: |
276 |
||
277 |
matchtagids = [] |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
278 |
|
| 165 | 279 |
tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']).filter(~Q(dbpedia_uri = None)) |
| 154 | 280 |
countryqs = Country.objects |
| 165 | 281 |
discqs = Tag.objects.filter(~Q(dbpedia_uri = None), category__label = u'Discipline artistique').select_related('dbpedia_fields') |
| 154 | 282 |
yearqs = TagYears.objects |
283 |
||
284 |
contentqs = Datasheet.objects.filter(validated=True) |
|
285 |
labeltranslations = [] |
|
286 |
||
287 |
if label or periode or country or contentlist : |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
288 |
matchtagqslist = [] |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
289 |
|
| 154 | 290 |
if periode: |
291 |
years = periode.split(",") |
|
292 |
start_year = int(years[0]) |
|
293 |
end_year = int(years[0:2][-1]) |
|
294 |
delta = max(1, (end_year-start_year)/2) |
|
295 |
minstart = start_year - delta |
|
296 |
maxend = end_year + delta |
|
| 165 | 297 |
matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None), |
298 |
years__end_year__gte = start_year, |
|
299 |
years__start_year__lte = end_year, |
|
300 |
years__end_year__lte = maxend, |
|
301 |
years__start_year__gte = minstart, |
|
302 |
) |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
303 |
matchtagqslist.append(matchtagqs) |
| 154 | 304 |
|
305 |
if label: |
|
306 |
for txtlbl in label.split(","): |
|
| 165 | 307 |
matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = txtlbl.strip()) |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
308 |
matchtagqslist.append(matchtagqs) |
| 154 | 309 |
|
310 |
if country: |
|
311 |
for country_uri in country.split(","): |
|
| 165 | 312 |
matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),locatedin__country__dbpedia_uri = country_uri) |
| 154 | 313 |
matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids] |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
314 |
matchtagqslist.append(matchtagqs) |
| 154 | 315 |
if contentlist: |
316 |
contentqs = contentqs.filter(id__in = contentlist.split(",")) |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
317 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
318 |
tagcond = None |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
319 |
tagcondid = None |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
320 |
for matchtagqs in matchtagqslist: |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
321 |
newcond = Q(id__in = TaggedSheet.objects.filter(tag__in = copy.deepcopy(matchtagqs), order__lte = max_tag_order).values('datasheet_id')) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
322 |
newcondid = Q(id__in = matchtagqs) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
323 |
tagcond = newcond if tagcond is None else (tagcond & newcond) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
324 |
tagcondid = newcondid if tagcondid is None else (tagcondid | newcondid) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
325 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
326 |
contentqs = contentqs.filter(tagcond).distinct() |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
327 |
matchtagidsqs = list(Tag.objects.select_related("dbpedia_fields").only("id").filter(tagcondid)) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
328 |
matchtagids = [t.id for t in matchtagidsqs] |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
329 |
masters = [t.dbpedia_fields for t in matchtagidsqs if t.dbpedia_fields is not None] |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
330 |
|
| 235 | 331 |
translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang) |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
332 |
labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
333 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
334 |
|
| 154 | 335 |
tagqs = tagqs.filter(datasheet__in = contentqs) |
336 |
countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) |
|
337 |
discqs = discqs.filter(datasheet__in = contentqs) |
|
338 |
yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) |
|
|
122
fde8335a037c
Added Geographic Inclusion in the Django version (server only)
veltr
parents:
119
diff
changeset
|
339 |
|
| 154 | 340 |
if contentlist is None: |
341 |
contentqs.order_by('?') |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
342 |
|
| 154 | 343 |
cont_count = contentqs.count() |
344 |
||
345 |
contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) |
|
346 |
contentids = contenus.keys() |
|
| 127 | 347 |
|
| 154 | 348 |
qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) |
349 |
for dse in qs: |
|
350 |
contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} |
|
351 |
||
352 |
qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) |
|
353 |
||
| 235 | 354 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = lang) |
| 154 | 355 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
|
122
fde8335a037c
Added Geographic Inclusion in the Django version (server only)
veltr
parents:
119
diff
changeset
|
356 |
|
| 154 | 357 |
for ts in qs: |
358 |
match_tag = ts.tag.id in matchtagids |
|
359 |
contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label}) |
|
360 |
||
361 |
if match_tag: |
|
362 |
contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order |
|
363 |
||
364 |
if contentlist is None: |
|
365 |
contenus = sorted(contenus.values(),key=lambda e: -e['score']) |
|
366 |
else: |
|
367 |
contenus = contenus.values() |
|
| 152 | 368 |
|
| 154 | 369 |
#tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] |
370 |
tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] |
|
371 |
#.select_related('dbpedia_fields') |
|
372 |
# hack to add only necessary fields in the group by |
|
373 |
# contournement bug https://code.djangoproject.com/ticket/17144 |
|
374 |
tagqs.query.clear_select_fields() |
|
375 |
tagqs.query.add_fields(['id','label'], False) |
|
376 |
tagqs.query.set_group_by() |
|
377 |
||
378 |
tagqslist = list(tagqs) |
|
| 135 | 379 |
|
| 154 | 380 |
dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) |
| 135 | 381 |
|
| 235 | 382 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = lang) |
| 154 | 383 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
| 127 | 384 |
|
| 154 | 385 |
tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist] |
386 |
||
387 |
countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) |
|
388 |
countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) |
|
389 |
||
390 |
discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) |
|
| 127 | 391 |
|
| 235 | 392 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = lang) |
| 154 | 393 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
394 |
||
395 |
||
396 |
disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] |
|
397 |
||
398 |
years = {} |
|
399 |
yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) |
|
400 |
for ty in yearqs: |
|
401 |
for year in range(ty.start_year, ty.end_year): |
|
402 |
years[year] = ty.nb + (years[year] if year in years else 0) |
|
403 |
||
404 |
yearchange = [] |
|
405 |
for year in sorted(years.keys()): |
|
406 |
score = years[year] |
|
407 |
if year < 2011: |
|
408 |
if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score): |
|
409 |
yearchange.append({'year': year, 'score': score}) |
|
410 |
if year+1 not in years and year != -1 and score != 0: |
|
411 |
yearchange.append({'year': year+1, 'score': 0}) |
|
412 |
||
413 |
tag_translations = {} |
|
414 |
for t in itertools.chain(labeltranslations,disciplines,tags): |
|
| 135 | 415 |
tag_translations[t['label']] = t['translated_label'] |
| 154 | 416 |
for c in contenus: |
417 |
for t in c['tags']: |
|
418 |
tag_translations[t['label']] = t['translated_label'] |
|
419 |
||
420 |
output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} |
|
421 |
outputstr = json.dumps(output) |
|
422 |
cache.set(cache_key, outputstr) |
|
423 |
||
424 |
return HttpResponse(content=outputstr, mimetype='application/json') |
|
| 238 | 425 |
|
426 |
def contentsbytag(request): |
|
427 |
||
428 |
lang = request.GET.get('lang',request.LANGUAGE_CODE) |
|
429 |
label = request.GET.get('label', None).strip().lower() |
|
430 |
max_tag_order = int(request.GET.get('mto', '30')) |
|
431 |
content_count = int(request.GET.get('contentcount', '30')) |
|
432 |
||
433 |
cache_key = fix_cache_key("contentsbytag-%s"%unicode(label).encode("utf-8")) |
|
434 |
||
435 |
outputstr = cache.get(cache_key) |
|
436 |
||
437 |
if outputstr is None: |
|
438 |
||
439 |
matchtagids = [] |
|
440 |
||
441 |
contentqs = Datasheet.objects.filter(validated=True) |
|
442 |
||
443 |
no_translate_langs = [ 'fr' ] |
|
444 |
if lang in no_translate_langs: |
|
445 |
tagsqs = Tag.objects.filter( label__iexact = label ) |
|
446 |
else: |
|
447 |
tagsqs = Tag.objects.select_related('dbpedia_fields').filter(dbpedia_fields__translations__label__iexact = label, dbpedia_fields__translations__language_code=lang) |
|
448 |
||
449 |
contentqs = contentqs.select_related('taggedsheet__tag').filter(taggedsheet__tag__in=tagsqs).distinct() |
|
450 |
||
451 |
cont_count = contentqs.count() |
|
452 |
||
453 |
contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) |
|
454 |
contentids = contenus.keys() |
|
455 |
||
456 |
qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) |
|
457 |
for dse in qs: |
|
458 |
contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} |
|
459 |
||
460 |
qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids).order_by('order')) |
|
461 |
||
462 |
if lang in no_translate_langs: |
|
463 |
translations = {} |
|
464 |
else: |
|
465 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = lang) |
|
466 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
|
467 |
||
468 |
for ts in qs: |
|
469 |
translated_label = translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label |
|
470 |
match_tag = ( label == translated_label.lower() ) |
|
471 |
contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translated_label}) |
|
472 |
||
473 |
if match_tag: |
|
474 |
contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order |
|
475 |
||
476 |
contenus = sorted(contenus.values(),key=lambda e: -e['score']) |
|
477 |
||
478 |
output = {'count': cont_count, 'contents': contenus} |
|
479 |
outputstr = json.dumps(output) |
|
480 |
cache.set(cache_key, outputstr) |
|
481 |
||
482 |
return HttpResponse(content=outputstr, mimetype='application/json') |