| author | veltr |
| Tue, 03 Jul 2012 17:50:26 +0200 | |
| changeset 205 | 8ff4fd1a6e9c |
| parent 204 | 0a1744477bc1 |
| child 206 | 7070d3acc3d4 |
| permissions | -rw-r--r-- |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
1 |
# -*- coding: utf-8 -*- |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
2 |
''' |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
3 |
Created on Jan 31, 2012 |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
4 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
5 |
@author: ymh |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
6 |
''' |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
7 |
from django.conf import settings |
| 154 | 8 |
from django.core.cache import cache |
| 135 | 9 |
from django.db.models import Q, Count |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
10 |
from django.http import HttpResponse |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
11 |
from hdabo.models import Tag, Datasheet, TaggedSheet |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
12 |
from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras |
| 152 | 13 |
from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields |
| 204 | 14 |
from hdalab.models.categories import WpCategory, WpCategoryInclusion, TagWpCategory |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
15 |
from hdalab.utils import fix_cache_key |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
16 |
import copy |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
17 |
import django.utils.simplejson as json |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
18 |
import hmac |
| 135 | 19 |
import itertools |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
20 |
import uuid |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
21 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
22 |
def taginfo(request): |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
23 |
label = request.GET.get('label', None) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
24 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
25 |
resobj = {'requested_label' : label} |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
26 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
27 |
resobj["content_count"] = Datasheet.objects.filter(taggedsheet__tag__label__iexact = label).distinct().count() |
| 135 | 28 |
|
| 165 | 29 |
res = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = label).order_by('-dbpedia_uri')[0:1] |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
30 |
if len(res) == 1: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
31 |
restag = res.get() |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
32 |
resobj["dbpedia_uri"] = restag.dbpedia_uri |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
33 |
if resobj["dbpedia_uri"] is not None and restag.dbpedia_fields is not None: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
34 |
dbfield = restag.dbpedia_fields |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
35 |
resobj["abstract"] = dbfield.abstract |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
36 |
resobj["dbpedia_label"] = dbfield.label |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
37 |
resobj["thumbnail"] = dbfield.thumbnail |
| 135 | 38 |
transqs = DbpediaFieldsTranslation.objects.filter(master=dbfield, language_code=request.LANGUAGE_CODE)[0:1] |
39 |
if transqs: |
|
40 |
trans = transqs.get() |
|
41 |
resobj['translated_abstract'] = trans.abstract |
|
42 |
resobj['translated_label'] = trans.label |
|
43 |
else: |
|
44 |
resobj['translated_abstract'] = dbfield.abstract |
|
45 |
resobj['translated_label'] = dbfield.label |
|
46 |
||
47 |
#res = Tag.objects.filter(label__iexact = label).order_by('-wikipedia_url')[0:1] |
|
48 |
#if len(res) == 1: |
|
49 |
# resobj["wikipedia_url"] = res.get().wikipedia_url |
|
50 |
if 'translated_label' in resobj: |
|
51 |
wikipedia_label = resobj['translated_label'] |
|
52 |
else: |
|
53 |
wikipedia_label = label |
|
54 |
wikipedia_label = wikipedia_label[0].capitalize() + wikipedia_label[1:] |
|
55 |
resobj["wikipedia_url"] = "http://%s.wikipedia.org/wiki/%s" % (request.LANGUAGE_CODE,wikipedia_label.replace(' ', '_')) |
|
| 150 | 56 |
# We don't use links at the moment, so I'll comment this line to speed up requests |
57 |
# resobj["links"] = [{'subject':tl.subject.label, 'object':tl.object.label} for tl in TagLinks.objects.select_related().filter(Q(subject__label__iexact = label) | Q(object__label__iexact = label))] |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
58 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
59 |
return HttpResponse(content=json.dumps(resobj), mimetype='application/json') |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
60 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
61 |
|
| 135 | 62 |
def tagtranslation(request): |
63 |
||
64 |
labels = request.GET.get('labels',None) |
|
65 |
||
66 |
if not labels: |
|
67 |
return HttpResponse(content=json.dumps({}), mimetype='application/json') |
|
68 |
||
69 |
labelslist = [lbl.strip() for lbl in labels.split(",")] |
|
70 |
masters = [] |
|
71 |
||
72 |
for lbl in labelslist: |
|
| 165 | 73 |
labelqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = lbl)[0:1] |
| 135 | 74 |
if len(labelqs) > 0: |
75 |
tag = labelqs.get() |
|
76 |
if tag.dbpedia_fields: |
|
77 |
masters.append(tag.dbpedia_fields) |
|
78 |
||
79 |
translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE) |
|
80 |
||
81 |
translations = dict([(t.master.label, t.label) for t in translationqs]) |
|
82 |
||
83 |
return HttpResponse(content=json.dumps(translations), mimetype='application/json') |
|
| 204 | 84 |
|
85 |
def subcat(category, globtags, level, max_level ): |
|
86 |
# recursive function used by cattree |
|
87 |
tags = Tag.objects.select_related('wp_categories__wp_category').filter(wp_categories__wp_category = category) |
|
88 |
taglabels = [k for k in dict([(t.label,t.label) for t in tags])] |
|
89 |
catlabel = category.label |
|
90 |
resobj = { |
|
91 |
'category': category.label, |
|
92 |
'tags': [], |
|
93 |
'contents': [] |
|
94 |
} |
|
95 |
for label in taglabels: |
|
96 |
if label == catlabel: |
|
97 |
globtags[label] = {'level': level, 'access': resobj } |
|
98 |
else: |
|
99 |
tag_in_list = {'label' : label, 'contents': []} |
|
100 |
resobj['tags'].append(tag_in_list) |
|
101 |
globtags[label] = {'level': (level + 1), 'access': tag_in_list } |
|
102 |
||
103 |
if level < max_level: |
|
104 |
subcats = WpCategory.objects.select_related('parent_categories__parent_category').filter(parent_categories__parent_category = category) |
|
105 |
resobj['sub_categories'] = [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))] |
|
106 |
return resobj |
|
107 |
||
108 |
def cleantags(category): |
|
109 |
if category.has_key('contents') and len(category['contents']) == 0: |
|
110 |
del category['contents'] |
|
111 |
if category.has_key('tags'): |
|
112 |
category['tags'] = [tag for tag in category['tags'] if len(tag['contents'])] |
|
113 |
if len(category['tags']) == 0: |
|
114 |
del category['tags'] |
|
115 |
if category.has_key('sub_categories'): |
|
116 |
sub_cats = [] |
|
117 |
for sub_cat in category['sub_categories']: |
|
118 |
cat = cleantags(sub_cat) |
|
119 |
if cat.has_key('tags') or cat.has_key('sub_categories') or cat.has_key('contents'): |
|
120 |
sub_cats.append(cat) |
|
121 |
category['sub_categories'] = sub_cats |
|
122 |
if len(category['sub_categories']) == 0: |
|
123 |
del category['sub_categories'] |
|
124 |
return category |
|
125 |
||
126 |
def cattree(request): |
|
127 |
# Gets the category tree from a label |
|
128 |
MAX_TAG_ORDER = 5 |
|
129 |
MAX_LEVEL = 3 |
|
130 |
label = request.GET.get('label', None) |
|
131 |
globtags = {} |
|
132 |
resobj = None |
|
133 |
master_category = WpCategory.objects.filter(label__iexact=label)[0:1] |
|
134 |
if len(master_category): |
|
135 |
resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL ) |
|
| 135 | 136 |
|
| 204 | 137 |
tag_list = [k for k in globtags] |
138 |
||
139 |
if len(tag_list): |
|
140 |
contents = [] |
|
| 205 | 141 |
datasheets = Datasheet.objects.select_related('taggedsheet__tag').filter(validated = True, taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct() |
| 204 | 142 |
for datasheet in datasheets: |
143 |
# Calculating where we add the datasheet in the tree |
|
144 |
maintag = None |
|
145 |
maintagscore = -5 |
|
146 |
for ts in TaggedSheet.objects.select_related('tag','datasheet').filter(datasheet__id=datasheet.id,order__lte=MAX_TAG_ORDER): |
|
147 |
label = ts.tag.label |
|
148 |
if globtags.has_key(label): |
|
149 |
score = 3 * globtags[label]['level'] - ts.order |
|
150 |
if score > maintagscore: |
|
151 |
maintagscore = score |
|
152 |
maintag = label |
|
153 |
if maintag is not None: |
|
| 205 | 154 |
globtags[maintag]['access']['contents'].append({'id': datasheet.id, 'title': datasheet.title, 'url': datasheet.url}) |
| 204 | 155 |
cleantags(resobj) |
156 |
||
157 |
# resobj['contents'] = [{'id': d.id, 'title': d.title, 'tags': [t.label for t in d.tags.filter(taggedsheet__order__lte=5)]} for d in datasheets] |
|
158 |
||
159 |
return HttpResponse(content=json.dumps(resobj), mimetype='application/json') |
|
| 135 | 160 |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
161 |
def sessioninfo(request): |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
162 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
163 |
data = json.loads(request.GET.get('data', "{}")) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
164 |
write = False |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
165 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
166 |
if 'sessionid' in request.GET: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
167 |
request.session['sessionid'] = request.GET['sessionid'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
168 |
if 'sessionkey' in request.GET: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
169 |
request.session['sessionkey'] = request.GET['sessionkey'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
170 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
171 |
if 'sessionid' in request.session: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
172 |
sessionid = request.session['sessionid'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
173 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
174 |
if HdaSession.objects.filter(sessionid=sessionid).count() == 1: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
175 |
sessionkey = request.session.get('sessionkey',None) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
176 |
hm = hmac.new(settings.SECRET_KEY, sessionid) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
177 |
if hm.hexdigest() == sessionkey: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
178 |
write = True |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
179 |
else: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
180 |
del request.session['sessionid'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
181 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
182 |
if 'sessionid' not in request.session: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
183 |
sessionid = unicode(uuid.uuid1()) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
184 |
HdaSession.objects.create(sessionid=sessionid, data=json.dumps({})) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
185 |
write = True |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
186 |
request.session['sessionid'] = sessionid |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
187 |
request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest() |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
188 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
189 |
if write and data: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
190 |
HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data)) |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
191 |
else: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
192 |
data = HdaSession.objects.get(sessionid=sessionid).data |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
193 |
data = json.loads(data) if data else {} |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
194 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
195 |
resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid } |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
196 |
if write: |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
197 |
resobj['sessionkey'] = request.session['sessionkey'] |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
198 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
199 |
return HttpResponse(content=json.dumps(resobj), mimetype='application/json') |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
200 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
201 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
202 |
def tagsearch(request): |
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
203 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
204 |
q = request.GET.get('term',None) |
| 150 | 205 |
if q: |
206 |
lq = q.lower() |
|
| 175 | 207 |
qs = Tag.objects.filter(datasheet__validated=True).filter( Q(label__icontains = q ) | Q(dbpedia_fields__translations__label__icontains = q, dbpedia_fields__translations__language_code=request.LANGUAGE_CODE), ~Q(dbpedia_uri = None)) if q else Tag.objects.filter(~Q(dbpedia_uri = None)) |
| 150 | 208 |
qs = qs.annotate(nb=Count('datasheet')).order_by('-nb')[:20] |
| 152 | 209 |
|
210 |
qslist = list(qs) |
|
211 |
||
212 |
transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=request.LANGUAGE_CODE).select_related("master") |
|
213 |
||
214 |
translations = dict([(tr.master.tag_id, tr.label) for tr in transqs]) |
|
215 |
||
| 150 | 216 |
res = [] |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
217 |
|
| 152 | 218 |
for t in qslist: |
| 150 | 219 |
resobj = {'value':t.label,'nb':t.nb} |
| 152 | 220 |
if t.id in translations: |
221 |
resobj['label'] = translations[t.id] |
|
| 150 | 222 |
else: |
223 |
resobj['label'] = t.label |
|
224 |
if q is None or resobj['label'].lower().find(lq) != -1: |
|
225 |
res.append(resobj) |
|
226 |
||
227 |
return HttpResponse(content=json.dumps(res), mimetype='application/json') |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
228 |
|
| 205 | 229 |
def catsearch(request): |
230 |
||
231 |
q = request.GET.get('term',None) |
|
232 |
||
233 |
qs = WpCategory.objects.filter( Q(label__icontains = ' ' + q ) | Q(label__istartswith = q)) |
|
234 |
||
235 |
res = [{'value':t.label} for t in qs] |
|
236 |
||
237 |
return HttpResponse(content=json.dumps(res), mimetype='application/json') |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
238 |
|
|
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
239 |
def filter(request): |
| 127 | 240 |
|
| 158 | 241 |
periode = request.GET.get('period',None) |
242 |
label = request.GET.get('label', None) |
|
243 |
country = request.GET.get('country', None) |
|
244 |
contentlist = request.GET.get('contentlist', None) |
|
245 |
max_tag_order = request.GET.get('mto', 12) |
|
246 |
content_count = request.GET.get('contentcount', 12) |
|
247 |
tag_count = request.GET.get('tagcount', 30) |
|
248 |
||
| 163 | 249 |
key_parts = ("filter",request.LANGUAGE_CODE,periode,label,country,contentlist,max_tag_order,content_count,tag_count) |
250 |
key_parts = [unicode(p).encode("utf-8") for p in key_parts] |
|
251 |
||
252 |
cache_key = fix_cache_key("-".join(key_parts)) |
|
| 154 | 253 |
|
254 |
outputstr = cache.get(cache_key) |
|
| 127 | 255 |
|
| 154 | 256 |
if outputstr is None: |
257 |
||
258 |
matchtagids = [] |
|
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
259 |
|
| 165 | 260 |
tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']).filter(~Q(dbpedia_uri = None)) |
| 154 | 261 |
countryqs = Country.objects |
| 165 | 262 |
discqs = Tag.objects.filter(~Q(dbpedia_uri = None), category__label = u'Discipline artistique').select_related('dbpedia_fields') |
| 154 | 263 |
yearqs = TagYears.objects |
264 |
||
265 |
contentqs = Datasheet.objects.filter(validated=True) |
|
266 |
labeltranslations = [] |
|
267 |
||
268 |
if label or periode or country or contentlist : |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
269 |
matchtagqslist = [] |
|
119
e3ebe3545f72
first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
270 |
|
| 154 | 271 |
if periode: |
272 |
years = periode.split(",") |
|
273 |
start_year = int(years[0]) |
|
274 |
end_year = int(years[0:2][-1]) |
|
275 |
delta = max(1, (end_year-start_year)/2) |
|
276 |
minstart = start_year - delta |
|
277 |
maxend = end_year + delta |
|
| 165 | 278 |
matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None), |
279 |
years__end_year__gte = start_year, |
|
280 |
years__start_year__lte = end_year, |
|
281 |
years__end_year__lte = maxend, |
|
282 |
years__start_year__gte = minstart, |
|
283 |
) |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
284 |
matchtagqslist.append(matchtagqs) |
| 154 | 285 |
|
286 |
if label: |
|
287 |
for txtlbl in label.split(","): |
|
| 165 | 288 |
matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = txtlbl.strip()) |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
289 |
matchtagqslist.append(matchtagqs) |
| 154 | 290 |
|
291 |
if country: |
|
292 |
for country_uri in country.split(","): |
|
| 165 | 293 |
matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),locatedin__country__dbpedia_uri = country_uri) |
| 154 | 294 |
matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids] |
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
295 |
matchtagqslist.append(matchtagqs) |
| 154 | 296 |
if contentlist: |
297 |
contentqs = contentqs.filter(id__in = contentlist.split(",")) |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
298 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
299 |
tagcond = None |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
300 |
tagcondid = None |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
301 |
for matchtagqs in matchtagqslist: |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
302 |
newcond = Q(id__in = TaggedSheet.objects.filter(tag__in = copy.deepcopy(matchtagqs), order__lte = max_tag_order).values('datasheet_id')) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
303 |
newcondid = Q(id__in = matchtagqs) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
304 |
tagcond = newcond if tagcond is None else (tagcond & newcond) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
305 |
tagcondid = newcondid if tagcondid is None else (tagcondid | newcondid) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
306 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
307 |
contentqs = contentqs.filter(tagcond).distinct() |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
308 |
matchtagidsqs = list(Tag.objects.select_related("dbpedia_fields").only("id").filter(tagcondid)) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
309 |
matchtagids = [t.id for t in matchtagidsqs] |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
310 |
masters = [t.dbpedia_fields for t in matchtagidsqs if t.dbpedia_fields is not None] |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
311 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
312 |
translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE) |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
313 |
labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] |
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
314 |
|
|
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
315 |
|
| 154 | 316 |
tagqs = tagqs.filter(datasheet__in = contentqs) |
317 |
countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) |
|
318 |
discqs = discqs.filter(datasheet__in = contentqs) |
|
319 |
yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) |
|
|
122
fde8335a037c
Added Geographic Inclusion in the Django version (server only)
veltr
parents:
119
diff
changeset
|
320 |
|
| 154 | 321 |
if contentlist is None: |
322 |
contentqs.order_by('?') |
|
|
172
8f47c67c6d28
Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents:
165
diff
changeset
|
323 |
|
| 154 | 324 |
cont_count = contentqs.count() |
325 |
||
326 |
contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) |
|
327 |
contentids = contenus.keys() |
|
| 127 | 328 |
|
| 154 | 329 |
qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) |
330 |
for dse in qs: |
|
331 |
contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} |
|
332 |
||
333 |
qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) |
|
334 |
||
335 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE) |
|
336 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
|
|
122
fde8335a037c
Added Geographic Inclusion in the Django version (server only)
veltr
parents:
119
diff
changeset
|
337 |
|
| 154 | 338 |
for ts in qs: |
339 |
match_tag = ts.tag.id in matchtagids |
|
340 |
contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label}) |
|
341 |
||
342 |
if match_tag: |
|
343 |
contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order |
|
344 |
||
345 |
if contentlist is None: |
|
346 |
contenus = sorted(contenus.values(),key=lambda e: -e['score']) |
|
347 |
else: |
|
348 |
contenus = contenus.values() |
|
| 152 | 349 |
|
| 154 | 350 |
#tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] |
351 |
tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] |
|
352 |
#.select_related('dbpedia_fields') |
|
353 |
# hack to add only necessary fields in the group by |
|
354 |
# contournement bug https://code.djangoproject.com/ticket/17144 |
|
355 |
tagqs.query.clear_select_fields() |
|
356 |
tagqs.query.add_fields(['id','label'], False) |
|
357 |
tagqs.query.set_group_by() |
|
358 |
||
359 |
tagqslist = list(tagqs) |
|
| 135 | 360 |
|
| 154 | 361 |
dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) |
| 135 | 362 |
|
| 154 | 363 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE) |
364 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
|
| 127 | 365 |
|
| 154 | 366 |
tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist] |
367 |
||
368 |
countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) |
|
369 |
countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) |
|
370 |
||
371 |
discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) |
|
| 127 | 372 |
|
| 154 | 373 |
transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE) |
374 |
translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
|
375 |
||
376 |
||
377 |
disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] |
|
378 |
||
379 |
years = {} |
|
380 |
yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) |
|
381 |
for ty in yearqs: |
|
382 |
for year in range(ty.start_year, ty.end_year): |
|
383 |
years[year] = ty.nb + (years[year] if year in years else 0) |
|
384 |
||
385 |
yearchange = [] |
|
386 |
for year in sorted(years.keys()): |
|
387 |
score = years[year] |
|
388 |
if year < 2011: |
|
389 |
if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score): |
|
390 |
yearchange.append({'year': year, 'score': score}) |
|
391 |
if year+1 not in years and year != -1 and score != 0: |
|
392 |
yearchange.append({'year': year+1, 'score': 0}) |
|
393 |
||
394 |
tag_translations = {} |
|
395 |
for t in itertools.chain(labeltranslations,disciplines,tags): |
|
| 135 | 396 |
tag_translations[t['label']] = t['translated_label'] |
| 154 | 397 |
for c in contenus: |
398 |
for t in c['tags']: |
|
399 |
tag_translations[t['label']] = t['translated_label'] |
|
400 |
||
401 |
output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} |
|
402 |
outputstr = json.dumps(output) |
|
403 |
cache.set(cache_key, outputstr) |
|
404 |
||
405 |
return HttpResponse(content=outputstr, mimetype='application/json') |