149 |
151 |
150 return HttpResponse(content=json.dumps(res), mimetype='application/json') |
152 return HttpResponse(content=json.dumps(res), mimetype='application/json') |
151 |
153 |
152 |
154 |
153 def filter(request): |
155 def filter(request): |
154 periode = request.GET.get('period',None) |
156 |
155 label = request.GET.get('label', None) |
157 cache_key = fix_cache_key(request.get_full_path()) |
156 country = request.GET.get('country', None) |
158 |
157 contentlist = request.GET.get('contentlist', None) |
159 outputstr = cache.get(cache_key) |
158 max_tag_order = request.GET.get('mto', 12) |
160 |
159 content_count = request.GET.get('contentcount', 12) |
161 if outputstr is None: |
160 tag_count = request.GET.get('tagcount', 30) |
162 periode = request.GET.get('period',None) |
161 |
163 label = request.GET.get('label', None) |
162 matchtagids = [] |
164 country = request.GET.get('country', None) |
163 |
165 contentlist = request.GET.get('contentlist', None) |
164 tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']) |
166 max_tag_order = request.GET.get('mto', 12) |
165 countryqs = Country.objects |
167 content_count = request.GET.get('contentcount', 12) |
166 discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields') |
168 tag_count = request.GET.get('tagcount', 30) |
167 yearqs = TagYears.objects |
169 |
168 |
170 matchtagids = [] |
169 contentqs = Datasheet.objects.filter(validated=True) |
171 |
170 labeltranslations = [] |
172 tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']) |
171 |
173 countryqs = Country.objects |
172 if label or periode or country or contentlist : |
174 discqs = Tag.objects.filter(category__label = u'Discipline artistique').select_related('dbpedia_fields') |
173 |
175 yearqs = TagYears.objects |
174 if periode: |
176 |
175 years = periode.split(",") |
177 contentqs = Datasheet.objects.filter(validated=True) |
176 start_year = int(years[0]) |
178 labeltranslations = [] |
177 end_year = int(years[0:2][-1]) |
179 |
178 delta = max(1, (end_year-start_year)/2) |
180 if label or periode or country or contentlist : |
179 minstart = start_year - delta |
181 |
180 maxend = end_year + delta |
182 if periode: |
181 matchtagqs = Tag.objects.filter(years__end_year__gte = start_year, |
183 years = periode.split(",") |
182 years__start_year__lte = end_year, |
184 start_year = int(years[0]) |
183 years__end_year__lte = maxend, |
185 end_year = int(years[0:2][-1]) |
184 years__start_year__gte = minstart) |
186 delta = max(1, (end_year-start_year)/2) |
185 matchtagids += [t.id for t in matchtagqs] |
187 minstart = start_year - delta |
186 contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, |
188 maxend = end_year + delta |
187 taggedsheet__order__lte = max_tag_order) |
189 matchtagqs = Tag.objects.filter(years__end_year__gte = start_year, |
188 |
190 years__start_year__lte = end_year, |
189 if label: |
191 years__end_year__lte = maxend, |
190 masters = [] |
192 years__start_year__gte = minstart) |
191 for txtlbl in label.split(","): |
193 matchtagids += [t.id for t in matchtagqs] |
192 matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip()) |
|
193 for t in matchtagqs: |
|
194 if t.id not in matchtagids: |
|
195 matchtagids.append(t.id) |
|
196 if t.dbpedia_fields: |
|
197 masters.append(t.dbpedia_fields) |
|
198 |
|
199 contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, |
194 contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, |
200 taggedsheet__order__lte = max_tag_order) |
195 taggedsheet__order__lte = max_tag_order) |
201 translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE) |
196 |
202 labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] |
197 if label: |
203 |
198 masters = [] |
204 if country: |
199 for txtlbl in label.split(","): |
205 for country_uri in country.split(","): |
200 matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(label__iexact = txtlbl.strip()) |
206 matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri) |
201 for t in matchtagqs: |
207 matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids] |
202 if t.id not in matchtagids: |
208 contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, |
203 matchtagids.append(t.id) |
209 taggedsheet__order__lte = max_tag_order) |
204 if t.dbpedia_fields: |
210 if contentlist: |
205 masters.append(t.dbpedia_fields) |
211 contentqs = contentqs.filter(id__in = contentlist.split(",")) |
206 |
212 |
207 contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, |
213 tagqs = tagqs.filter(datasheet__in = contentqs) |
208 taggedsheet__order__lte = max_tag_order) |
214 countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) |
209 translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE) |
215 discqs = discqs.filter(datasheet__in = contentqs) |
210 labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs] |
216 yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) |
211 |
217 |
212 if country: |
218 if contentlist is None: |
213 for country_uri in country.split(","): |
219 contentqs.order_by('?') |
214 matchtagqs = Tag.objects.filter(locatedin__country__dbpedia_uri = country_uri) |
220 |
215 matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids] |
221 cont_count = contentqs.count() |
216 contentqs = contentqs.filter(taggedsheet__tag__in = matchtagqs, |
222 |
217 taggedsheet__order__lte = max_tag_order) |
223 contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) |
218 if contentlist: |
224 contentids = contenus.keys() |
219 contentqs = contentqs.filter(id__in = contentlist.split(",")) |
225 |
220 |
226 qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) |
221 tagqs = tagqs.filter(datasheet__in = contentqs) |
227 for dse in qs: |
222 countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs) |
228 contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} |
223 discqs = discqs.filter(datasheet__in = contentqs) |
229 |
224 yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs) |
230 qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) |
225 |
231 |
226 if contentlist is None: |
232 transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE) |
227 contentqs.order_by('?') |
233 translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
228 |
234 |
229 cont_count = contentqs.count() |
235 for ts in qs: |
230 |
236 match_tag = ts.tag.id in matchtagids |
231 contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]]) |
237 contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label}) |
232 contentids = contenus.keys() |
238 |
233 |
239 if match_tag: |
234 qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids) |
240 contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order |
235 for dse in qs: |
241 |
236 contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude} |
242 if contentlist is None: |
237 |
243 contenus = sorted(contenus.values(),key=lambda e: -e['score']) |
238 qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order')) |
244 else: |
239 |
245 contenus = contenus.values() |
240 transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE) |
246 |
241 translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
247 #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] |
242 |
248 tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] |
243 for ts in qs: |
249 #.select_related('dbpedia_fields') |
244 match_tag = ts.tag.id in matchtagids |
250 # hack to add only necessary fields in the group by |
245 contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label}) |
251 # contournement bug https://code.djangoproject.com/ticket/17144 |
246 |
252 tagqs.query.clear_select_fields() |
247 if match_tag: |
253 tagqs.query.add_fields(['id','label'], False) |
248 contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order |
254 tagqs.query.set_group_by() |
249 |
255 |
250 if contentlist is None: |
256 tagqslist = list(tagqs) |
251 contenus = sorted(contenus.values(),key=lambda e: -e['score']) |
257 |
252 else: |
258 dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) |
253 contenus = contenus.values() |
259 |
254 |
260 transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE) |
255 #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count] |
261 translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
256 tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count] |
262 |
257 #.select_related('dbpedia_fields') |
263 tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist] |
258 # hack to add only necessary fields in the group by |
264 |
259 # contournement bug https://code.djangoproject.com/ticket/17144 |
265 countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) |
260 tagqs.query.clear_select_fields() |
266 countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) |
261 tagqs.query.add_fields(['id','label'], False) |
267 |
262 tagqs.query.set_group_by() |
268 discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) |
263 |
269 |
264 tagqslist = list(tagqs) |
270 transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE) |
265 |
271 translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
266 dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)]) |
272 |
267 |
273 |
268 transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE) |
274 disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] |
269 translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
275 |
270 |
276 years = {} |
271 tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist] |
277 yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) |
272 |
278 for ty in yearqs: |
273 countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet')) |
279 for year in range(ty.start_year, ty.end_year): |
274 countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs]) |
280 years[year] = ty.nb + (years[year] if year in years else 0) |
275 |
281 |
276 discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10]) |
282 yearchange = [] |
277 |
283 for year in sorted(years.keys()): |
278 transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE) |
284 score = years[year] |
279 translations = dict([(trans.master_id,trans.label) for trans in transqs]) |
285 if year < 2011: |
280 |
286 if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score): |
281 |
287 yearchange.append({'year': year, 'score': score}) |
282 disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist] |
288 if year+1 not in years and year != -1 and score != 0: |
283 |
289 yearchange.append({'year': year+1, 'score': 0}) |
284 years = {} |
290 |
285 yearqs = yearqs.annotate(nb=Count('tag__taggedsheet')) |
291 tag_translations = {} |
286 for ty in yearqs: |
292 for t in itertools.chain(labeltranslations,disciplines,tags): |
287 for year in range(ty.start_year, ty.end_year): |
293 tag_translations[t['label']] = t['translated_label'] |
288 years[year] = ty.nb + (years[year] if year in years else 0) |
294 for c in contenus: |
289 |
295 for t in c['tags']: |
290 yearchange = [] |
|
291 for year in sorted(years.keys()): |
|
292 score = years[year] |
|
293 if year < 2011: |
|
294 if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score): |
|
295 yearchange.append({'year': year, 'score': score}) |
|
296 if year+1 not in years and year != -1 and score != 0: |
|
297 yearchange.append({'year': year+1, 'score': 0}) |
|
298 |
|
299 tag_translations = {} |
|
300 for t in itertools.chain(labeltranslations,disciplines,tags): |
296 tag_translations[t['label']] = t['translated_label'] |
301 tag_translations[t['label']] = t['translated_label'] |
297 |
302 for c in contenus: |
298 output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} |
303 for t in c['tags']: |
299 |
304 tag_translations[t['label']] = t['translated_label'] |
300 return HttpResponse(content=json.dumps(output), mimetype='application/json') |
305 |
|
306 output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations} |
|
307 outputstr = json.dumps(output) |
|
308 cache.set(cache_key, outputstr) |
|
309 |
|
310 return HttpResponse(content=outputstr, mimetype='application/json') |