web/hdalab/views/ajax.py
author veltr
Thu, 11 Oct 2012 18:22:45 +0200
changeset 252 a46461f8a74f
parent 250 7c3f54ce68af
child 253 1113c3874dd6
permissions -rw-r--r--
Separated Thesaurus and WP Categories
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
'''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Created on Jan 31, 2012
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
@author: ymh
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
'''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
from django.conf import settings
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
     8
from django.core.cache import cache
248
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
     9
from django.db.models import Q, Count, Min
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.http import HttpResponse
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from hdabo.models import Tag, Datasheet, TaggedSheet
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
    12
from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
    13
from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    14
from hdalab.models.categories import WpCategory, WpCategoryInclusion, TagWpCategory
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
    15
from hdalab.utils import fix_cache_key
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
    16
import copy
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
import django.utils.simplejson as json
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
import hmac
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    19
import itertools
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
import uuid
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    22
def tagtranslation(request):
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    23
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
    24
    lang = request.GET.get('lang',request.LANGUAGE_CODE)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    25
    labels = request.GET.get('labels',None)
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    26
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    27
    if not labels:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    28
        return HttpResponse(content=json.dumps({}), mimetype='application/json')
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    29
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    30
    labelslist = [lbl.strip() for lbl in labels.split(",")]
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    31
    masters = []
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    32
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    33
    for lbl in labelslist:
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
    34
        labelqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = lbl)[0:1]
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    35
        if len(labelqs) > 0:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    36
            tag = labelqs.get()
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    37
            if tag.dbpedia_fields:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    38
                masters.append(tag.dbpedia_fields)
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    39
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
    40
    translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    41
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    42
    translations = dict([(t.master.label, t.label) for t in translationqs])
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    43
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    44
    return HttpResponse(content=json.dumps(translations), mimetype='application/json')
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    45
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    46
def subcat(category, globtags, level, max_level ):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    47
    # recursive function used by cattree
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    48
    catlabel = category.label
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    49
    tags = Tag.objects.filter(wp_categories__wp_category = category).distinct()
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    50
    taglabels = [k for k in dict([(t.label,t.label) for t in tags])]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    51
    resobj = {
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    52
          'label': category.label,
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    53
          'themes': [],
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    54
          'contents': []
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    55
      }
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    56
    for label in taglabels:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    57
        if label == catlabel:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    58
            globtags[label] = {'level': level, 'access': resobj }
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    59
        else:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    60
            tag_in_list = {'label' : label, 'contents': []}
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    61
            resobj['themes'].append(tag_in_list)
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    62
            globtags[label] = {'level': (level + 1), 'access': tag_in_list }
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    63
            
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    64
    if level < max_level:
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    65
        subcats = WpCategory.objects.filter(parent_categories__parent_category = category)
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    66
        resobj['themes'] += [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))]
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    67
    return resobj
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    68
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    69
def cleantags(category):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    70
    if category.has_key('contents') and len(category['contents']) == 0:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    71
        del category['contents']
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    72
    if category.has_key('contents'):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    73
        category['contents'] = sorted(category['contents'], key=lambda content: -content['score'])
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    74
    if category.has_key('themes'):
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    75
        themes = []
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    76
        for theme in category['themes']:
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    77
            clean_theme = cleantags(theme)
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    78
            if clean_theme.has_key('themes') or clean_theme.has_key('contents'):
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    79
                themes.append(clean_theme)
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    80
        category['themes'] = sorted(themes, key=lambda cat: cat['label'])
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    81
        if len(category['themes']) == 0:
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
    82
            del category['themes']
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    83
    return category
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    84
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    85
def cattree(request):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    86
    # Gets the category tree from a label
232
eb02eed5f7d1 UI Improvements (cf. B Sajus .odp)
veltr
parents: 212
diff changeset
    87
    ROOT_MAX_TAG_ORDER = 8
eb02eed5f7d1 UI Improvements (cf. B Sajus .odp)
veltr
parents: 212
diff changeset
    88
    MAX_TAG_ORDER = 8
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    89
    MAX_LEVEL = 3
212
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
    90
    LEVEL_COEFF = 5
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    91
    label = request.GET.get('label', None)
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    92
    lowerlabel = label.lower()
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    93
    globtags = {}
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    94
    resobj = None
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    95
    master_category = WpCategory.objects.filter(label__iexact=label)[0:1]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    96
    if len(master_category):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    97
        resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL )
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    98
    
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    99
#    tag_list = [k for k in globtags]
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   100
    
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   101
#    if len(tag_list):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   102
    contents = []
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   103
#    datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct()
232
eb02eed5f7d1 UI Improvements (cf. B Sajus .odp)
veltr
parents: 212
diff changeset
   104
    datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__iexact = label, taggedsheet__order__lte = ROOT_MAX_TAG_ORDER).select_related('organisation').distinct()
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   105
    for datasheet in datasheets:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   106
        # Calculating where we add the datasheet in the tree
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   107
        maintag = None
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   108
        maintagscore = -5
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   109
        dsscore = 0
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   110
        rootscore = 0
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   111
        for ts in TaggedSheet.objects.select_related('tag','datasheet').filter(datasheet__id=datasheet.id,order__lte=MAX_TAG_ORDER):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   112
            label = ts.tag.label
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   113
            if globtags.has_key(label):
212
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   114
                score = LEVEL_COEFF * globtags[label]['level'] - ts.order
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   115
                if score > maintagscore:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   116
                    maintagscore = score
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   117
                    maintag = label
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   118
                    dsscore = (MAX_TAG_ORDER - ts.order)
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   119
                if label.lower() == lowerlabel:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   120
                    rootscore = (ROOT_MAX_TAG_ORDER - ts.order)
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   121
        if maintag is not None:
212
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   122
            globtags[maintag]['access']['contents'].append({
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   123
                'id': datasheet.id,
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   124
                'title': datasheet.title,
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   125
                'url': datasheet.url,
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   126
                'description': datasheet.description,
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   127
                'hda_id': datasheet.hda_id,
232
eb02eed5f7d1 UI Improvements (cf. B Sajus .odp)
veltr
parents: 212
diff changeset
   128
                'organization': datasheet.organisation.name,
eb02eed5f7d1 UI Improvements (cf. B Sajus .odp)
veltr
parents: 212
diff changeset
   129
                'score': max(dsscore, rootscore)
212
3e1193c6dce3 Display improvements on category tree
veltr
parents: 208
diff changeset
   130
            })
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   131
    cleantags(resobj)
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   132
                    
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   133
#        resobj['contents'] = [{'id': d.id, 'title': d.title, 'tags': [t.label for t in d.tags.filter(taggedsheet__order__lte=5)]} for d in datasheets]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   134
    
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   135
    return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   136
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   137
def sessioninfo(request):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   138
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
    data = json.loads(request.GET.get('data', "{}"))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   140
    write = False
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   141
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   142
    if 'sessionid' in request.GET:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
        request.session['sessionid'] = request.GET['sessionid']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   144
    if 'sessionkey' in request.GET:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   145
        request.session['sessionkey'] = request.GET['sessionkey']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   146
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   147
    if 'sessionid' in request.session:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   148
        sessionid = request.session['sessionid']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   149
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   150
        if HdaSession.objects.filter(sessionid=sessionid).count() == 1:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   151
            sessionkey = request.session.get('sessionkey',None)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   152
            hm = hmac.new(settings.SECRET_KEY, sessionid)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   153
            if hm.hexdigest() == sessionkey:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   154
                write = True            
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   155
        else:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   156
            del request.session['sessionid']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   157
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   158
    if 'sessionid' not in request.session:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   159
        sessionid = unicode(uuid.uuid1())
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   160
        HdaSession.objects.create(sessionid=sessionid, data=json.dumps({}))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   161
        write = True
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   162
        request.session['sessionid'] = sessionid
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   163
        request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest()
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   164
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
    if write and data:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   166
        HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   167
    else:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   168
        data = HdaSession.objects.get(sessionid=sessionid).data
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   169
        data = json.loads(data) if data else {}           
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   170
         
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   171
    resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid }
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   172
    if write:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
        resobj['sessionkey'] = request.session['sessionkey']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
    return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   176
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   177
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
def tagsearch(request):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   179
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
    q = request.GET.get('term',None)
238
f818b9430585 Corrections in Completion
veltr
parents: 235
diff changeset
   181
    maxcount = int(request.GET.get('count','40'))
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   182
    lang = request.GET.get('lang',request.LANGUAGE_CODE)
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   183
    
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   184
    stemming_langs = [ 'fr', 'en', 'de', 'it' ]
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   185
    # For Japanese, there are no word boundaries, we should not use the regexp in that case
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   186
    no_translate_langs = [ 'fr' ]
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   187
    
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   188
    if q:
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   189
        lq = q.lower()
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   190
        qs = Tag.objects.select_related('dbpedia_fields').filter(datasheet__validated=True)
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   191
        qrx = '(\\m|\\b)%s'%q
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   192
        if lang in no_translate_langs:
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   193
            if lang in stemming_langs:
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   194
                qs = qs.filter( label__iregex = qrx )
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   195
            else:
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   196
                qs = qs.filter( label__icontains = q )
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   197
        else:
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   198
            if lang in stemming_langs:
243
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   199
                qs = qs.filter(dbpedia_fields__translations__label__iregex=qrx, dbpedia_fields__translations__language_code=lang, dbpedia_fields__translations__is_label_translated = True)
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   200
            else:
243
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   201
                qs = qs.filter(dbpedia_fields__translations__label__icontains=q, dbpedia_fields__translations__language_code=lang, dbpedia_fields__translations__is_label_translated = True)
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   202
    else:
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   203
        qs = Tag.objects.filter(~Q(dbpedia_uri = None))
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   204
           
238
f818b9430585 Corrections in Completion
veltr
parents: 235
diff changeset
   205
    qs = qs.annotate(nb=Count('datasheet',distinct=True)).order_by('-nb')[:maxcount]
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   206
    
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   207
    qslist = list(qs)
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   208
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   209
    if lang in no_translate_langs:
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   210
        translations = {}
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   211
    else:
243
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   212
        transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=lang, is_label_translated=True).select_related("master")
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   213
        translations = dict([(tr.master.tag_id, {'label':tr.label,'abstract':tr.abstract, 'is_label_translated': tr.is_label_translated}) for tr in transqs])
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   214
    
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   215
    res = []
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   216
    
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   217
    for t in qslist:
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   218
        dbfields = t.dbpedia_fields
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   219
        resobj = {'original_label':t.label,'nb':t.nb}
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   220
        resobj['thumbnail'] = dbfields.thumbnail if dbfields is not None else None
243
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   221
        
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   222
#        if t.id in translations and not translations[t.id].get('is_label_translated', True):
1f2840354865 correct filter on tag completion to avoid tags that are not translated
ymh <ymh.work@gmail.com>
parents: 238
diff changeset
   223
#            continue
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   224
        if t.id in translations:
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   225
            resobj['value'] = translations[t.id]['label']
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   226
            resobj['abstract'] = translations[t.id]['abstract']
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   227
        else:
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   228
            resobj['value'] = t.label
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   229
            resobj['abstract'] = dbfields.abstract if dbfields is not None else None
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   230
        if q is None or resobj['value'].lower().find(lq) != -1:
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   231
            res.append(resobj)
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   232
    
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   233
    return HttpResponse(content=json.dumps(res), mimetype='application/json')
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   234
205
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   235
def catsearch(request):
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   236
    
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   237
    q = request.GET.get('term',None)
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   238
    
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   239
    # On ne récupère que les catégories qui sont également des tags
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   240
    qrx = '(\\m|\\b)%s'%q
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   241
    qs = Tag.objects.filter(label__iregex=q)
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   242
    
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   243
    labels = [tag.label for tag in qs]
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   244
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   245
    qs = WpCategory.objects.annotate(nb=Count('child_categories__child_category__tags')).filter(label__in = labels, nb__gt=0)
205
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   246
    
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   247
    res = [{'value':t.label} for t in qs]
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   248
    
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   249
    return HttpResponse(content=json.dumps(res), mimetype='application/json')
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   250
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   251
def filter(request):
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   252
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   253
    lang = request.GET.get('lang',request.LANGUAGE_CODE)
158
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   254
    periode = request.GET.get('period',None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   255
    label = request.GET.get('label', None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   256
    country = request.GET.get('country', None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   257
    contentlist = request.GET.get('contentlist', None)
238
f818b9430585 Corrections in Completion
veltr
parents: 235
diff changeset
   258
    max_tag_order = int(request.GET.get('mto', '12'))
158
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   259
    content_count = request.GET.get('contentcount', 12)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   260
    tag_count = request.GET.get('tagcount', 30)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   261
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   262
    key_parts = ("filter",lang,periode,label,country,contentlist,max_tag_order,content_count,tag_count)
163
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   263
    key_parts = [unicode(p).encode("utf-8") for p in key_parts]
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   264
    
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   265
    cache_key = fix_cache_key("-".join(key_parts))
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   266
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   267
    outputstr = cache.get(cache_key)
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   268
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   269
    if outputstr is None:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   270
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   271
        matchtagids = []
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   272
        
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   273
        tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']).filter(~Q(dbpedia_uri = None))
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   274
        countryqs = Country.objects
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   275
        discqs = Tag.objects.filter(~Q(dbpedia_uri = None), category__label = u'Discipline artistique').select_related('dbpedia_fields')
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   276
        yearqs = TagYears.objects
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   277
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   278
        contentqs = Datasheet.objects.filter(validated=True)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   279
        labeltranslations = []
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   280
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   281
        if label or periode or country or contentlist :
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   282
            matchtagqslist = []
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   283
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   284
            if periode:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   285
                years = periode.split(",")
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   286
                start_year = int(years[0])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   287
                end_year = int(years[0:2][-1])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   288
                delta = max(1, (end_year-start_year)/2)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   289
                minstart = start_year - delta
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   290
                maxend = end_year + delta
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   291
                matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   292
                                                years__end_year__gte = start_year, 
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   293
                                                years__start_year__lte = end_year,
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   294
                                                years__end_year__lte = maxend,
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   295
                                                years__start_year__gte = minstart,
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   296
                                                )
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   297
                matchtagqslist.append(matchtagqs)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   298
                
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   299
            if label:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   300
                for txtlbl in label.split(","):
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   301
                    matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = txtlbl.strip())
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   302
                    matchtagqslist.append(matchtagqs)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   303
                
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   304
            if country:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   305
                for country_uri in country.split(","):
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   306
                    matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),locatedin__country__dbpedia_uri = country_uri)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   307
                    matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids]
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   308
                    matchtagqslist.append(matchtagqs)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   309
            if contentlist:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   310
                contentqs = contentqs.filter(id__in = contentlist.split(","))
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   311
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   312
            tagcond = None
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   313
            tagcondid = None
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   314
            for matchtagqs in matchtagqslist:
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   315
                newcond = Q(id__in = TaggedSheet.objects.filter(tag__in = copy.deepcopy(matchtagqs), order__lte = max_tag_order).values('datasheet_id'))
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   316
                newcondid = Q(id__in = matchtagqs)
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   317
                tagcond = newcond if tagcond is None else (tagcond & newcond)
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   318
                tagcondid = newcondid if tagcondid is None else (tagcondid | newcondid)
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   319
            
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   320
            contentqs = contentqs.filter(tagcond).distinct()
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   321
            matchtagidsqs = list(Tag.objects.select_related("dbpedia_fields").only("id").filter(tagcondid))
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   322
            matchtagids = [t.id for t in matchtagidsqs]            
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   323
            masters = [t.dbpedia_fields for t in matchtagidsqs if t.dbpedia_fields is not None]
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   324
            
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   325
            translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=lang)    
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   326
            labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs]
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   327
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   328
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   329
            tagqs = tagqs.filter(datasheet__in = contentqs)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   330
            countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   331
            discqs = discqs.filter(datasheet__in = contentqs)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   332
            yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs)
122
fde8335a037c Added Geographic Inclusion in the Django version (server only)
veltr
parents: 119
diff changeset
   333
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   334
        if contentlist is None:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   335
            contentqs.order_by('?')
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   336
                        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   337
        cont_count = contentqs.count()
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   338
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   339
        contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   340
        contentids = contenus.keys()
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   341
        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   342
        qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   343
        for dse in qs:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   344
            contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   345
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   346
        qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order'))
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   347
        
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   348
        transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = lang)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   349
        translations = dict([(trans.master_id,trans.label) for trans in transqs])
122
fde8335a037c Added Geographic Inclusion in the Django version (server only)
veltr
parents: 119
diff changeset
   350
        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   351
        for ts in qs:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   352
            match_tag = ts.tag.id in matchtagids
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   353
            contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label})
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   354
            
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   355
            if match_tag:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   356
                contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   357
            
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   358
        if contentlist is None:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   359
            contenus = sorted(contenus.values(),key=lambda e: -e['score'])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   360
        else:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   361
            contenus = contenus.values()
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   362
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   363
        #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   364
        tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   365
        #.select_related('dbpedia_fields')
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   366
        # hack to add only necessary fields in the group by
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   367
        # contournement bug https://code.djangoproject.com/ticket/17144
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   368
        tagqs.query.clear_select_fields()
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   369
        tagqs.query.add_fields(['id','label'], False)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   370
        tagqs.query.set_group_by()
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   371
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   372
        tagqslist = list(tagqs)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   373
        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   374
        dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)])
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   375
    
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   376
        transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = lang)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   377
        translations = dict([(trans.master_id,trans.label) for trans in transqs])
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   378
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   379
        tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   380
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   381
        countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet'))
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   382
        countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   383
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   384
        discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10])
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   385
            
235
b41f0b200940 Added autocomplete
veltr
parents: 232
diff changeset
   386
        transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = lang)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   387
        translations = dict([(trans.master_id,trans.label) for trans in transqs])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   388
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   389
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   390
        disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   391
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   392
        years = {}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   393
        yearqs = yearqs.annotate(nb=Count('tag__taggedsheet'))
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   394
        for ty in yearqs:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   395
            for year in range(ty.start_year, ty.end_year):
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   396
                years[year] = ty.nb + (years[year] if year in years else 0)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   397
                
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   398
        yearchange = []
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   399
        for year in sorted(years.keys()):
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   400
            score = years[year]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   401
            if year < 2011:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   402
                if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score):
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   403
                    yearchange.append({'year': year, 'score': score})
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   404
                if year+1 not in years and year != -1 and score != 0:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   405
                    yearchange.append({'year': year+1, 'score': 0})
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   406
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   407
        tag_translations = {}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   408
        for t in itertools.chain(labeltranslations,disciplines,tags):
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   409
            tag_translations[t['label']] = t['translated_label']
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   410
        for c in contenus:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   411
            for t in c['tags']:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   412
                tag_translations[t['label']] = t['translated_label']
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   413
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   414
        output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   415
        outputstr = json.dumps(output)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   416
        cache.set(cache_key, outputstr)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   417
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   418
    return HttpResponse(content=outputstr, mimetype='application/json')
238
f818b9430585 Corrections in Completion
veltr
parents: 235
diff changeset
   419
248
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   420
def subtree(tree):
250
7c3f54ce68af Merged Completion List back into Facettes
veltr
parents: 248
diff changeset
   421
    MAX_TAG_ORDER = 16
248
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   422
    label = tree['label']
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   423
    sub = tree.get('contents',[])
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   424
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   425
    datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__iexact = label, taggedsheet__order__lte = MAX_TAG_ORDER).annotate(tagorder=Min('taggedsheet__order')).select_related('organisation').distinct()
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   426
    
250
7c3f54ce68af Merged Completion List back into Facettes
veltr
parents: 248
diff changeset
   427
    contents = [{ 'description': ds.description, 'title': ds.title, 'url': ds.url, 'score': int((MAX_TAG_ORDER - ds.tagorder)/2), 'id': ds.id, 'hda_id': ds.hda_id, 'organization': ds.organisation.name } for ds in datasheets]
248
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   428
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   429
    contents = sorted(contents, key=lambda e: -e['score'])
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   430
    
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
   431
    res = { 'label': label }
248
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   432
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   433
    if len(contents):
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   434
        res['contents'] = contents
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   435
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   436
    if len(sub):
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   437
        subcats = [subtree(st) for st in sub]
252
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
   438
        subcats = [sc for sc in subcats if len(sc.get('contents',[])) or len(sc.get('themes',[]))]
a46461f8a74f Separated Thesaurus and WP Categories
veltr
parents: 250
diff changeset
   439
        res['themes'] = subcats
248
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   440
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   441
    return res
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   442
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   443
def filltree(request):
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   444
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   445
    tree = request.GET.get('tree','{}')
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   446
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   447
    treeobj = json.loads(tree)
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   448
    
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   449
    res = subtree(treeobj)
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   450
        
91bc8521e3cb Ajout de la navigation par thesaurus
veltr
parents: 243
diff changeset
   451
    return HttpResponse(content=json.dumps(res), mimetype='application/json')