web/hdalab/views/ajax.py
author veltr
Thu, 12 Jul 2012 10:45:01 +0200
changeset 208 d4e3ea751966
parent 206 7070d3acc3d4
child 212 3e1193c6dce3
permissions -rw-r--r--
Wikipedia tree algorithm changes
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
'''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Created on Jan 31, 2012
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
@author: ymh
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
'''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
from django.conf import settings
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
     8
from django.core.cache import cache
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
     9
from django.db.models import Q, Count
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
from django.http import HttpResponse
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from hdabo.models import Tag, Datasheet, TaggedSheet
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
    12
from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
    13
from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    14
from hdalab.models.categories import WpCategory, WpCategoryInclusion, TagWpCategory
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
    15
from hdalab.utils import fix_cache_key
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
    16
import copy
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
import django.utils.simplejson as json
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
import hmac
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    19
import itertools
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    20
import uuid
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    21
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    22
def taginfo(request):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    23
    label = request.GET.get('label', None)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    24
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
    resobj = {'requested_label' : label}
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    resobj["content_count"] = Datasheet.objects.filter(taggedsheet__tag__label__iexact = label).distinct().count()
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    28
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
    29
    res = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = label).order_by('-dbpedia_uri')[0:1]
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
    if len(res) == 1:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
        restag = res.get()
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
        resobj["dbpedia_uri"] = restag.dbpedia_uri
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
        if resobj["dbpedia_uri"] is not None and restag.dbpedia_fields is not None:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
            dbfield = restag.dbpedia_fields
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
            resobj["abstract"] = dbfield.abstract
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
            resobj["dbpedia_label"] = dbfield.label
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
            resobj["thumbnail"] = dbfield.thumbnail
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    38
            transqs = DbpediaFieldsTranslation.objects.filter(master=dbfield, language_code=request.LANGUAGE_CODE)[0:1]
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    39
            if transqs:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    40
                trans = transqs.get()
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    41
                resobj['translated_abstract'] = trans.abstract
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    42
                resobj['translated_label'] = trans.label
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    43
            else:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    44
                resobj['translated_abstract'] = dbfield.abstract
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    45
                resobj['translated_label'] = dbfield.label
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    46
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    47
    #res = Tag.objects.filter(label__iexact = label).order_by('-wikipedia_url')[0:1]
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    48
    #if len(res) == 1:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    49
    #    resobj["wikipedia_url"] = res.get().wikipedia_url
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    50
    if 'translated_label' in resobj:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    51
        wikipedia_label = resobj['translated_label']
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    52
    else:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    53
        wikipedia_label = label
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    54
    wikipedia_label = wikipedia_label[0].capitalize() + wikipedia_label[1:]
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    55
    resobj["wikipedia_url"] = "http://%s.wikipedia.org/wiki/%s" % (request.LANGUAGE_CODE,wikipedia_label.replace(' ', '_'))
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
    56
# We don't use links at the moment, so I'll comment this line to speed up requests
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
    57
#    resobj["links"] = [{'subject':tl.subject.label, 'object':tl.object.label} for tl in TagLinks.objects.select_related().filter(Q(subject__label__iexact = label) | Q(object__label__iexact = label))]
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
    return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    62
def tagtranslation(request):
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    63
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    64
    labels = request.GET.get('labels',None)
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    65
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    66
    if not labels:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    67
        return HttpResponse(content=json.dumps({}), mimetype='application/json')
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    68
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    69
    labelslist = [lbl.strip() for lbl in labels.split(",")]
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    70
    masters = []
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    71
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    72
    for lbl in labelslist:
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
    73
        labelqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = lbl)[0:1]
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    74
        if len(labelqs) > 0:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    75
            tag = labelqs.get()
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    76
            if tag.dbpedia_fields:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    77
                masters.append(tag.dbpedia_fields)
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    78
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    79
    translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    80
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    81
    translations = dict([(t.master.label, t.label) for t in translationqs])
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    82
    
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
    83
    return HttpResponse(content=json.dumps(translations), mimetype='application/json')
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    84
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    85
def subcat(category, globtags, level, max_level ):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    86
    # recursive function used by cattree
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    87
    catlabel = category.label
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
    88
    tags = Tag.objects.filter(wp_categories__wp_category = category).distinct()
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    89
    taglabels = [k for k in dict([(t.label,t.label) for t in tags])]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    90
    resobj = {
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    91
          'category': category.label,
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    92
          'tags': [],
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    93
          'contents': []
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    94
          }
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    95
    for label in taglabels:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    96
        if label == catlabel:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    97
            globtags[label] = {'level': level, 'access': resobj }
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    98
        else:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
    99
            tag_in_list = {'label' : label, 'contents': []}
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   100
            resobj['tags'].append(tag_in_list)
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   101
            globtags[label] = {'level': (level + 1), 'access': tag_in_list }
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   102
            
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   103
    if level < max_level:
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   104
        subcats = WpCategory.objects.filter(parent_categories__parent_category = category)
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   105
        resobj['sub_categories'] = [subcat(subcats[i], globtags, level + 1, max_level ) for i in range(len(subcats))]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   106
    return resobj
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   107
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   108
def cleantags(category):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   109
    if category.has_key('contents') and len(category['contents']) == 0:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   110
        del category['contents']
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   111
    if category.has_key('contents'):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   112
        category['contents'] = sorted(category['contents'], key=lambda content: -content['score'])
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   113
    if category.has_key('tags'):
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   114
        category['tags'] = [cleantags(tag) for tag in category['tags'] if len(tag['contents'])]
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   115
        if len(category['tags']) == 0:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   116
            del category['tags']
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   117
        else:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   118
            category['tags'] = sorted(category['tags'], key=lambda tag: tag['label'])
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   119
    if category.has_key('sub_categories'):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   120
        sub_cats = []
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   121
        for sub_cat in category['sub_categories']:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   122
            cat = cleantags(sub_cat)
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   123
            if cat.has_key('tags') or cat.has_key('sub_categories') or cat.has_key('contents'):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   124
                sub_cats.append(cat)
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   125
        category['sub_categories'] = sorted(sub_cats, key=lambda cat: cat['category'])
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   126
        if len(category['sub_categories']) == 0:
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   127
            del category['sub_categories']
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   128
    return category
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   129
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   130
def cattree(request):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   131
    # Gets the category tree from a label
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   132
    ROOT_MAX_TAG_ORDER = 5
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   133
    MAX_TAG_ORDER = 5
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   134
    MAX_LEVEL = 3
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   135
    label = request.GET.get('label', None)
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   136
    lowerlabel = label.lower()
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   137
    globtags = {}
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   138
    resobj = None
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   139
    master_category = WpCategory.objects.filter(label__iexact=label)[0:1]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   140
    if len(master_category):
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   141
        resobj = subcat(master_category[0], globtags, 1, MAX_LEVEL )
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   142
    
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   143
#    tag_list = [k for k in globtags]
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   144
    
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   145
#    if len(tag_list):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   146
    contents = []
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   147
#    datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__in = tag_list, taggedsheet__order__lte = MAX_TAG_ORDER).distinct()
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   148
    datasheets = Datasheet.objects.filter(validated = True, taggedsheet__tag__label__iexact = label, taggedsheet__order__lte = ROOT_MAX_TAG_ORDER).distinct()
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   149
    for datasheet in datasheets:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   150
        # Calculating where we add the datasheet in the tree
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   151
        maintag = None
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   152
        maintagscore = -5
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   153
        dsscore = 0
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   154
        rootscore = 0
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   155
        for ts in TaggedSheet.objects.select_related('tag','datasheet').filter(datasheet__id=datasheet.id,order__lte=MAX_TAG_ORDER):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   156
            label = ts.tag.label
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   157
            if globtags.has_key(label):
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   158
                score = 3 * globtags[label]['level'] - ts.order
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   159
                if score > maintagscore:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   160
                    maintagscore = score
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   161
                    maintag = label
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   162
                    dsscore = (MAX_TAG_ORDER - ts.order)
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   163
                if label.lower() == lowerlabel:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   164
                    rootscore = (ROOT_MAX_TAG_ORDER - ts.order)
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   165
        if maintag is not None:
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   166
            globtags[maintag]['access']['contents'].append({'id': datasheet.id, 'title': datasheet.title, 'url': datasheet.url, 'score': (dsscore + rootscore)})
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   167
    cleantags(resobj)
204
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   168
                    
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   169
#        resobj['contents'] = [{'id': d.id, 'title': d.title, 'tags': [t.label for t in d.tags.filter(taggedsheet__order__lte=5)]} for d in datasheets]
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   170
    
0a1744477bc1 Added category tree Ajax API + Tables
veltr
parents: 175
diff changeset
   171
    return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   172
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   173
def sessioninfo(request):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   174
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   175
    data = json.loads(request.GET.get('data', "{}"))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   176
    write = False
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   177
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
    if 'sessionid' in request.GET:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   179
        request.session['sessionid'] = request.GET['sessionid']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
    if 'sessionkey' in request.GET:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   181
        request.session['sessionkey'] = request.GET['sessionkey']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   182
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   183
    if 'sessionid' in request.session:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   184
        sessionid = request.session['sessionid']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   185
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   186
        if HdaSession.objects.filter(sessionid=sessionid).count() == 1:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   187
            sessionkey = request.session.get('sessionkey',None)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   188
            hm = hmac.new(settings.SECRET_KEY, sessionid)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   189
            if hm.hexdigest() == sessionkey:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   190
                write = True            
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   191
        else:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   192
            del request.session['sessionid']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   193
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   194
    if 'sessionid' not in request.session:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   195
        sessionid = unicode(uuid.uuid1())
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   196
        HdaSession.objects.create(sessionid=sessionid, data=json.dumps({}))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   197
        write = True
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   198
        request.session['sessionid'] = sessionid
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   199
        request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest()
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   200
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   201
    if write and data:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   202
        HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   203
    else:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   204
        data = HdaSession.objects.get(sessionid=sessionid).data
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   205
        data = json.loads(data) if data else {}           
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   206
         
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   207
    resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid }
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   208
    if write:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   209
        resobj['sessionkey'] = request.session['sessionkey']
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   210
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   211
    return HttpResponse(content=json.dumps(resobj), mimetype='application/json')
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   212
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   213
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   214
def tagsearch(request):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   215
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   216
    q = request.GET.get('term',None)
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   217
    if q:
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   218
        lq = q.lower()
175
ebc92daef815 First try at autocomplete
veltr
parents: 172
diff changeset
   219
    qs = Tag.objects.filter(datasheet__validated=True).filter( Q(label__icontains = q ) | Q(dbpedia_fields__translations__label__icontains = q, dbpedia_fields__translations__language_code=request.LANGUAGE_CODE), ~Q(dbpedia_uri = None)) if q else Tag.objects.filter(~Q(dbpedia_uri = None))       
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   220
    qs = qs.annotate(nb=Count('datasheet')).order_by('-nb')[:20]
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   221
    
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   222
    qslist = list(qs)
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   223
    
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   224
    transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=request.LANGUAGE_CODE).select_related("master")
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   225
    
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   226
    translations = dict([(tr.master.tag_id, tr.label) for tr in transqs])
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   227
    
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   228
    res = []
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   229
    
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   230
    for t in qslist:
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   231
        resobj = {'value':t.label,'nb':t.nb}
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   232
        if t.id in translations:
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   233
            resobj['label'] = translations[t.id]
150
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   234
        else:
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   235
            resobj['label'] = t.label
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   236
        if q is None or resobj['label'].lower().find(lq) != -1:
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   237
            res.append(resobj)
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   238
    
b0a34ce6fdcf Added multilingual possibilities to tagsearch view
veltr
parents: 135
diff changeset
   239
    return HttpResponse(content=json.dumps(res), mimetype='application/json')
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   240
205
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   241
def catsearch(request):
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   242
    
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   243
    q = request.GET.get('term',None)
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   244
    
208
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   245
    # On ne récupère que les catégories qui sont également des tags
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   246
    
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   247
    qs = Tag.objects.filter(Q(label__icontains = ' ' + q ) | Q(label__istartswith = q))
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   248
    
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   249
    labels = [tag.label for tag in qs]
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   250
    
d4e3ea751966 Wikipedia tree algorithm changes
veltr
parents: 206
diff changeset
   251
    qs = WpCategory.objects.filter(label__in = labels)
205
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   252
    
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   253
    res = [{'value':t.label} for t in qs]
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   254
    
8ff4fd1a6e9c Added categories page
veltr
parents: 204
diff changeset
   255
    return HttpResponse(content=json.dumps(res), mimetype='application/json')
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   256
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   257
def filter(request):
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   258
    
158
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   259
    periode = request.GET.get('period',None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   260
    label = request.GET.get('label', None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   261
    country = request.GET.get('country', None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   262
    contentlist = request.GET.get('contentlist', None)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   263
    max_tag_order = request.GET.get('mto', 12)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   264
    content_count = request.GET.get('contentcount', 12)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   265
    tag_count = request.GET.get('tagcount', 30)
f105b62dce5d fix cache key
ymh <ymh.work@gmail.com>
parents: 154
diff changeset
   266
    
163
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   267
    key_parts = ("filter",request.LANGUAGE_CODE,periode,label,country,contentlist,max_tag_order,content_count,tag_count)
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   268
    key_parts = [unicode(p).encode("utf-8") for p in key_parts]
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   269
    
e4ff12317fa3 correct accent in keys
ymh <ymh.work@gmail.com>
parents: 162
diff changeset
   270
    cache_key = fix_cache_key("-".join(key_parts))
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   271
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   272
    outputstr = cache.get(cache_key)
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   273
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   274
    if outputstr is None:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   275
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   276
        matchtagids = []
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   277
        
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   278
        tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']).filter(~Q(dbpedia_uri = None))
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   279
        countryqs = Country.objects
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   280
        discqs = Tag.objects.filter(~Q(dbpedia_uri = None), category__label = u'Discipline artistique').select_related('dbpedia_fields')
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   281
        yearqs = TagYears.objects
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   282
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   283
        contentqs = Datasheet.objects.filter(validated=True)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   284
        labeltranslations = []
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   285
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   286
        if label or periode or country or contentlist :
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   287
            matchtagqslist = []
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   288
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   289
            if periode:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   290
                years = periode.split(",")
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   291
                start_year = int(years[0])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   292
                end_year = int(years[0:2][-1])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   293
                delta = max(1, (end_year-start_year)/2)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   294
                minstart = start_year - delta
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   295
                maxend = end_year + delta
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   296
                matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   297
                                                years__end_year__gte = start_year, 
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   298
                                                years__start_year__lte = end_year,
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   299
                                                years__end_year__lte = maxend,
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   300
                                                years__start_year__gte = minstart,
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   301
                                                )
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   302
                matchtagqslist.append(matchtagqs)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   303
                
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   304
            if label:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   305
                for txtlbl in label.split(","):
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   306
                    matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = txtlbl.strip())
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   307
                    matchtagqslist.append(matchtagqs)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   308
                
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   309
            if country:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   310
                for country_uri in country.split(","):
165
b9b93ff09ebe Remove unsemantized tags
ymh <ymh.work@gmail.com>
parents: 163
diff changeset
   311
                    matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),locatedin__country__dbpedia_uri = country_uri)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   312
                    matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids]
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   313
                    matchtagqslist.append(matchtagqs)
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   314
            if contentlist:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   315
                contentqs = contentqs.filter(id__in = contentlist.split(","))
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   316
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   317
            tagcond = None
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   318
            tagcondid = None
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   319
            for matchtagqs in matchtagqslist:
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   320
                newcond = Q(id__in = TaggedSheet.objects.filter(tag__in = copy.deepcopy(matchtagqs), order__lte = max_tag_order).values('datasheet_id'))
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   321
                newcondid = Q(id__in = matchtagqs)
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   322
                tagcond = newcond if tagcond is None else (tagcond & newcond)
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   323
                tagcondid = newcondid if tagcondid is None else (tagcondid | newcondid)
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   324
            
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   325
            contentqs = contentqs.filter(tagcond).distinct()
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   326
            matchtagidsqs = list(Tag.objects.select_related("dbpedia_fields").only("id").filter(tagcondid))
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   327
            matchtagids = [t.id for t in matchtagidsqs]            
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   328
            masters = [t.dbpedia_fields for t in matchtagidsqs if t.dbpedia_fields is not None]
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   329
            
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   330
            translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)    
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   331
            labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs]
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   332
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   333
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   334
            tagqs = tagqs.filter(datasheet__in = contentqs)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   335
            countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   336
            discqs = discqs.filter(datasheet__in = contentqs)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   337
            yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs)
122
fde8335a037c Added Geographic Inclusion in the Django version (server only)
veltr
parents: 119
diff changeset
   338
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   339
        if contentlist is None:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   340
            contentqs.order_by('?')
172
8f47c67c6d28 Optimize query, allow unlimited facette parameters in query.
ymh <ymh.work@gmail.com>
parents: 165
diff changeset
   341
                        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   342
        cont_count = contentqs.count()
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   343
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   344
        contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   345
        contentids = contenus.keys()
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   346
        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   347
        qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   348
        for dse in qs:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   349
            contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   350
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   351
        qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order'))
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   352
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   353
        transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   354
        translations = dict([(trans.master_id,trans.label) for trans in transqs])
122
fde8335a037c Added Geographic Inclusion in the Django version (server only)
veltr
parents: 119
diff changeset
   355
        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   356
        for ts in qs:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   357
            match_tag = ts.tag.id in matchtagids
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   358
            contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label})
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   359
            
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   360
            if match_tag:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   361
                contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   362
            
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   363
        if contentlist is None:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   364
            contenus = sorted(contenus.values(),key=lambda e: -e['score'])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   365
        else:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   366
            contenus = contenus.values()
152
e2ceeb903de6 improve requests
ymh <ymh.work@gmail.com>
parents: 150
diff changeset
   367
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   368
        #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   369
        tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   370
        #.select_related('dbpedia_fields')
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   371
        # hack to add only necessary fields in the group by
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   372
        # contournement bug https://code.djangoproject.com/ticket/17144
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   373
        tagqs.query.clear_select_fields()
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   374
        tagqs.query.add_fields(['id','label'], False)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   375
        tagqs.query.set_group_by()
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   376
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   377
        tagqslist = list(tagqs)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   378
        
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   379
        dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)])
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   380
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   381
        transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   382
        translations = dict([(trans.master_id,trans.label) for trans in transqs])
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   383
    
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   384
        tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   385
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   386
        countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet'))
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   387
        countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   388
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   389
        discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10])
127
8642f1fb6499 Integration scripts django et html
veltr
parents: 122
diff changeset
   390
            
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   391
        transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   392
        translations = dict([(trans.master_id,trans.label) for trans in transqs])
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   393
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   394
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   395
        disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   396
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   397
        years = {}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   398
        yearqs = yearqs.annotate(nb=Count('tag__taggedsheet'))
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   399
        for ty in yearqs:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   400
            for year in range(ty.start_year, ty.end_year):
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   401
                years[year] = ty.nb + (years[year] if year in years else 0)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   402
                
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   403
        yearchange = []
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   404
        for year in sorted(years.keys()):
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   405
            score = years[year]
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   406
            if year < 2011:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   407
                if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score):
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   408
                    yearchange.append({'year': year, 'score': score})
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   409
                if year+1 not in years and year != -1 and score != 0:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   410
                    yearchange.append({'year': year+1, 'score': 0})
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   411
    
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   412
        tag_translations = {}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   413
        for t in itertools.chain(labeltranslations,disciplines,tags):
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 134
diff changeset
   414
            tag_translations[t['label']] = t['translated_label']
154
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   415
        for c in contenus:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   416
            for t in c['tags']:
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   417
                tag_translations[t['label']] = t['translated_label']
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   418
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   419
        output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations}
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   420
        outputstr = json.dumps(output)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   421
        cache.set(cache_key, outputstr)
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   422
        
8527c5a3ddb7 add cache management to filter
ymh <ymh.work@gmail.com>
parents: 152
diff changeset
   423
    return HttpResponse(content=outputstr, mimetype='application/json')