1 # -*- coding: utf-8 -*-

     2 '''

     3 Created on Jan 31, 2012

     4 

     5 @author: ymh

     6 '''

     7 from django.conf import settings

     8 from django.core.cache import cache

     9 from django.db.models import Q, Count

    10 from django.http import HttpResponse

    11 from hdabo.models import Tag, Datasheet, TaggedSheet

    12 from hdalab.models import HdaSession, Country, TagYears, DatasheetExtras

    13 from hdalab.models.dataviz import DbpediaFieldsTranslation, DbpediaFields

    14 from hdalab.utils import fix_cache_key

    15 import copy

    16 import django.utils.simplejson as json

    17 import hmac

    18 import itertools

    19 import uuid

    20 

    21 def taginfo(request):

    22     label = request.GET.get('label', None)

    23     

    24     resobj = {'requested_label' : label}

    25         

    26     resobj["content_count"] = Datasheet.objects.filter(taggedsheet__tag__label__iexact = label).distinct().count()

    27 

    28     res = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = label).order_by('-dbpedia_uri')[0:1]

    29     if len(res) == 1:

    30         restag = res.get()

    31         resobj["dbpedia_uri"] = restag.dbpedia_uri

    32         if resobj["dbpedia_uri"] is not None and restag.dbpedia_fields is not None:

    33             dbfield = restag.dbpedia_fields

    34             resobj["abstract"] = dbfield.abstract

    35             resobj["dbpedia_label"] = dbfield.label

    36             resobj["thumbnail"] = dbfield.thumbnail

    37             transqs = DbpediaFieldsTranslation.objects.filter(master=dbfield, language_code=request.LANGUAGE_CODE)[0:1]

    38             if transqs:

    39                 trans = transqs.get()

    40                 resobj['translated_abstract'] = trans.abstract

    41                 resobj['translated_label'] = trans.label

    42             else:

    43                 resobj['translated_abstract'] = dbfield.abstract

    44                 resobj['translated_label'] = dbfield.label

    45 

    46     #res = Tag.objects.filter(label__iexact = label).order_by('-wikipedia_url')[0:1]

    47     #if len(res) == 1:

    48     #    resobj["wikipedia_url"] = res.get().wikipedia_url

    49     if 'translated_label' in resobj:

    50         wikipedia_label = resobj['translated_label']

    51     else:

    52         wikipedia_label = label

    53     wikipedia_label = wikipedia_label[0].capitalize() + wikipedia_label[1:]

    54     resobj["wikipedia_url"] = "http://%s.wikipedia.org/wiki/%s" % (request.LANGUAGE_CODE,wikipedia_label.replace(' ', '_'))

    55 # We don't use links at the moment, so I'll comment this line to speed up requests

    56 #    resobj["links"] = [{'subject':tl.subject.label, 'object':tl.object.label} for tl in TagLinks.objects.select_related().filter(Q(subject__label__iexact = label) | Q(object__label__iexact = label))]

    57 

    58     return HttpResponse(content=json.dumps(resobj), mimetype='application/json')

    59 

    60 

    61 def tagtranslation(request):

    62     

    63     labels = request.GET.get('labels',None)

    64 

    65     if not labels:

    66         return HttpResponse(content=json.dumps({}), mimetype='application/json')

    67     

    68     labelslist = [lbl.strip() for lbl in labels.split(",")]

    69     masters = []

    70     

    71     for lbl in labelslist:

    72         labelqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = lbl)[0:1]

    73         if len(labelqs) > 0:

    74             tag = labelqs.get()

    75             if tag.dbpedia_fields:

    76                 masters.append(tag.dbpedia_fields)

    77     

    78     translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)

    79     

    80     translations = dict([(t.master.label, t.label) for t in translationqs])

    81     

    82     return HttpResponse(content=json.dumps(translations), mimetype='application/json')

    83     

    84 

    85 

    86 def sessioninfo(request):

    87     

    88     data = json.loads(request.GET.get('data', "{}"))

    89     write = False

    90     

    91     if 'sessionid' in request.GET:

    92         request.session['sessionid'] = request.GET['sessionid']

    93     if 'sessionkey' in request.GET:

    94         request.session['sessionkey'] = request.GET['sessionkey']

    95         

    96     if 'sessionid' in request.session:

    97         sessionid = request.session['sessionid']

    98         

    99         if HdaSession.objects.filter(sessionid=sessionid).count() == 1:

   100             sessionkey = request.session.get('sessionkey',None)

   101             hm = hmac.new(settings.SECRET_KEY, sessionid)

   102             if hm.hexdigest() == sessionkey:

   103                 write = True            

   104         else:

   105             del request.session['sessionid']

   106         

   107     if 'sessionid' not in request.session:

   108         sessionid = unicode(uuid.uuid1())

   109         HdaSession.objects.create(sessionid=sessionid, data=json.dumps({}))

   110         write = True

   111         request.session['sessionid'] = sessionid

   112         request.session['sessionkey'] = hmac.new(settings.SECRET_KEY, sessionid).hexdigest()

   113         

   114     if write and data:

   115         HdaSession.objects.filter(sessionid=sessionid).update(data=json.dumps(data))

   116     else:

   117         data = HdaSession.objects.get(sessionid=sessionid).data

   118         data = json.loads(data) if data else {}           

   119          

   120     resobj = {'data': data, "write_allowed" : write, "sessionid": sessionid }

   121     if write:

   122         resobj['sessionkey'] = request.session['sessionkey']

   123         

   124     return HttpResponse(content=json.dumps(resobj), mimetype='application/json')

   125 

   126 

   127 def tagsearch(request):

   128     

   129     q = request.GET.get('term',None)

   130     if q:

   131         lq = q.lower()

   132     qs = Tag.objects.filter(datasheet__validated=True).filter( Q(label__icontains = q ) | Q(dbpedia_fields__translations__label__icontains = q, dbpedia_fields__translations__language_code=request.LANGUAGE_CODE), ~Q(dbpedia_uri = None)) if q else Tag.objects.filter(~Q(dbpedia_uri = None))       

   133     qs = qs.annotate(nb=Count('datasheet')).order_by('-nb')[:20]

   134     

   135     qslist = list(qs)

   136     

   137     transqs = DbpediaFieldsTranslation.objects.filter(master__tag__in = qslist, language_code=request.LANGUAGE_CODE).select_related("master")

   138     

   139     translations = dict([(tr.master.tag_id, tr.label) for tr in transqs])

   140     

   141     res = []

   142     

   143     for t in qslist:

   144         resobj = {'value':t.label,'nb':t.nb}

   145         if t.id in translations:

   146             resobj['label'] = translations[t.id]

   147         else:

   148             resobj['label'] = t.label

   149         if q is None or resobj['label'].lower().find(lq) != -1:

   150             res.append(resobj)

   151     

   152     return HttpResponse(content=json.dumps(res), mimetype='application/json')

   153 

   154 

   155 def filter(request):

   156     

   157     periode = request.GET.get('period',None)

   158     label = request.GET.get('label', None)

   159     country = request.GET.get('country', None)

   160     contentlist = request.GET.get('contentlist', None)

   161     max_tag_order = request.GET.get('mto', 12)

   162     content_count = request.GET.get('contentcount', 12)

   163     tag_count = request.GET.get('tagcount', 30)

   164     

   165     key_parts = ("filter",request.LANGUAGE_CODE,periode,label,country,contentlist,max_tag_order,content_count,tag_count)

   166     key_parts = [unicode(p).encode("utf-8") for p in key_parts]

   167     

   168     cache_key = fix_cache_key("-".join(key_parts))

   169     

   170     outputstr = cache.get(cache_key)

   171     

   172     if outputstr is None:

   173         

   174         matchtagids = []

   175         

   176         tagqs = Tag.objects.exclude(category__label__in = ['Datation', 'Localisation', 'Discipline artistique']).filter(~Q(dbpedia_uri = None))

   177         countryqs = Country.objects

   178         discqs = Tag.objects.filter(~Q(dbpedia_uri = None), category__label = u'Discipline artistique').select_related('dbpedia_fields')

   179         yearqs = TagYears.objects

   180         

   181         contentqs = Datasheet.objects.filter(validated=True)

   182         labeltranslations = []

   183     

   184         if label or periode or country or contentlist :

   185             matchtagqslist = []

   186             

   187             if periode:

   188                 years = periode.split(",")

   189                 start_year = int(years[0])

   190                 end_year = int(years[0:2][-1])

   191                 delta = max(1, (end_year-start_year)/2)

   192                 minstart = start_year - delta

   193                 maxend = end_year + delta

   194                 matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),

   195                                                 years__end_year__gte = start_year, 

   196                                                 years__start_year__lte = end_year,

   197                                                 years__end_year__lte = maxend,

   198                                                 years__start_year__gte = minstart,

   199                                                 )

   200                 matchtagqslist.append(matchtagqs)

   201                 

   202             if label:

   203                 for txtlbl in label.split(","):

   204                     matchtagqs = Tag.objects.select_related('dbpedia_fields').filter(~Q(dbpedia_uri = None), label__iexact = txtlbl.strip())

   205                     matchtagqslist.append(matchtagqs)

   206                 

   207             if country:

   208                 for country_uri in country.split(","):

   209                     matchtagqs = Tag.objects.filter(~Q(dbpedia_uri = None),locatedin__country__dbpedia_uri = country_uri)

   210                     matchtagids += [t.id for t in matchtagqs if t.id not in matchtagids]

   211                     matchtagqslist.append(matchtagqs)

   212             if contentlist:

   213                 contentqs = contentqs.filter(id__in = contentlist.split(","))

   214 

   215             tagcond = None

   216             tagcondid = None

   217             for matchtagqs in matchtagqslist:

   218                 newcond = Q(id__in = TaggedSheet.objects.filter(tag__in = copy.deepcopy(matchtagqs), order__lte = max_tag_order).values('datasheet_id'))

   219                 newcondid = Q(id__in = matchtagqs)

   220                 tagcond = newcond if tagcond is None else (tagcond & newcond)

   221                 tagcondid = newcondid if tagcondid is None else (tagcondid | newcondid)

   222             

   223             contentqs = contentqs.filter(tagcond).distinct()

   224             matchtagidsqs = list(Tag.objects.select_related("dbpedia_fields").only("id").filter(tagcondid))

   225             matchtagids = [t.id for t in matchtagidsqs]            

   226             masters = [t.dbpedia_fields for t in matchtagidsqs if t.dbpedia_fields is not None]

   227             

   228             translationqs = DbpediaFieldsTranslation.objects.select_related("master", "master__tag").filter(master__in = masters, language_code=request.LANGUAGE_CODE)    

   229             labeltranslations = [{'label':t.master.label, 'translated_label':t.label} for t in translationqs]

   230 

   231             

   232             tagqs = tagqs.filter(datasheet__in = contentqs)

   233             countryqs = countryqs.filter(includes__tag__taggedsheet__datasheet__in = contentqs)

   234             discqs = discqs.filter(datasheet__in = contentqs)

   235             yearqs = yearqs.filter(tag__taggedsheet__datasheet__in = contentqs)

   236             

   237         if contentlist is None:

   238             contentqs.order_by('?')

   239                         

   240         cont_count = contentqs.count()

   241         

   242         contenus = dict([(content.id, {'score' : 0, 'tags' : [], 'id':content.id, 'title': content.title, 'description': content.description, 'url': content.url}) for content in contentqs[0:content_count]])

   243         contentids = contenus.keys()

   244         

   245         qs = DatasheetExtras.objects.select_related('insee').filter(datasheet__in = contentids)

   246         for dse in qs:

   247             contenus[dse.datasheet_id]['coords'] = {'city_name': dse.insee.city_name, 'latitude': dse.insee.latitude, 'longitude': dse.insee.longitude}

   248         

   249         qs = list(TaggedSheet.objects.select_related('tag', 'tag__dbpedia_fields').filter(datasheet__in = contentids, order__lte = max_tag_order).order_by('order'))

   250         

   251         transqs = DbpediaFieldsTranslation.objects.filter(master__in = [ts.tag.dbpedia_fields for ts in qs], language_code = request.LANGUAGE_CODE)

   252         translations = dict([(trans.master_id,trans.label) for trans in transqs])

   253         

   254         for ts in qs:

   255             match_tag = ts.tag.id in matchtagids

   256             contenus[ts.datasheet_id]['tags'].append({'id': ts.tag.id, 'label':ts.tag.label, 'order':ts.order, 'match': match_tag , 'translated_label': translations.get(ts.tag.dbpedia_fields.id, ts.tag.label) if ts.tag.dbpedia_fields is not None else ts.tag.label})

   257             

   258             if match_tag:

   259                 contenus[ts.datasheet_id]['score'] += 2*max_tag_order - ts.order

   260             

   261         if contentlist is None:

   262             contenus = sorted(contenus.values(),key=lambda e: -e['score'])

   263         else:

   264             contenus = contenus.values()

   265     

   266         #tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb')[:tag_count]

   267         tagqs = tagqs.annotate(nb=Count('datasheet')).order_by('-nb').only('id','label')[:tag_count]

   268         #.select_related('dbpedia_fields')

   269         # hack to add only necessary fields in the group by

   270         # contournement bug https://code.djangoproject.com/ticket/17144

   271         tagqs.query.clear_select_fields()

   272         tagqs.query.add_fields(['id','label'], False)

   273         tagqs.query.set_group_by()

   274     

   275         tagqslist = list(tagqs)

   276         

   277         dbpediafields = dict([(df.tag_id, df) for df in DbpediaFields.objects.filter(tag__in = tagqslist)])

   278     

   279         transqs = DbpediaFieldsTranslation.objects.filter(master__in = dbpediafields.values(), language_code = request.LANGUAGE_CODE)

   280         translations = dict([(trans.master_id,trans.label) for trans in transqs])

   281     

   282         tags = [{'id': tag.id, 'label': tag.label, 'score': tag.nb, 'translated_label': translations.get(dbpediafields[tag.id].id, tag.label) if tag.id in dbpediafields else tag.label} for tag in tagqslist]

   283     

   284         countryqs = countryqs.annotate(nb=Count('includes__tag__taggedsheet'))

   285         countries = dict([(country.dbpedia_uri, country.nb) for country in countryqs])

   286     

   287         discqslist = list(discqs.annotate(nb=Count('taggedsheet')).order_by('-nb')[:10])

   288             

   289         transqs = DbpediaFieldsTranslation.objects.filter(master__in = [tag.dbpedia_fields for tag in discqslist], language_code = request.LANGUAGE_CODE)

   290         translations = dict([(trans.master_id,trans.label) for trans in transqs])

   291     

   292         

   293         disciplines = [{'label':tag.label,'score':tag.nb, 'translated_label': translations.get(tag.dbpedia_fields.id, tag.label) if tag.dbpedia_fields is not None else tag.label} for tag in discqslist]

   294         

   295         years = {}

   296         yearqs = yearqs.annotate(nb=Count('tag__taggedsheet'))

   297         for ty in yearqs:

   298             for year in range(ty.start_year, ty.end_year):

   299                 years[year] = ty.nb + (years[year] if year in years else 0)

   300                 

   301         yearchange = []

   302         for year in sorted(years.keys()):

   303             score = years[year]

   304             if year < 2011:

   305                 if (year-1 not in years and score != 0) or (year-1 in years and years[year-1] != score):

   306                     yearchange.append({'year': year, 'score': score})

   307                 if year+1 not in years and year != -1 and score != 0:

   308                     yearchange.append({'year': year+1, 'score': 0})

   309     

   310         tag_translations = {}

   311         for t in itertools.chain(labeltranslations,disciplines,tags):

   312             tag_translations[t['label']] = t['translated_label']

   313         for c in contenus:

   314             for t in c['tags']:

   315                 tag_translations[t['label']] = t['translated_label']

   316         

   317         output = {'count': cont_count, 'contents': contenus, 'tags':tags, 'sparkline':yearchange, 'countries':countries, 'disciplines':disciplines, 'tagtranslations': tag_translations}

   318         outputstr = json.dumps(output)

   319         cache.set(cache_key, outputstr)

   320         

   321     return HttpResponse(content=outputstr, mimetype='application/json')