web/hdalab/management/commands/query_dbpedia.py
author ymh <ymh.work@gmail.com>
Sat, 08 Sep 2012 03:49:10 +0200
changeset 240 c8627191f2d7
parent 135 dd6578e36a57
permissions -rw-r--r--
add indication that the labels has been translated, and use sparql requests instead of full n3 download
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     2
'''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     3
Created on Jan 30, 2012
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     4
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     5
@author: ymh
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     6
'''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     7
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     8
from django.core.management.base import NoArgsCommand
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
     9
from django.core.management.color import no_style
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    10
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    11
from optparse import make_option
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
    12
from django.conf import settings
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    13
from django.db.models import Count
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    14
from django.db import transaction
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    15
from hdabo.models import Tag
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    16
from hdalab.models import DbpediaFields, TagLinks
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    17
from hdabo.utils import show_progress
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    18
from rdflib.graph import Graph
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    19
from rdflib import URIRef
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
    20
import traceback
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
    21
import sys
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
    22
from hdalab.models.dataviz import DbpediaFieldsTranslation
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
    23
from django import db
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
    24
from SPARQLWrapper import SPARQLWrapper, RDF
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    25
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    26
class Command(NoArgsCommand):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    27
    '''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    28
    query and update wikipedia for tag title.
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    29
    '''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    30
    options = ''
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    31
    help = """query and update wikipedia for tag title."""
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    32
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    33
    option_list = NoArgsCommand.option_list + (
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    34
        make_option('--all',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    35
            action='store_true',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    36
            dest='all',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    37
            default=False,
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    38
            help='force all tags to be updated, not only those not yet processed'),
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    39
        make_option('--force',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    40
            action='store_true',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    41
            dest='force',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    42
            default=False,
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    43
            help='ask no questions'),
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    44
        make_option('--random',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    45
            action='store_true',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    46
            dest='random',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    47
            default=False,
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    48
            help='randomize query on tags'),
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    49
        make_option('--limit',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    50
            action='store',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    51
            type='int',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    52
            dest='limit',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    53
            default= -1,
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    54
            help='number of tag to process'),
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    55
        make_option('--start',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    56
            action='store',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    57
            type='int',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    58
            dest='start',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    59
            default=0,
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    60
            help='number of tag to ignore'),
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    61
        make_option('--tag',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    62
            action='append',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    63
            dest='tags',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    64
            type='string',
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    65
            default=[],
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    66
            help='the tag to query'),
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    67
    )
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    68
    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    69
    def handle_noargs(self, **options):
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
    70
                
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    71
        self.style = no_style()
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    72
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    73
        self.interactive = options.get('interactive', True)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    74
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    75
        self.verbosity = int(options.get('verbosity', '1'))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    76
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    77
        self.force = options.get('force', False)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    78
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    79
        self.limit = options.get("limit", -1)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    80
        self.start = options.get("start", 0)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    81
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    82
        self.random = options.get('random', False)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    83
                        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    84
        if self.verbosity > 2:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    85
            print "option passed : " + repr(options)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    86
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    87
        self.tag_list = options.get("tags", []);
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    88
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    89
        queryset = Tag.objects.exclude(dbpedia_uri= None)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    90
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    91
                
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    92
        if self.tag_list:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    93
            queryset = queryset.filter(label__in=self.tag_list)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    94
        elif not options.get('all',False):            
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    95
            queryset = queryset.annotate(dbfc=Count('dbpedia_fields')).filter(dbfc = 0)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    96
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    97
        if self.random:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    98
            queryset = queryset.order_by("?")
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
    99
        else:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   100
            queryset = queryset.order_by("label")
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   101
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   102
        if self.limit >= 0:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   103
            queryset = queryset[self.start:self.limit]
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   104
        elif self.start > 0:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   105
            queryset = queryset[self.start:]
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   106
        
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   107
        if self.verbosity > 2 :
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   108
            print "Tag Query is %s" % (queryset.query)
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   109
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   110
        count = queryset.count()
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   111
        
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   112
        if count == 0:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   113
            print "No tag to query : exit."
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   114
            return
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   115
            
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   116
            
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   117
        if not self.force and self.interactive:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   118
            confirm = raw_input("You have requested to query and replace the dbpedia information for %d tags.\n Are you sure you want to do this? \nType 'yes' to continue, or 'no' to cancel: " % (count))
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   119
        else:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   120
            confirm = 'yes'
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   121
            
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   122
        if confirm != "yes":
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   123
            print "dbpedia query cancelled"
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   124
            return
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   125
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   126
        endpoint = SPARQLWrapper("http://dbpedia.org/sparql", returnFormat=RDF)
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   127
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   128
        writer = None
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   129
        for i,tag in enumerate(queryset):
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   130
            writer = show_progress(i+1, count, tag.label, 50, writer)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   131
            db.reset_queries()
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   132
            
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   133
            #abstract query
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   134
            #"select ?y
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   135
            # where {<%s>  <http://dbpedia.org/ontology/abstract> ?y}" % (tag.dbpedia_uri)
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   136
            
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   137
            #rdf_uri = re.sub('\/resource\/', "/data/", tag.dbpedia_uri) + ".n3"            
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   138
            #g = Graph()
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   139
            try :
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   140
                abstracts = {}
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   141
                labels = {}
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   142
                thumbnail = None
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   143
                with transaction.commit_on_success():
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   144
                    endpoint.setQuery("select distinct ?y where {<%s>  <http://dbpedia.org/ontology/abstract> ?y}" % (tag.dbpedia_uri))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   145
                    res_abstracts = endpoint.queryAndConvert()
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   146
                    for _,_,o in res_abstracts.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   147
                        abstracts[o.language] = (unicode(o), True)
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   148
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   149
                    endpoint.setQuery("select distinct ?y where {<%s>  <http://www.w3.org/2000/01/rdf-schema#label> ?y}" % (tag.dbpedia_uri))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   150
                    res_labels = endpoint.queryAndConvert()
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   151
                    for _,_,o in res_labels.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   152
                        labels[o.language] = (unicode(o), True)
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   153
                            
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   154
                    endpoint.setQuery("select distinct ?y where {<%s>  <http://dbpedia.org/ontology/thumbnail> ?y} limit 1" % (tag.dbpedia_uri))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   155
                    res_thumbnails = endpoint.queryAndConvert()
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   156
                    for _,_,o in res_thumbnails.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   157
                        thumbnail = unicode(o)
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   158
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   159
                    endpoint.setQuery('select distinct ?y where { <%s> ?p ?y . FILTER regex(?y, "^http://dbpedia.org/resource")}' % (tag.dbpedia_uri))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   160
                    res_links = endpoint.queryAndConvert()
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   161
                    for _,_,o in res_links.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   162
                        tagqs = Tag.objects.filter(dbpedia_uri=unicode(o))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   163
                        if tagqs:
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   164
                            TagLinks.objects.get_or_create(subject=tag, object=tagqs[0])                        
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   165
                    
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   166
                    ref_label_lang, (ref_label, _) = ('fr',labels['fr']) if 'fr' in labels else ('en',labels['en']) if 'en' in labels else labels.items()[0] if len(labels) > 0 else ('fr',(tag.label, True))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   167
                    ref_abstract_lang, (ref_abstract, _) = ('fr',abstracts['fr']) if 'fr' in abstracts else ('en',abstracts['en']) if 'en' in abstracts else abstracts.items()[0] if len(abstracts) > 0 else ('fr',(None, 'True'))
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   168
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   169
                    for lang in settings.LANGUAGES:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   170
                        if lang[0] not in labels:
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   171
                            labels[lang[0]]= (ref_label, False)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   172
                        if lang[0] not in abstracts:
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   173
                            abstracts[lang[0]] = (ref_abstract, False)
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   174
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   175
                    dbfield , created = DbpediaFields.objects.get_or_create(dbpedia_uri=tag.dbpedia_uri,tag=tag, defaults={'abstract':ref_abstract, 'thumbnail':thumbnail, 'label':ref_label}) #@UndefinedVariable                    
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   176
                    if not created:
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   177
                        dbfield.abstract = ref_abstract
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   178
                        dbfield.thumbnail = thumbnail
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   179
                        dbfield.label = ref_label
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   180
                        dbfield.save()
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   181
                        DbpediaFieldsTranslation.objects.filter(master=dbfield).delete()
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   182
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   183
                    consolidated_trans = {}
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   184
                    for lang,label in labels.iteritems():
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   185
                        consolidated_trans[lang] = [label,(ref_abstract, lang==ref_abstract_lang)]
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   186
                    for lang,abstract in abstracts.iteritems():
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   187
                        if lang in consolidated_trans:
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   188
                            consolidated_trans[lang][1] = abstract
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   189
                        else:
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   190
                            consolidated_trans[lang] = [(ref_label, lang==ref_label_lang), abstract] 
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   191
                         
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   192
                    for lang, trans in consolidated_trans.iteritems():
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   193
                        label, abstract = tuple(trans)
240
c8627191f2d7 add indication that the labels has been translated, and use sparql requests instead of full n3 download
ymh <ymh.work@gmail.com>
parents: 135
diff changeset
   194
                        DbpediaFieldsTranslation.objects.create(master=dbfield, language_code=lang, label=label[0], is_label_translated=label[1], abstract=abstract[0], is_abstract_translated=abstract[1])
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   195
                    
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   196
                    
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   197
            except Exception as e:
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   198
                print "\nError processing resource %s : %s" %(rdf_uri,unicode(e))
135
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   199
                traceback.print_exception(type(e), e, sys.exc_info()[2])
dd6578e36a57 translate interface
ymh <ymh.work@gmail.com>
parents: 119
diff changeset
   200
                
119
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   201
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   202
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   203
            
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   204
            
e3ebe3545f72 first implementation of django version.
ymh <ymh.work@gmail.com>
parents:
diff changeset
   205