1 # -*- coding: utf-8 -*- |
1 # -*- coding: utf-8 -*- |
|
2 |
|
3 import logging |
|
4 from optparse import make_option |
2 |
5 |
3 from django.core.management import BaseCommand |
6 from django.core.management import BaseCommand |
4 from django.db import reset_queries, transaction |
7 from django.db import reset_queries, transaction |
5 from optparse import make_option |
8 from rdflib import BNode, URIRef |
|
9 from rdflib.plugins.sparql import prepareQuery |
|
10 |
|
11 from p4l.management.constants import get_empty_graph, IIEP |
6 from p4l.models import Record, Language |
12 from p4l.models import Record, Language |
7 from p4l.utils import show_progress |
13 from p4l.utils import show_progress |
8 from rdflib import Graph, Namespace, BNode, URIRef |
|
9 from rdflib.plugins.sparql import prepareQuery |
|
10 import logging |
|
11 import xml.etree.cElementTree as ET |
14 import xml.etree.cElementTree as ET |
12 |
15 |
|
16 |
13 logger = logging.getLogger(__name__) |
17 logger = logging.getLogger(__name__) |
14 |
18 |
15 |
|
16 RDF = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#") |
|
17 RDFS = Namespace("http://www.w3.org/2000/01/rdf-schema#") |
|
18 DCT = Namespace("http://purl.org/dc/terms/") |
|
19 IIEP = Namespace("http://www.iiep.unesco.org/plan4learning/model.owl#") |
|
20 UNESCO = Namespace("http://www.iiep.unesco.org/Ontology/") |
|
21 |
19 |
22 DEFAULT_LANGUAGE_URI = "http://psi.oasis-open.org/iso/639/#eng" |
20 DEFAULT_LANGUAGE_URI = "http://psi.oasis-open.org/iso/639/#eng" |
23 |
21 |
24 DEFAULT_LANGUAGE_QUERY = """SELECT ( COALESCE(?lang, ?other_lang) as ?main_lang) WHERE { |
22 DEFAULT_LANGUAGE_QUERY = """SELECT ( COALESCE(?lang, ?other_lang) as ?main_lang) WHERE { |
25 OPTIONAL { ?s dct:language ?lang }. |
23 OPTIONAL { ?s dct:language ?lang }. |
51 |
49 |
52 return self.__query_cache[query] \ |
50 return self.__query_cache[query] \ |
53 if query in self.__query_cache \ |
51 if query in self.__query_cache \ |
54 else self.__query_cache.setdefault(query, prepareQuery(query, initNs=namespaces)) |
52 else self.__query_cache.setdefault(query, prepareQuery(query, initNs=namespaces)) |
55 |
53 |
56 def get_empty_graph(self): |
|
57 record_graph = Graph() |
|
58 record_graph.bind('iiep',"http://www.iiep.unesco.org/plan4learning/model.owl#") |
|
59 record_graph.bind('dct',"http://purl.org/dc/terms/") |
|
60 return record_graph |
|
61 |
54 |
62 def extract_single_value_form_graph(self, graph, q, bindings={}, index=0, convert=lambda v:unicode(v) if v is not None else None): |
55 def extract_single_value_form_graph(self, graph, q, bindings={}, index=0, convert=lambda v:unicode(v) if v is not None else None): |
63 return next(self.extract_multiple_values_from_graph(graph, q, bindings, index, convert), None) |
56 return next(self.extract_multiple_values_from_graph(graph, q, bindings, index, convert), None) |
64 |
57 |
65 def extract_multiple_values_from_graph(self, graph, q, bindings={}, index=0, convert=lambda v:unicode(v) if v is not None else None): |
58 def extract_multiple_values_from_graph(self, graph, q, bindings={}, index=0, convert=lambda v:unicode(v) if v is not None else None): |
318 for _,elem in context: |
311 for _,elem in context: |
319 if elem.tag == "{%s}Record" % IIEP: |
312 if elem.tag == "{%s}Record" % IIEP: |
320 i += 1 |
313 i += 1 |
321 writer = show_progress(i, total_records, "Processing record nb %d " % i, 50, writer=writer) |
314 writer = show_progress(i, total_records, "Processing record nb %d " % i, 50, writer=writer) |
322 try: |
315 try: |
323 record_graph = self.get_empty_graph() |
316 record_graph = get_empty_graph() |
324 record_graph.parse(data=ET.tostring(elem, encoding='utf-8'), format='xml') |
317 record_graph.parse(data=ET.tostring(elem, encoding='utf-8'), format='xml') |
325 # add transaction management |
318 # add transaction management |
326 self.build_record(record_graph) |
319 self.build_record(record_graph) |
327 except Exception as e: |
320 except Exception as e: |
328 transaction.rollback() |
321 transaction.rollback() |