plan4learning: comparison src/p4l/management/commands/import

equal deleted inserted replaced

-:57efd01f1715
+:6296aa12fd71
 from django.core.management import BaseCommand
 from django.db import reset_queries, transaction
 from optparse import make_option
 from p4l.models import Record, Language
-from p4l.utils import show_progress, get_code_from_language_uri
+from p4l.utils import show_progress
 from rdflib import Graph, Namespace, BNode, URIRef
 from rdflib.plugins.sparql import prepareQuery
 import logging
 import xml.etree.cElementTree as ET
 return True
 else:
 return False
 return bool(val)
-def convert_lang(self, val, default_lang):
-return unicode(val) if (val is not None and len(unicode(val))>0) else default_lang
-def get_record_default_language(self, g, record_uri):
-lang_uri = self.extract_single_value_form_graph(g, DEFAULT_LANGUAGE_QUERY, bindings={'s': URIRef(record_uri)})
-if not lang_uri:
-lang_uri = DEFAULT_LANGUAGE_URI
-lang_code = get_code_from_language_uri(lang_uri)
-if lang_code is None:
-logger.warn("get_record_default_language: no code found for %s in record %s" % (lang_uri, record_uri))
-return get_code_from_language_uri(DEFAULT_LANGUAGE_URI)
-return lang_code
 def add_to_related_collection(self, coll, graph, fields, q, bindings={},  convert=lambda v: unicode(v) if v is not None else None, through_fields=None):
 for val in self.extract_multiple_values_from_graph(graph, q, bindings=bindings, index=fields, convert=convert):
 def build_record(self, graph):
 record_uri = self.extract_single_value_form_graph(graph,"SELECT DISTINCT ?s WHERE { ?s rdf:type iiep:Record .}")
-default_language_code = self.get_record_default_language(graph, record_uri)
 record = Record()
 record.uri = record_uri
 record.identifier = self.extract_single_value_form_graph(graph,"SELECT DISTINCT ?o WHERE { ?s dct:identifier ?o .}", bindings={'s':URIRef(record.uri)})
 record.notes = self.extract_single_value_form_graph(graph,"SELECT DISTINCT ?o WHERE { ?s iiep:notes ?o .}", bindings={'s':URIRef(record.uri)})
 record.periodicals,
 graph,
 ['label','lang'],
 "SELECT DISTINCT ?o  ( lang(?o) as ?l) WHERE { ?s iiep:periodical ?o .}",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)},
 through_fields = ['lang']
 )
 self.add_to_related_collection(
 record.meetings,
 graph,
 ['label', 'meetingNumber', 'meetingPlace', 'meetingDate', 'meetingYear', 'lang'],
 "SELECT ?l ?mn ?mp ?md ?my (lang(COALESCE(?l,?nm, ?mp,?md,?my)) as ?lang) WHERE { [iiep:meeting ?bnode]. OPTIONAL { ?bnode rdfs:label ?l }. OPTIONAL { ?bnode iiep:meetingNumber ?mn }. OPTIONAL { ?bnode iiep:meetingPlace ?mp }.  OPTIONAL { ?bnode iiep:meetingDate ?md }. OPTIONAL { ?bnode iiep:meetingYear ?my }}",
-convert={'lang':lambda l: self.convert_lang(l, default_language_code), 'meetingYear' : lambda y: int(y) if y is not None else None},
+convert={'meetingYear' : lambda y: int(y) if y is not None else None},
 through_fields = ['lang']
 )
 self.add_to_related_collection(
 record.series,
 graph,
 ['title', 'volume', 'lang'],
 "SELECT ?t ?vol (lang(COALESCE(?t,?vol)) as ?lang) WHERE { [iiep:serie ?bnode]. OPTIONAL { ?bnode dct:title ?t }. OPTIONAL { ?bnode iiep:volume ?vol } }",
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)},
 through_fields = ['lang']
 )
 self.add_to_related_collection(
 record.subjectCorporateBodies,
 record.issns,
 graph,
 ['issn', 'lang'],
 "SELECT ?issn (lang(COALESCE(?issn)) as ?lang) WHERE { ?s iiep:issn ?issn . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.isbns,
 graph,
 ['isbn', 'lang'],
 "SELECT ?isbn (lang(COALESCE(?isbn)) as ?lang) WHERE { ?s iiep:isbn ?isbn . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.documentCodes,
 graph,
 ['documentCode', 'lang'],
 "SELECT ?c (lang(COALESCE(?c)) as ?lang) WHERE { ?s iiep:documentCode ?c . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.titles,
 graph,
 ['title', 'lang'],
 "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s dct:title ?t . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
+)
+self.add_to_related_collection(
+record.abstracts,
+graph,
+['abstract', 'lang'],
+"SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s dct:abstract ?t . }",
+bindings={'s':URIRef(record.uri)},
 )
 self.add_to_related_collection(
 record.addedTitles,
 graph,
 ['title', 'lang'],
 "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s iiep:addedTitle ?t . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.titlesMainDocument,
 graph,
 ['title', 'lang'],
 "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s iiep:titleMainDocument ?t . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.imprints,
 graph,
 ['imprintCity', 'publisher', 'imprintDate', 'lang'],
 "SELECT ?c ?p ?d (lang(COALESCE(?c, ?p, ?d)) as ?lang) WHERE { [ iiep:imprint ?bnode ]. OPTIONAL { ?bnode iiep:imprintCity ?c }. OPTIONAL { ?bnode dct:publisher ?p }. OPTIONAL { ?bnode iiep:imprintDate ?d }}",
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.collations,
 graph,
 ['collation', 'lang'],
 "SELECT ?c (lang(COALESCE(?c)) as ?lang) WHERE { ?s iiep:collation ?c . }",
 bindings={'s':URIRef(record.uri)},
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.volumeIssues,
 graph,
 ['volume', 'number', 'lang'],
 "SELECT ?v ?n (lang(COALESCE(?v, ?n)) as ?lang) WHERE { [ iiep:volumeIssue ?bnode ]. OPTIONAL { ?bnode iiep:volume ?v }. OPTIONAL { ?bnode iiep:number ?n }}",
-convert={'lang':lambda l: self.convert_lang(l, default_language_code)}
 )
 self.add_to_related_collection(
 record.urls,
 graph,
 return errors
 # def process_url(self, records_url, options):
 #     #open graph with rdflib
-#     #TODO: manage memory
+#
 #     g = Graph()
 #     print("Loading %s" % records_url)
 #     g.parse(records_url)
 #     print("Parsing %s done" % records_url)
 #     for i,record_uri in enumerate(g[:RDF.type:IIEP.Record]):

changeset 13	6296aa12fd71
parent 7	02008d61c3c8
child 14	52fa6990e0bb