diff -r 57efd01f1715 -r 6296aa12fd71 src/p4l/management/commands/import_record.py --- a/src/p4l/management/commands/import_record.py Fri Aug 30 17:09:14 2013 +0200 +++ b/src/p4l/management/commands/import_record.py Fri Aug 30 17:58:35 2013 +0200 @@ -4,7 +4,7 @@ from django.db import reset_queries, transaction from optparse import make_option from p4l.models import Record, Language -from p4l.utils import show_progress, get_code_from_language_uri +from p4l.utils import show_progress from rdflib import Graph, Namespace, BNode, URIRef from rdflib.plugins.sparql import prepareQuery import logging @@ -100,20 +100,6 @@ return False return bool(val) - def convert_lang(self, val, default_lang): - return unicode(val) if (val is not None and len(unicode(val))>0) else default_lang - - - def get_record_default_language(self, g, record_uri): - lang_uri = self.extract_single_value_form_graph(g, DEFAULT_LANGUAGE_QUERY, bindings={'s': URIRef(record_uri)}) - if not lang_uri: - lang_uri = DEFAULT_LANGUAGE_URI - lang_code = get_code_from_language_uri(lang_uri) - if lang_code is None: - logger.warn("get_record_default_language: no code found for %s in record %s" % (lang_uri, record_uri)) - return get_code_from_language_uri(DEFAULT_LANGUAGE_URI) - return lang_code - def add_to_related_collection(self, coll, graph, fields, q, bindings={}, convert=lambda v: unicode(v) if v is not None else None, through_fields=None): @@ -146,7 +132,6 @@ def build_record(self, graph): record_uri = self.extract_single_value_form_graph(graph,"SELECT DISTINCT ?s WHERE { ?s rdf:type iiep:Record .}") - default_language_code = self.get_record_default_language(graph, record_uri) record = Record() record.uri = record_uri @@ -176,7 +161,6 @@ ['label','lang'], "SELECT DISTINCT ?o ( lang(?o) as ?l) WHERE { ?s iiep:periodical ?o .}", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)}, through_fields = ['lang'] ) @@ -185,7 +169,7 @@ graph, ['label', 'meetingNumber', 'meetingPlace', 'meetingDate', 'meetingYear', 'lang'], "SELECT ?l ?mn ?mp ?md ?my (lang(COALESCE(?l,?nm, ?mp,?md,?my)) as ?lang) WHERE { [iiep:meeting ?bnode]. OPTIONAL { ?bnode rdfs:label ?l }. OPTIONAL { ?bnode iiep:meetingNumber ?mn }. OPTIONAL { ?bnode iiep:meetingPlace ?mp }. OPTIONAL { ?bnode iiep:meetingDate ?md }. OPTIONAL { ?bnode iiep:meetingYear ?my }}", - convert={'lang':lambda l: self.convert_lang(l, default_language_code), 'meetingYear' : lambda y: int(y) if y is not None else None}, + convert={'meetingYear' : lambda y: int(y) if y is not None else None}, through_fields = ['lang'] ) @@ -194,7 +178,6 @@ graph, ['title', 'volume', 'lang'], "SELECT ?t ?vol (lang(COALESCE(?t,?vol)) as ?lang) WHERE { [iiep:serie ?bnode]. OPTIONAL { ?bnode dct:title ?t }. OPTIONAL { ?bnode iiep:volume ?vol } }", - convert={'lang':lambda l: self.convert_lang(l, default_language_code)}, through_fields = ['lang'] ) @@ -226,7 +209,6 @@ ['issn', 'lang'], "SELECT ?issn (lang(COALESCE(?issn)) as ?lang) WHERE { ?s iiep:issn ?issn . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -235,7 +217,6 @@ ['isbn', 'lang'], "SELECT ?isbn (lang(COALESCE(?isbn)) as ?lang) WHERE { ?s iiep:isbn ?isbn . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -244,7 +225,6 @@ ['documentCode', 'lang'], "SELECT ?c (lang(COALESCE(?c)) as ?lang) WHERE { ?s iiep:documentCode ?c . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -253,7 +233,14 @@ ['title', 'lang'], "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s dct:title ?t . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} + ) + + self.add_to_related_collection( + record.abstracts, + graph, + ['abstract', 'lang'], + "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s dct:abstract ?t . }", + bindings={'s':URIRef(record.uri)}, ) self.add_to_related_collection( @@ -262,7 +249,6 @@ ['title', 'lang'], "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s iiep:addedTitle ?t . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -271,7 +257,6 @@ ['title', 'lang'], "SELECT ?t (lang(COALESCE(?t)) as ?lang) WHERE { ?s iiep:titleMainDocument ?t . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -279,7 +264,6 @@ graph, ['imprintCity', 'publisher', 'imprintDate', 'lang'], "SELECT ?c ?p ?d (lang(COALESCE(?c, ?p, ?d)) as ?lang) WHERE { [ iiep:imprint ?bnode ]. OPTIONAL { ?bnode iiep:imprintCity ?c }. OPTIONAL { ?bnode dct:publisher ?p }. OPTIONAL { ?bnode iiep:imprintDate ?d }}", - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -288,7 +272,6 @@ ['collation', 'lang'], "SELECT ?c (lang(COALESCE(?c)) as ?lang) WHERE { ?s iiep:collation ?c . }", bindings={'s':URIRef(record.uri)}, - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -296,7 +279,6 @@ graph, ['volume', 'number', 'lang'], "SELECT ?v ?n (lang(COALESCE(?v, ?n)) as ?lang) WHERE { [ iiep:volumeIssue ?bnode ]. OPTIONAL { ?bnode iiep:volume ?v }. OPTIONAL { ?bnode iiep:number ?n }}", - convert={'lang':lambda l: self.convert_lang(l, default_language_code)} ) self.add_to_related_collection( @@ -358,7 +340,7 @@ # def process_url(self, records_url, options): # #open graph with rdflib - # #TODO: manage memory + # # g = Graph() # print("Loading %s" % records_url) # g.parse(records_url)