# HG changeset patch # User ymh # Date 1416239316 -3600 # Node ID a1b85604132c732e2cac932972efd5c85df008ed # Parent 6129e7390abef36cd0be172181af1cece7c7a5a0 upgrade rdflib to correct unicode bugs diff -r 6129e7390abe -r a1b85604132c src/hdalab/management/commands/query_dbpedia.py --- a/src/hdalab/management/commands/query_dbpedia.py Sun Nov 16 19:35:12 2014 +0100 +++ b/src/hdalab/management/commands/query_dbpedia.py Mon Nov 17 16:48:36 2014 +0100 @@ -14,7 +14,6 @@ import sys import traceback -from SPARQLWrapper import SPARQLWrapper, N3 from django import db from django.conf import settings from django.core.management.base import NoArgsCommand @@ -22,6 +21,7 @@ from django.db import transaction from django.db.models import Count from rdflib import URIRef, Graph +import requests logger = logging.getLogger(__name__) @@ -69,12 +69,17 @@ help='the tag to query'), ) - def query_dbpedia(self, query, fmt=N3): - endpoint = SPARQLWrapper(settings.DBPEDIA_URI_TEMPLATE % ( 'sparql', '' ), returnFormat=fmt) - endpoint.setQuery(query) - return Graph().parse(data=endpoint.queryAndConvert(), format=fmt) - - + def query_dbpedia(self, query, fmt='n3'): + url = settings.DBPEDIA_URI_TEMPLATE % ( 'sparql', '' ) + params = { + 'query': query, + 'format': {'n3':'text/turtle', 'rdf/xml':"application/rdf+xml", 'nt': 'text/plain'}.get(fmt, 'text/turtle') + } + resp = requests.get(url, params=params) + logger.debug("Query dbpedia : %s", resp.text) + return Graph().parse(data=resp.text, format=fmt) + + def handle_noargs(self, **options): self.style = no_style() @@ -148,21 +153,21 @@ labels = {} thumbnail = None with transaction.commit_on_success(): - res_abstracts = self.query_dbpedia("select distinct ?y where {<%s> ?y}" % (tag.dbpedia_uri), N3) + res_abstracts = self.query_dbpedia("select distinct ?y where {<%s> ?y}" % (tag.dbpedia_uri), 'n3') for _,_,o in res_abstracts.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)): abstracts[o.language] = (unicode(o), True) logger.debug("Abstracts: %r" % abstracts) - res_labels = self.query_dbpedia("select distinct ?y where {<%s> ?y}" % (tag.dbpedia_uri), N3) + res_labels = self.query_dbpedia("select distinct ?y where {<%s> ?y}" % (tag.dbpedia_uri), 'n3') for _,_,o in res_labels.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)): labels[o.language] = (unicode(o), True) logger.debug("Labels: %r" % labels) - res_thumbnails = self.query_dbpedia("select distinct ?y where {<%s> ?y} limit 1" % (tag.dbpedia_uri), N3) + res_thumbnails = self.query_dbpedia("select distinct ?y where {<%s> ?y} limit 1" % (tag.dbpedia_uri), 'n3') for _,_,o in res_thumbnails.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)): thumbnail = unicode(o) - res_links = self.query_dbpedia('select distinct ?y where { <%s> ?p ?y . FILTER regex(?y, "^%s")}' % (tag.dbpedia_uri, settings.DBPEDIA_URI_TEMPLATE % ( 'resource', '' )), N3) + res_links = self.query_dbpedia('select distinct ?y where { <%s> ?p ?y . FILTER regex(?y, "^%s")}' % (tag.dbpedia_uri, settings.DBPEDIA_URI_TEMPLATE % ( 'resource', '' )), 'n3') for _,_,o in res_links.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)): tagqs = Tag.objects.filter(dbpedia_uri=unicode(o)) if tagqs: @@ -177,8 +182,9 @@ if lang[0] not in abstracts: abstracts[lang[0]] = (ref_abstract, False) - dbfield , created = DbpediaFields.objects.get_or_create(dbpedia_uri=tag.dbpedia_uri,tag=tag, defaults={'abstract':ref_abstract, 'thumbnail':thumbnail, 'label':ref_label}) #@UndefinedVariable + dbfield , created = DbpediaFields.objects.get_or_create(tag=tag, defaults={'dbpedia_uri':tag.dbpedia_uri, 'abstract':ref_abstract, 'thumbnail':thumbnail, 'label':ref_label}) #@UndefinedVariable if not created: + dbfield.dbpedia_uri = tag.dbpedia_uri dbfield.abstract = ref_abstract dbfield.thumbnail = thumbnail dbfield.label = ref_label diff -r 6129e7390abe -r a1b85604132c virtualenv/res/lib/lib_create_env.py --- a/virtualenv/res/lib/lib_create_env.py Sun Nov 16 19:35:12 2014 +0100 +++ b/virtualenv/res/lib/lib_create_env.py Mon Nov 17 16:48:36 2014 +0100 @@ -29,8 +29,8 @@ 'HAYSTACK': {'setup': 'django-haystack', 'url': 'https://github.com/toastdriven/django-haystack/archive/v2.2.0.tar.gz', 'local': 'django-haystack-2.2.0.tar.gz', 'install':{'method':'pip', 'option_str': None, 'dict_extra_env': None}}, 'WHOOSH': {'setup': 'whoosh', 'url':'https://pypi.python.org/packages/source/W/Whoosh/Whoosh-2.5.7.tar.gz', 'local':'whoosh-2.5.7.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}}, 'WIKITOOLS' : { 'setup': 'wikitools', 'url': 'https://github.com/alexz-enwp/wikitools/archive/1.2.tar.gz', 'local': 'wikitools-1.2.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'ISODATE' : {'setup': 'isodate', 'url': 'http://pypi.python.org/packages/source/i/isodate/isodate-0.5.0.tar.gz', 'local': 'isodate-0.5.0.tar.gz', 'install' : {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, - 'RDFLIB' : { 'setup': 'rdflib', 'url': 'https://github.com/RDFLib/rdflib/archive/4.1.2.tar.gz', 'local': 'rdflib-4.1.2.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'ISODATE' : {'setup': 'isodate', 'url': 'http://pypi.python.org/packages/source/i/isodate/isodate-0.5.1.tar.gz', 'local': 'isodate-0.5.1.tar.gz', 'install' : {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, + 'RDFLIB' : { 'setup': 'rdflib', 'url': 'https://github.com/RDFLib/rdflib/archive/96c30f98bbb628e13aaa32c9c392584b0fbf8788.tar.gz', 'local': 'rdflib-4.2-dev.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'SPARQLWRAPPER' : { 'setup': 'SPARQLWrapper', 'url': 'https://github.com/RDFLib/sparqlwrapper/archive/1.6.4.tar.gz', 'local' : 'SPARQLWrapper-1.6.4.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'REQUESTS' : { 'setup': 'requests', 'url': 'https://github.com/kennethreitz/requests/archive/v2.4.3.tar.gz', 'local' : 'requests-2.4.3.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, 'ELASTICSEARCH' : { 'setup': 'elasticsearch', 'url': 'https://github.com/elasticsearch/elasticsearch-py/archive/1.2.0.tar.gz', 'local' : 'elasticsearch-1.2.0.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}}, diff -r 6129e7390abe -r a1b85604132c virtualenv/res/src/isodate-0.5.0.tar.gz Binary file virtualenv/res/src/isodate-0.5.0.tar.gz has changed diff -r 6129e7390abe -r a1b85604132c virtualenv/res/src/isodate-0.5.1.tar.gz Binary file virtualenv/res/src/isodate-0.5.1.tar.gz has changed diff -r 6129e7390abe -r a1b85604132c virtualenv/res/src/rdflib-4.1.2.tar.gz Binary file virtualenv/res/src/rdflib-4.1.2.tar.gz has changed diff -r 6129e7390abe -r a1b85604132c virtualenv/res/src/rdflib-4.2-dev.tar.gz Binary file virtualenv/res/src/rdflib-4.2-dev.tar.gz has changed diff -r 6129e7390abe -r a1b85604132c virtualenv/web/res/requirements.txt --- a/virtualenv/web/res/requirements.txt Sun Nov 16 19:35:12 2014 +0100 +++ b/virtualenv/web/res/requirements.txt Mon Nov 17 16:48:36 2014 +0100 @@ -12,7 +12,7 @@ django-registration==1.1 httplib2==0.9 html5lib==0.999 -isodate==0.5.0 +isodate==0.5.1 lxml==3.4.0 psycopg2==2.5.4 pycrypto==2.6.1 @@ -21,7 +21,8 @@ requests==2.4.3 simplejson==3.6.5 six==1.8.0 -rdflib==4.1.2 +rdflib==4.2-dev +#-e git+git://github.com/IRI-Research/rdflib.git@96c30f98bbb628e13aaa32c9c392584b0fbf8788#egg=rdflib urllib3==1.9.1 wikitools==1.2 wsgiref==0.1.2