--- a/src/hdalab/management/commands/query_dbpedia.py Sun Nov 16 19:35:12 2014 +0100
+++ b/src/hdalab/management/commands/query_dbpedia.py Mon Nov 17 16:48:36 2014 +0100
@@ -14,7 +14,6 @@
import sys
import traceback
-from SPARQLWrapper import SPARQLWrapper, N3
from django import db
from django.conf import settings
from django.core.management.base import NoArgsCommand
@@ -22,6 +21,7 @@
from django.db import transaction
from django.db.models import Count
from rdflib import URIRef, Graph
+import requests
logger = logging.getLogger(__name__)
@@ -69,12 +69,17 @@
help='the tag to query'),
)
- def query_dbpedia(self, query, fmt=N3):
- endpoint = SPARQLWrapper(settings.DBPEDIA_URI_TEMPLATE % ( 'sparql', '' ), returnFormat=fmt)
- endpoint.setQuery(query)
- return Graph().parse(data=endpoint.queryAndConvert(), format=fmt)
-
-
+ def query_dbpedia(self, query, fmt='n3'):
+ url = settings.DBPEDIA_URI_TEMPLATE % ( 'sparql', '' )
+ params = {
+ 'query': query,
+ 'format': {'n3':'text/turtle', 'rdf/xml':"application/rdf+xml", 'nt': 'text/plain'}.get(fmt, 'text/turtle')
+ }
+ resp = requests.get(url, params=params)
+ logger.debug("Query dbpedia : %s", resp.text)
+ return Graph().parse(data=resp.text, format=fmt)
+
+
def handle_noargs(self, **options):
self.style = no_style()
@@ -148,21 +153,21 @@
labels = {}
thumbnail = None
with transaction.commit_on_success():
- res_abstracts = self.query_dbpedia("select distinct ?y where {<%s> <http://dbpedia.org/ontology/abstract> ?y}" % (tag.dbpedia_uri), N3)
+ res_abstracts = self.query_dbpedia("select distinct ?y where {<%s> <http://dbpedia.org/ontology/abstract> ?y}" % (tag.dbpedia_uri), 'n3')
for _,_,o in res_abstracts.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
abstracts[o.language] = (unicode(o), True)
logger.debug("Abstracts: %r" % abstracts)
- res_labels = self.query_dbpedia("select distinct ?y where {<%s> <http://www.w3.org/2000/01/rdf-schema#label> ?y}" % (tag.dbpedia_uri), N3)
+ res_labels = self.query_dbpedia("select distinct ?y where {<%s> <http://www.w3.org/2000/01/rdf-schema#label> ?y}" % (tag.dbpedia_uri), 'n3')
for _,_,o in res_labels.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
labels[o.language] = (unicode(o), True)
logger.debug("Labels: %r" % labels)
- res_thumbnails = self.query_dbpedia("select distinct ?y where {<%s> <http://dbpedia.org/ontology/thumbnail> ?y} limit 1" % (tag.dbpedia_uri), N3)
+ res_thumbnails = self.query_dbpedia("select distinct ?y where {<%s> <http://dbpedia.org/ontology/thumbnail> ?y} limit 1" % (tag.dbpedia_uri), 'n3')
for _,_,o in res_thumbnails.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
thumbnail = unicode(o)
- res_links = self.query_dbpedia('select distinct ?y where { <%s> ?p ?y . FILTER regex(?y, "^%s")}' % (tag.dbpedia_uri, settings.DBPEDIA_URI_TEMPLATE % ( 'resource', '' )), N3)
+ res_links = self.query_dbpedia('select distinct ?y where { <%s> ?p ?y . FILTER regex(?y, "^%s")}' % (tag.dbpedia_uri, settings.DBPEDIA_URI_TEMPLATE % ( 'resource', '' )), 'n3')
for _,_,o in res_links.triples((None, URIRef('http://www.w3.org/2005/sparql-results#value'), None)):
tagqs = Tag.objects.filter(dbpedia_uri=unicode(o))
if tagqs:
@@ -177,8 +182,9 @@
if lang[0] not in abstracts:
abstracts[lang[0]] = (ref_abstract, False)
- dbfield , created = DbpediaFields.objects.get_or_create(dbpedia_uri=tag.dbpedia_uri,tag=tag, defaults={'abstract':ref_abstract, 'thumbnail':thumbnail, 'label':ref_label}) #@UndefinedVariable
+ dbfield , created = DbpediaFields.objects.get_or_create(tag=tag, defaults={'dbpedia_uri':tag.dbpedia_uri, 'abstract':ref_abstract, 'thumbnail':thumbnail, 'label':ref_label}) #@UndefinedVariable
if not created:
+ dbfield.dbpedia_uri = tag.dbpedia_uri
dbfield.abstract = ref_abstract
dbfield.thumbnail = thumbnail
dbfield.label = ref_label
--- a/virtualenv/res/lib/lib_create_env.py Sun Nov 16 19:35:12 2014 +0100
+++ b/virtualenv/res/lib/lib_create_env.py Mon Nov 17 16:48:36 2014 +0100
@@ -29,8 +29,8 @@
'HAYSTACK': {'setup': 'django-haystack', 'url': 'https://github.com/toastdriven/django-haystack/archive/v2.2.0.tar.gz', 'local': 'django-haystack-2.2.0.tar.gz', 'install':{'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'WHOOSH': {'setup': 'whoosh', 'url':'https://pypi.python.org/packages/source/W/Whoosh/Whoosh-2.5.7.tar.gz', 'local':'whoosh-2.5.7.tar.gz', 'install' : {'method':'pip', 'option_str': None, 'dict_extra_env': None}},
'WIKITOOLS' : { 'setup': 'wikitools', 'url': 'https://github.com/alexz-enwp/wikitools/archive/1.2.tar.gz', 'local': 'wikitools-1.2.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'ISODATE' : {'setup': 'isodate', 'url': 'http://pypi.python.org/packages/source/i/isodate/isodate-0.5.0.tar.gz', 'local': 'isodate-0.5.0.tar.gz', 'install' : {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
- 'RDFLIB' : { 'setup': 'rdflib', 'url': 'https://github.com/RDFLib/rdflib/archive/4.1.2.tar.gz', 'local': 'rdflib-4.1.2.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'ISODATE' : {'setup': 'isodate', 'url': 'http://pypi.python.org/packages/source/i/isodate/isodate-0.5.1.tar.gz', 'local': 'isodate-0.5.1.tar.gz', 'install' : {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
+ 'RDFLIB' : { 'setup': 'rdflib', 'url': 'https://github.com/RDFLib/rdflib/archive/96c30f98bbb628e13aaa32c9c392584b0fbf8788.tar.gz', 'local': 'rdflib-4.2-dev.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'SPARQLWRAPPER' : { 'setup': 'SPARQLWrapper', 'url': 'https://github.com/RDFLib/sparqlwrapper/archive/1.6.4.tar.gz', 'local' : 'SPARQLWrapper-1.6.4.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'REQUESTS' : { 'setup': 'requests', 'url': 'https://github.com/kennethreitz/requests/archive/v2.4.3.tar.gz', 'local' : 'requests-2.4.3.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
'ELASTICSEARCH' : { 'setup': 'elasticsearch', 'url': 'https://github.com/elasticsearch/elasticsearch-py/archive/1.2.0.tar.gz', 'local' : 'elasticsearch-1.2.0.tar.gz', 'install': {'method': 'pip', 'option_str': None, 'dict_extra_env': None}},
Binary file virtualenv/res/src/isodate-0.5.0.tar.gz has changed
Binary file virtualenv/res/src/isodate-0.5.1.tar.gz has changed
Binary file virtualenv/res/src/rdflib-4.1.2.tar.gz has changed
Binary file virtualenv/res/src/rdflib-4.2-dev.tar.gz has changed
--- a/virtualenv/web/res/requirements.txt Sun Nov 16 19:35:12 2014 +0100
+++ b/virtualenv/web/res/requirements.txt Mon Nov 17 16:48:36 2014 +0100
@@ -12,7 +12,7 @@
django-registration==1.1
httplib2==0.9
html5lib==0.999
-isodate==0.5.0
+isodate==0.5.1
lxml==3.4.0
psycopg2==2.5.4
pycrypto==2.6.1
@@ -21,7 +21,8 @@
requests==2.4.3
simplejson==3.6.5
six==1.8.0
-rdflib==4.1.2
+rdflib==4.2-dev
+#-e git+git://github.com/IRI-Research/rdflib.git@96c30f98bbb628e13aaa32c9c392584b0fbf8788#egg=rdflib
urllib3==1.9.1
wikitools==1.2
wsgiref==0.1.2