# HG changeset patch # User ymh # Date 1478134346 -3600 # Node ID 7fba86fa8604700123c01206b439ebd637c3113c # Parent c731ab9b934d4a8f73a2ef3d28f493e2a19f56bc Sparql client implementation diff -r c731ab9b934d -r 7fba86fa8604 dev/provisioning/modules/sysconfig/templates/handle/handle_config.dct.erb --- a/dev/provisioning/modules/sysconfig/templates/handle/handle_config.dct.erb Mon Oct 31 14:24:23 2016 +0100 +++ b/dev/provisioning/modules/sysconfig/templates/handle/handle_config.dct.erb Thu Nov 03 01:52:26 2016 +0100 @@ -38,6 +38,9 @@ "300:0.NA/<%= @hdl_prefix %>" "300:0.NA/<%= @hdl_test_prefix %>" "300:0.NA/<%= @hdl_test_prefix %>_DSA" + "300:<%= @hdl_prefix %>/<%= @hdl_prefix_admin %>" + "300:<%= @hdl_test_prefix %>/<%= @hdl_prefix_admin %>" + "300:<%= @hdl_test_prefix %>/<%= @hdl_prefix_admin %>_DSA" ) "replication_admins" = ( diff -r c731ab9b934d -r 7fba86fa8604 server/src/app/Http/Controllers/Sparql/SparqlClientController.php --- a/server/src/app/Http/Controllers/Sparql/SparqlClientController.php Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/app/Http/Controllers/Sparql/SparqlClientController.php Thu Nov 03 01:52:26 2016 +0100 @@ -5,6 +5,9 @@ use Log; use Illuminate\Http\Request; +use Illuminate\Pagination\LengthAwarePaginator; +use Illuminate\Pagination\Paginator; + use GuzzleHttp\Client; use EasyRdf\Sparql\Result as SparqlResult; use EasyRdf\Graph; @@ -52,17 +55,25 @@ $this->httpClient = $httpClient; } + public function getSparqlClient($timeout = null) { + if(is_null($timeout)) { + $timeout = config('corpusparole.sparql_client_timeout'); + } + $queryUrl = config('corpusparole.sesame_query_url'); + if($timeout > 0) { + $queryUrl = $queryUrl . + ((strlen(parse_url($queryUrl, PHP_URL_QUERY)) > 0)?"&":"?"). + "timeout=$timeout"; + } + return new \EasyRdf\Sparql\Client($queryUrl, config('corpusparole.sesame_update_url')); + } + // display form public function index() { return view('sparql/sparqlClientForm'); } - private function querySelect(Request $request, $query, $analyser) { - $countResult = $this->sparqlClient->query($analyser->getCountQuery()); - $countField = $countResult->getFields()[0]; - $count = $countResult->current()->$countField; - $docs = $this->sparqlClient->query($query); - $fields = $docs->getFields(); + private function readDocs($docs, $fields) { $results = []; foreach($docs as $row) { $results[] = array_reduce($fields, function($res, $field) use ($row) { @@ -74,10 +85,78 @@ return $res; }, []); } + return $results; + } + + private function abort($code, $message, $exception) { + throw new \Symfony\Component\HttpKernel\Exception\HttpException($code, $message, $exception, []); + } + + private function processQueryException($exception) { + $message = $exception->getMessage(); + if($exception instanceof \EasyRdf\Http\Exception) { + if(preg_match("/SPARQL\squery/", $message)) { + $this->abort(400, "La requête SPARQL n'est pas reconnue", $exception); + } else { + $this->abort(500, "Problème HTTP lors de la requête SPARQL", $exception); + } + } elseif($exception instanceof \EasyRdf\Exception) { + if(preg_match("/timed\sout/i", $message)) { + $this->abort(408, "Time-out causé par la requête SPARQL", $exception); + } else { + $this->abort(500, "Problème dans la requête SPARQL", $exception); + } + } else { + $this->abort(500, "Erreur serveur lors de la requête", $exception); + } + } + + private function querySelect(Request $request, $query, $analyser) { + + $limit = intval($request->input('limit', config('corpusparole.sparql_client_default_limit'))); + + if($limit === 0 || !is_null($analyser->getLimit()) || !is_null($analyser->getOffset()) ) { + try { + $docs = $this->getSparqlClient()->query($query); + } catch(\Exception $exception) { + $this->processQueryException($exception); + } + + $fields = $docs->getFields(); + $results = $this->readDocs($docs, $fields); + $count = count($results); + + } else { + + $page = Paginator::resolveCurrentPage(config('corpusparole.pagination_page_param')); + assert(is_null($page) || is_numeric($page)); + + $offset = max(0,($page - 1) * $limit); + + try { + $countResult = $this->getSparqlClient()->query($analyser->getCountQuery()); + $docs = $this->getSparqlClient()->query($query . " LIMIT $limit OFFSET $offset"); + } catch(\Exception $exception) { + $this->processQueryException($exception); + } + + + $countField = $countResult->getFields()[0]; + $count = $countResult->current()->$countField->getValue(); + + $fields = $docs->getFields(); + + $results = new LengthAwarePaginator($this->readDocs($docs, $fields), $count, $limit, $page, [ + 'path' => Paginator::resolveCurrentPath(), + 'pageName' => config('corpusparole.pagination_page_param'), + ]); + } + $namespaces = array_reduce(array_keys(RdfHelper::getPrefixes()), function($res, $p) { $res[$p] = RdfNamespace::namespaces()[$p]; return $res; }, []); + $data = [ 'query' => $query, 'count' => $count, @@ -92,8 +171,13 @@ } private function queryGraph(Request $request, $query, $analyser) { + try { + $docs = $this->getSparqlClient()->query($query); + } catch(\Exception $exception) { + $this->processQueryException($exception); + } - $docs = $this->sparqlClient->query($query); + $fields = ["subject", "predicate", "object"]; $results = []; foreach ($docs->resources() as $resource ) { @@ -129,8 +213,14 @@ } private function queryAsk(Request $request, $query, $analyser) { + try { + $result = $this->getSparqlClient()->query($query); + } catch(\Exception $exception) { + $this->processQueryException($exception); + } + $data = [ - 'results' => $this->sparqlClient->query($query), + 'results' => $result, 'namespaces' => $analyser->getPrefixes() ]; @@ -159,8 +249,7 @@ } elseif($queryType === SparqlQueryAnalyser::ASK_QUERY) { list($view, $data) = $this->queryAsk($request, $query, $analyser); } else { - //return 500 - abort(500, "Serialization format unknown"); + abort(400, "La requête n'est pas reconnue"); } return view($view, $data); @@ -182,7 +271,30 @@ $headers['Accept'] = $format; } - $sesameResp = $this->httpClient->get(config('corpusparole.sesame_query_url'), ['query' => $request->all(), 'headers' => $headers]); + $queryParams = $request->all(); + $queryParams['timeout'] = config('corpusparole.sparql_client_timeout'); + $queryUrl = config('corpusparole.sesame_query_url'); + + try { + $sesameResp = $this->httpClient->post($queryUrl, ['form_params' => $queryParams, 'headers' => $headers]); + } catch(\GuzzleHttp\Exception\ServerException $exception) { + if($exception->getCode() == 503) { + $this->abort(408, "Time-out causé par la requête SPARQL", $exception); + } else { + $this->abort(500, "Problème lors de la requête SPARQL", $exception); + } + + } catch(\GuzzleHttp\Exception\RequestException $exception) { + $message = $exception->getMessage(); + if(preg_match("/MALFORMED\sQUERY/i", $message)) { + $abortMessage = "Requête SPARQL mal-formée"; + } else { + $abortMessage = "Problème lors de la requête SPARQL"; + } + $this->abort($exception->getCode(), $abortMessage, $exception); + } catch(\Exception $exception) { + $this->abort(500, "Erreur serveur lors de la requête", $exception); + } $resp = response((string)$sesameResp->getBody(), $sesameResp->getStatusCode()); foreach ($sesameResp->getHeaders() as $name => $values) { diff -r c731ab9b934d -r 7fba86fa8604 server/src/app/Libraries/Handle/HandleClient.php --- a/server/src/app/Libraries/Handle/HandleClient.php Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/app/Libraries/Handle/HandleClient.php Thu Nov 03 01:52:26 2016 +0100 @@ -180,9 +180,10 @@ ]; # Send the request again with a valid correctly signed Authorization header $sessionResp = $this->httpClient->put($url.'this', ['headers' => $pkheaders, 'verify' => false]); - Log::debug('Create session with auth: '.$sessionResp->getStatusCode().' : '.$sessionResp->getReasonPhrase()); + $jsonResp = json_decode($sessionResp->getBody(), true); + Log::debug('Create session with auth: '.$sessionResp->getStatusCode().' : '.$sessionResp->getReasonPhrase(). " with body : \n".$sessionResp->getBody()); $this->session = $jsonResp['authenticated']?$jsonResp['sessionId']:""; $headers['Authorization'] = "Handle version=\"0\", sessionId=\"$this->session\""; diff -r c731ab9b934d -r 7fba86fa8604 server/src/app/Libraries/Sparql/SparqlClient.php --- a/server/src/app/Libraries/Sparql/SparqlClient.php Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/app/Libraries/Sparql/SparqlClient.php Thu Nov 03 01:52:26 2016 +0100 @@ -167,8 +167,8 @@ * @param string $query The query string to be executed * @return object EasyRdf\Sparql\Result|EasyRdf\Graph Result of the query. */ - public function query($query) { - return $this->sparqlClient->query($query); + public function query($query, $timeout=0) { + return $this->sparqlClient->query($query, $timeout); } } diff -r c731ab9b934d -r 7fba86fa8604 server/src/app/Libraries/Sparql/SparqlQueryAnalyser.php --- a/server/src/app/Libraries/Sparql/SparqlQueryAnalyser.php Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/app/Libraries/Sparql/SparqlQueryAnalyser.php Thu Nov 03 01:52:26 2016 +0100 @@ -47,35 +47,44 @@ return $this->queryType; } - private function extractPrefix() { - $prefixes = []; - $rawPrefixes = []; - $res = preg_replace_callback("%".self::SPARQL_PREFIX_BASE_REGEXP."%iu", function($m) use (&$prefixes, &$rawPrefixes) { - $rawPrefixes[] = trim($m[0]); - $prefixes[$m[3]?$m[3]:""] = $m[4]; + private function extractPrefixLimit() { + $this->prefixes = []; + $this->rawPrefixes = []; + $res = preg_replace_callback("%".self::SPARQL_PREFIX_BASE_REGEXP."%iu", function($m) { + $this->rawPrefixes[] = trim($m[0]); + $this->prefixes[$m[3]?$m[3]:""] = $m[4]; return ""; }, $this->query); + $res = preg_replace_callback("%".self::SPARQL_LIMIT_OFFSET_QUERY_REGEXP."%iu", function($m) { + for($i=0;$i<(count($m)-1)/2;$i++) { + if(Utils::startsWith(strtolower($m[2*$i+1]), "limit")) { + $this->limit = intval($m[$i*2+2]); + } elseif (Utils::startsWith(strtolower($m[2*$i+1]), "offset")) { + $this->offset = intval($m[$i*2+2]); + } + } + }, $res); - return [$rawPrefixes, $prefixes, trim($res)]; + $this->rawQuery = trim($res); } public function getRawPrefixes() { if($this->rawPrefixes === false) { - list($this->rawPrefixes, $this->prefixes, $this->rawQuery) = $this->extractPrefix(); + $this->extractPrefixLimit(); } return $this->rawPrefixes; } public function getPrefixes() { if($this->prefixes === false) { - list($this->rawPrefixes, $this->prefixes, $this->rawQuery) = $this->extractPrefix(); + $this->extractPrefixLimit(); } return $this->prefixes; } public function getRawQuery() { if($this->rawQuery === false) { - list($this->rawPrefixes, $this->prefixes, $this->rawQuery) = $this->extractPrefix(); + $this->extractPrefixLimit(); } return $this->rawQuery; } @@ -92,15 +101,7 @@ } private function setLimitOffset() { - if(preg_match("%".self::SPARQL_LIMIT_OFFSET_QUERY_REGEXP."%iu", $this->query, $m) === 1) { - for($i=0;$i<(count($m)-1)/2;$i++) { - if(Utils::startsWith(strtolower($m[2*$i+1]), "limit")) { - $this->limit = intval($m[$i*2+2]); - } elseif (Utils::startsWith(strtolower($m[2*$i+1]), "offset")) { - $this->offset = intval($m[$i*2+2]); - } - } - } + $this->extractPrefixLimit(); if($this->limit === false) { $this->limit = null; } diff -r c731ab9b934d -r 7fba86fa8604 server/src/app/Libraries/Sparql/TimeoutSparqlClient.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/app/Libraries/Sparql/TimeoutSparqlClient.php Thu Nov 03 01:52:26 2016 +0100 @@ -0,0 +1,100 @@ +request('query', $query, $timeout); + } + + protected function request($type, $query, $timeout=0) + { + $processed_query = $this->preprocessQuery($query); + $response = $this->executeQuery($processed_query, $type, $timeout); + if (!$response->isSuccessful()) { + throw new Http\Exception("HTTP request for SPARQL query failed", 0, null, $response->getBody()); + } + if ($response->getStatus() == 204) { + // No content + return $response; + } + return $this->parseResponseToQuery($response); + } + + /** + * Build http-client object, execute request and return a response + * + * @param string $processed_query + * @param string $type Should be either "query" or "update" + * + * @return Http\Response|\Zend\Http\Response + * @throws Exception + */ + protected function executeQuery($processed_query, $type, $timeout=0) + { + $client = Http::getDefaultHttpClient(); + $client->resetParameters(); + // Tell the server which response formats we can parse + $sparql_results_types = array( + 'application/sparql-results+json' => 1.0, + 'application/sparql-results+xml' => 0.8 + ); + if ($type == 'update') { + // accept anything, as "response body of a […] update request is implementation defined" + // @see http://www.w3.org/TR/sparql11-protocol/#update-success + $accept = Format::getHttpAcceptHeader($sparql_results_types); + $client->setHeaders('Accept', $accept); + $client->setMethod('POST'); + $client->setUri($this->updateUri); + $client->setRawData($processed_query); + $client->setHeaders('Content-Type', 'application/sparql-update'); + } elseif ($type == 'query') { + $re = '(?:(?:\s*BASE\s*<.*?>\s*)|(?:\s*PREFIX\s+.+:\s*<.*?>\s*))*'. + '(CONSTRUCT|SELECT|ASK|DESCRIBE)[\W]'; + $result = null; + $matched = mb_eregi($re, $processed_query, $result); + if (false === $matched or count($result) !== 2) { + // non-standard query. is this something non-standard? + $query_verb = null; + } else { + $query_verb = strtoupper($result[1]); + } + if ($query_verb === 'SELECT' or $query_verb === 'ASK') { + // only "results" + $accept = Format::formatAcceptHeader($sparql_results_types); + } elseif ($query_verb === 'CONSTRUCT' or $query_verb === 'DESCRIBE') { + // only "graph" + $accept = Format::getHttpAcceptHeader(); + } else { + // both + $accept = Format::getHttpAcceptHeader($sparql_results_types); + } + $client->setHeaders('Accept', $accept); + $encodedQuery = 'query=' . urlencode($processed_query).(($timeout>0)?"&timeout=$timeout":""); + // Use GET if the query is less than 2kB + // 2046 = 2kB minus 1 for '?' and 1 for NULL-terminated string on server + if (strlen($encodedQuery) + strlen($this->queryUri) <= 2046) { + $delimiter = $this->queryUri_has_params ? '&' : '?'; + $client->setMethod('GET'); + $client->setUri($this->queryUri . $delimiter . $encodedQuery); + } else { + // Fall back to POST instead (which is un-cacheable) + $client->setMethod('POST'); + $client->setUri($this->queryUri); + $client->setRawData($encodedQuery); + $client->setHeaders('Content-Type', 'application/x-www-form-urlencoded'); + } + } else { + throw new Exception('unexpected request-type: '.$type); + } + return $client->request(); + } + + +} diff -r c731ab9b934d -r 7fba86fa8604 server/src/config/corpusparole.php --- a/server/src/config/corpusparole.php Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/config/corpusparole.php Thu Nov 03 01:52:26 2016 +0100 @@ -383,6 +383,9 @@ 'filter_max_languages_nb'=> 200, 'filter_max_themes_nb'=> 200, 'filter_max_discourses_nb'=> 200, - 'filter_max_dates_nb'=> 200 + 'filter_max_dates_nb'=> 200, + + 'sparql_client_timeout' => 5, + 'sparql_client_default_limit' => 100 ]; diff -r c731ab9b934d -r 7fba86fa8604 server/src/public/css/app.css --- a/server/src/public/css/app.css Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/public/css/app.css Thu Nov 03 01:52:26 2016 +0100 @@ -6061,4 +6061,13 @@ .corpus-rdf-boolean-false { color: #a11; } +.error-btn.collapsed .caret { + border-top: 0; + border-bottom: 4px dashed; + border-bottom: 4px solid \9; + content: ""; } + +#baseExceptionCollapse { + margin-top: 10px; } + /*# sourceMappingURL=app.css.map */ diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/assets/js/sparqlclient.js --- a/server/src/resources/assets/js/sparqlclient.js Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/resources/assets/js/sparqlclient.js Thu Nov 03 01:52:26 2016 +0100 @@ -69,11 +69,6 @@ var yasqe = YASQE.fromTextArea($('#query').get(0), { sparql: { showQueryButton: false, - //endpoint: "{{ route('sparql_proxy') }}", - //requestMethod: "GET", - //acceptHeaderGraph: "application/rdf+json,/;q=0.9" - //acceptHeaderGraph: "text/turtle,/;q=0.9", - //acceptHeaderSelect: "application/x-turtle,/;q=0.9", } }); yasqe.on("update", function(instance) { @@ -94,10 +89,12 @@ $('#limit').val($(e.target).text()); }); - // var yasr = YASR($('#results').get(0), { - // getUsedPrefixes: yasqe.getPrefixesFromQuery, - // useGoogleCharts: false, - // outputPlugins: ["table", "error", "boolean", "rawResponse", "pivot"], - // }); - // yasqe.options.sparql.callbacks.complete = yasr.setResponse; + $("#reset-query").click(function(e) { + yasqe.setValue(""); + }); + + $("#format").change(function(e) { + $("#limit, #limit-btn").prop('disabled', ($(e.target).val() != "text/html") ); + }); + } diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/assets/sass/_app-core.scss --- a/server/src/resources/assets/sass/_app-core.scss Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/resources/assets/sass/_app-core.scss Thu Nov 03 01:52:26 2016 +0100 @@ -151,3 +151,14 @@ .corpus-rdf-boolean-false { color: #a11; } + +.error-btn.collapsed .caret { + border-top: 0; + border-bottom: $caret-width-base dashed; + border-bottom: $caret-width-base solid \9; // IE8 + content: ""; +} + +#baseExceptionCollapse { + margin-top: 10px; +} diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/views/errors/400.blade.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/resources/views/errors/400.blade.php Thu Nov 03 01:52:26 2016 +0100 @@ -0,0 +1,1 @@ +@extends('errors/base-errors') diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/views/errors/408.blade.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/resources/views/errors/408.blade.php Thu Nov 03 01:52:26 2016 +0100 @@ -0,0 +1,1 @@ +@extends('errors/base-errors') diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/views/errors/500.blade.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/resources/views/errors/500.blade.php Thu Nov 03 01:52:26 2016 +0100 @@ -0,0 +1,2 @@ +@extends('errors/base-errors') + diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/views/errors/base-errors.blade.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/resources/views/errors/base-errors.blade.php Thu Nov 03 01:52:26 2016 +0100 @@ -0,0 +1,17 @@ +@extends('base') + +@section('content') +
+
+

Code {{ $exception->getStatusCode() }} : Erreur lors de la requête

+

{{ $exception->getMessage() }}

+ +
+
{{ $exception->getPrevious()->getCode() }}: {{ htmlspecialchars($exception->getPrevious()->getMessage(), ENT_SUBSTITUTE) }}
+
+
+
+ +@endsection diff -r c731ab9b934d -r 7fba86fa8604 server/src/resources/views/sparql/sparqlClientForm.blade.php --- a/server/src/resources/views/sparql/sparqlClientForm.blade.php Mon Oct 31 14:24:23 2016 +0100 +++ b/server/src/resources/views/sparql/sparqlClientForm.blade.php Thu Nov 03 01:52:26 2016 +0100 @@ -20,22 +20,6 @@ @endsection @@ -59,16 +43,11 @@
- - - milliseconds (values less than 1000 are ignored) -
-
- +
- +