# HG changeset patch # User ymh # Date 1456431975 -3600 # Node ID 821253d361d115a04a04c4d852da36e86588235b # Parent d97eda8bc8ecf774e36cca263e96e3817f881165 add bnf resolver diff -r d97eda8bc8ec -r 821253d361d1 common/corpus-common-addon/app/services/bnf-resolver.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/corpus-common-addon/app/services/bnf-resolver.js Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,16 @@ +import Ember from 'ember'; +import store from 'corpus-common-addon/utils/store'; +import * as constants from 'corpus-common-addon/utils/constants' + +export default Ember.Service.extend({ + env: function() { + return Ember.getOwner(this).resolveRegistration('config:environment') + }, + _getStoreKey: function(id) { + return 'rameau:'+id; + }, + getName: function(id) { + + } + +}); diff -r d97eda8bc8ec -r 821253d361d1 server/src/app/Http/Controllers/Api/BnfController.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/app/Http/Controllers/Api/BnfController.php Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,37 @@ +bnfResolver = $bnfResolver; + } + + public function index() { + return print_r($this->bnfResolver, true); + } + + /** + * Display the specified resource. + * + * @param int $id od comma separated list of ids + * @return \Illuminate\Http\Response + */ + public function show($id) + { + $ids = array_map(function($rid) { return (strpos($rid,"ark:/12148")===0)?$rid:"ark:/12148/$rid"; },explode(",", $id)); + try { + return response()->json(['bnfids' => $this->bnfResolver->getLabels($ids)]); + } catch (BnfResolverException $e) { + abort(500, $e->getMessage()); + } + } + +} diff -r d97eda8bc8ec -r 821253d361d1 server/src/app/Http/routes.php --- a/server/src/app/Http/routes.php Thu Feb 25 12:24:30 2016 +0100 +++ b/server/src/app/Http/routes.php Thu Feb 25 21:26:15 2016 +0100 @@ -42,6 +42,8 @@ ['only' => ['show']]); Route::resource('lexvo', 'Api\LexvoController', ['only' => ['show']]); + Route::resource('bnf', 'Api\BnfController', + ['only' => ['index','show']]); Route::resource('languages', 'Api\LanguageController', ['only' => ['index']]); }); diff -r d97eda8bc8ec -r 821253d361d1 server/src/app/Providers/BnfServiceProvider.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/app/Providers/BnfServiceProvider.php Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,23 @@ +app->bind('CorpusParole\Services\BnfResolverInterface', function($app) { + $sparqlClient = $app->make('EasyRdf\Sparql\Client', [config('corpusparole.bnf_query_url'), null]); + return $app->make('CorpusParole\Services\BnfResolver', [$sparqlClient,]); + }); + } +} diff -r d97eda8bc8ec -r 821253d361d1 server/src/app/Services/BnfResolver.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/app/Services/BnfResolver.php Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,109 @@ +sparqlClient = $sparqlClient; + } + + private function checkBnfId($id) { + $bnfid = $id; + + if(strpos($id, config('corpusparole.bnf_ark_base_url')) === 0) { + $bnfid = config('corpusparole.bnf_base_url').substr($id, strlen(config('corpusparole.bnf_ark_base_url'))); + } + elseif(strpos($id, config('corpusparole.bnf_base_url')) !== 0) { + $bnfid = config('corpusparole.bnf_base_url').$id; + } + $bnfid = rtrim($bnfid, '/'); + if(preg_match("/^".preg_quote(config('corpusparole.bnf_base_url'),"/")."ark\:\/12148\/[[:alnum:]]/", $bnfid) !== 1) { + throw new BnfResolverException("the provided id \"$id\" is not a BNF id"); + } + return $bnfid; + } + + /** + * Get label from BNF id + * @param string $id The id to resolve. Can be an url starting with http://data.bnf.fr/ or http://ark.bnf.fr/ + * @return a string with the name + */ + public function getLabel($id) { + $res = $this->getlabels([$id,]); + assert(array_key_exists($id,$res), "the result must contains $id"); + return $res[$id]; + } + + /** + * Get a list of names from an array of viaf ids. + * @param array $ids The array of ids to resolve. + * Each id can be an url starting with http://data.bnf.fr/ or http://ark.bnf.fr/ + */ + public function getLabels(array $ids) { + + if(count($ids) > config('corpusparole.bnf_max_ids')) { + throw new BnfResolverException("Too manys ids provided"); + } + + $bnfids = array_map([$this, 'checkBnfId'], $ids); + $bnfidsMap = array_combine($bnfids, $ids); + + $results = []; + $missingBnfids = []; + + foreach ($bnfidsMap as $bnfid => $bnfidSource) { + $cachedValue = Cache::get("bnf:$bnfid"); + if(is_null($cachedValue)) { + array_push($missingBnfids, $bnfid); + } else { + $results[$bnfidSource] = $cachedValue; + } + } + + if(count($missingBnfids) == 0) { + return $results; + } + + $query = "SELECT ?s ?o WHERE {"; + foreach ($missingBnfids as $index => $bid) { + if($index > 0) { + $query .= " UNION "; + } + $query .= "{ <$bid> ?o. ?s ?o. FILTER(?s = <$bid> && lang(?o) = \"fr\")}"; + } + $query .= "}"; + + $docs = $this->sparqlClient->query($query); + + $resultsRaw = []; + + foreach ($docs as $doc) { + $bnfid = $doc->s->getUri(); + $bnflabel = $doc->o; + + $value = $bnflabel->getValue(); + + if(!empty($value)) { + $resultsRaw[$bnfid] = $bnflabel; + } + } + + foreach ($missingBnfids as $bnfid) { + $bnfidSource = $bnfidsMap[$bnfid]; + $missingValue = (array_key_exists($bnfid,$resultsRaw) && $resultsRaw[$bnfid])?mb_strtolower($resultsRaw[$bnfid]->getValue()):""; + if (mb_strlen($missingValue)>0) { + Cache::put("bnf:$bnfid", $missingValue, config('corpusparole.bnf_cache_expiration')); + $results[$bnfidSource] = $missingValue; + } + else { + $results[$bnfidSource] = null; + } + } + + return $results; + } + +} diff -r d97eda8bc8ec -r 821253d361d1 server/src/app/Services/BnfResolverException.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/app/Services/BnfResolverException.php Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,6 @@ + config('corpusparole.lexvo_max_ids')) { + throw new LexvoResolverException("Too manys ids provided"); + } + $lexvoids = array_map([$this, 'checkLexvoId'], $ids); $lexvoidsMap = array_combine($lexvoids, $ids); diff -r d97eda8bc8ec -r 821253d361d1 server/src/app/Services/ViafResolver.php --- a/server/src/app/Services/ViafResolver.php Thu Feb 25 12:24:30 2016 +0100 +++ b/server/src/app/Services/ViafResolver.php Thu Feb 25 21:26:15 2016 +0100 @@ -107,6 +107,11 @@ * Each id can be an url starting with http://viaf.org/viaf/ */ public function getNames(array $ids) { + + if(count($ids) > config('corpusparole.viaf_max_ids')) { + throw new ViafResolverException("Too manys ids provided"); + } + return array_combine($ids, array_map([$this,'getName'], $ids)); } diff -r d97eda8bc8ec -r 821253d361d1 server/src/config/app.php --- a/server/src/config/app.php Thu Feb 25 12:24:30 2016 +0100 +++ b/server/src/config/app.php Thu Feb 25 21:26:15 2016 +0100 @@ -154,6 +154,7 @@ 'CorpusParole\Providers\GuzzleServiceProvider', 'CorpusParole\Providers\ViafServiceProvider', 'CorpusParole\Providers\LexvoServiceProvider', + 'CorpusParole\Providers\BnfServiceProvider', 'CorpusParole\Providers\VersionServiceProvider', ], diff -r d97eda8bc8ec -r 821253d361d1 server/src/config/corpusparole.php --- a/server/src/config/corpusparole.php Thu Feb 25 12:24:30 2016 +0100 +++ b/server/src/config/corpusparole.php Thu Feb 25 21:26:15 2016 +0100 @@ -86,11 +86,19 @@ 'viaf_base_url' => 'http://viaf.org/viaf/', 'viaf_cache_expiration' => 60*24*30, + 'viaf_max_ids' => 500, 'lexvo_base_url' => 'http://lexvo.org/id/iso639-3/', 'lexvo_cache_expiration' => 60*24*30, + 'lexvo_max_ids' => 500, 'lexvo_sesame_query_url' => $sesameBaseUrl.'repositories/'.env('CORPUSPAROLE_SESAME_LEXVO_REPOSITORY'), + 'bnf_base_url' => 'http://data.bnf.fr/', + 'bnf_ark_base_url' => 'http://ark.bnf.fr/', + 'bnf_cache_expiration' => 60*24*30, + 'bnf_max_ids' => 500, + 'bnf_query_url' => 'http://data.bnf.fr/sparql', + 'bo_client_environment' => [ "modulePrefix" => "bo-client", "podModulePrefix" => "bo-client/pods", diff -r d97eda8bc8ec -r 821253d361d1 server/src/tests/Controllers/BnfControllerTest.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/tests/Controllers/BnfControllerTest.php Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,88 @@ +bnfResolver = m::mock('CorpusParole\Services\BnfResolverInterface'); + $this->app->instance('CorpusParole\Services\BnfResolverInterface', $this->bnfResolver); + } + + public function tearDown() { + m::close(); + parent::tearDown(); + } + + public function testSetup() { + //do nothing jsut test setup & teardown + } + + public function testShow() { + $this->bnfResolver + ->shouldReceive('getLabels') + ->with(['ark:/12148/cb11946662b', 'ark:/12148/cb11965628b']) + ->once() + ->andReturn([ + 'ark:/12148/cb11946662b' => 'parents et enfants', + 'ark:/12148/cb11965628b' => 'frères et soeurs' + ]); + $response = $this->get('/api/v1/bnf/cb11946662b,cb11965628b')-> + seeJsonEquals(['bnfids' => [ + 'ark:/12148/cb11946662b' => 'parents et enfants', + 'ark:/12148/cb11965628b' => 'frères et soeurs' + ]]); + } + + public function testShowOne() { + $this->bnfResolver + ->shouldReceive('getLabels') + ->with(['ark:/12148/cb11946662b']) + ->once() + ->andReturn([ + 'ark:/12148/cb11946662b' => 'parents et enfants' + ]); + $response = $this->get('/api/v1/bnf/cb11946662b')-> + seeJsonEquals(['bnfids' => [ + 'ark:/12148/cb11946662b' => 'parents et enfants' + ]]); + } + + public function testShowUnknown() { + $this->bnfResolver + ->shouldReceive('getLabels') + ->with(['ark:/12148/cb12345678b']) + ->once() + ->andReturn([ + 'ark:/12148/cb12345678b' => null + ]); + $response = $this->get('/api/v1/bnf/cb12345678b')-> + seeJsonEquals(['bnfids' => [ + 'ark:/12148/cb12345678b' => null + ]]); + } + + public function testShowMalformed() { + $this->bnfResolver + ->shouldReceive('getLabels') + ->with(['ark:/12148/abcdef','ark:/12148/ghij']) + ->once() + ->andThrow('CorpusParole\Services\BnfResolverException', "BnfId not in correct format", 500); + $response = $this->get('/api/v1/bnf/abcdef,ghij'); + + $this->assertResponseStatus(500); + } + +} diff -r d97eda8bc8ec -r 821253d361d1 server/src/tests/Services/BnfResolverTest.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/src/tests/Services/BnfResolverTest.php Thu Feb 25 21:26:15 2016 +0100 @@ -0,0 +1,299 @@ +shouldReceive('isSuccessful')->andReturn(true) + ->shouldReceive('getBody')->andReturn($queryResult) + ->shouldReceive('getStatus')->andReturn(200) + ->shouldReceive('getHeader')->andReturn('application/sparql-results+json;charset=UTF-8') + ->mock(); + $this->httpClient = m::mock('EasyRdf\Http\Client') + ->shouldReceive('setConfig') + ->shouldReceive('resetParameters') + ->shouldReceive('setHeaders')//->with(m::on(function($headers) { print("HEADER => $headers\n"); return true;}),m::on(function($value) { print("VALUE => $value\n"); return true;})) + ->shouldReceive('setMethod') + ->shouldReceive('setUri')//->with(m::on(function($uri) { print($uri."\n"); return true;})) + ->shouldReceive('request')->andReturn($response) + ->mock(); + Http::setDefaultHttpClient($this->httpClient); + $this->bnfResolver = $this->app->make('CorpusParole\Services\BnfResolverInterface'); + } + + public function setUp() { + parent::setUp(); + } + + public function tearDown() { + parent::tearDown(); + m::close(); + } + + /** + * Just test the setup + * + * @return void + */ + public function testSetUp() { + $this->assertTrue(true); + } + + /** + * resolve parents et enfants + * @return void + */ + public function testResolveSingleId() { + + $this->setUpSparqlClient(self::BNF_RDF_RESP); + + $reslabel = $this->bnfResolver->getLabel("http://data.bnf.fr/ark:/12148/cb11946662b"); + + $this->assertEquals('parents et enfants', $reslabel, "Result must be parents et enfants"); + } + + /** + * resolve parents et enfants ark url + * @return void + */ + public function testResolveSingleIdArkUrl() { + + $this->setUpSparqlClient(self::BNF_RDF_RESP); + + $reslabel = $this->bnfResolver->getLabel("http://ark.bnf.fr/ark:/12148/cb11946662b"); + + $this->assertEquals('parents et enfants', $reslabel, "Result must be parents et enfants"); + } + + /** + * resolve parents et enfants, ark id + * @return void + */ + public function testResolveSingleIdArk() { + + $this->setUpSparqlClient(self::BNF_RDF_RESP); + + $reslabel = $this->bnfResolver->getLabel("ark:/12148/cb11946662b"); + + $this->assertEquals('parents et enfants', $reslabel, "Result must be parents et enfants"); + } + + + /** + * resolve foo + * @return void + */ + public function testResolveBadId() { + + $this->setUpSparqlClient(self::BNF_EMPTY_RDF_RESP); + + $reslabel = $this->bnfResolver->getLabel('ark:/12148/cb12345678b'); + + $this->assertNull($reslabel, "Result must be null"); + } + + /** + * resolve foo + * @return void + * @expectedException CorpusParole\Services\BnfResolverException + * @expectedExceptionMessage the provided id "21dsasd;;" is not a BNF id + * @expectedExceptionCode 0 + */ + public function testResolveBadFormat() { + + $this->setUpSparqlClient(self::BNF_EMPTY_RDF_RESP); + + $reslabel = $this->bnfResolver->getLabel('21dsasd;;'); + } + + /** + * resolve foo + * @return void + * @expectedException CorpusParole\Services\BnfResolverException + * @expectedExceptionMessage the provided id "http://sdsasd.org/foo" is not a BNF id + * @expectedExceptionCode 0 + */ + public function testResolveBadFormatFullId() { + + $this->setUpSparqlClient(self::BNF_EMPTY_RDF_RESP); + + $reslabel = $this->bnfResolver->getLabel('http://sdsasd.org/foo'); + } + + + /** + * resolve parents et enfants and frères et soeurs + * @return void + */ + public function testResolveMultipleId() { + + $this->setUpSparqlClient(self::BNF_2_ID_RDF_RESP); + + $reslabels = $this->bnfResolver->getLabels(['ark:/12148/cb11965628b','ark:/12148/cb11946662b']); + + $this->assertCount(2, $reslabels, "Must have 2 results"); + $this->assertArrayHasKey('ark:/12148/cb11946662b', $reslabels); + $this->assertArrayHasKey('ark:/12148/cb11965628b', $reslabels); + + $this->assertEquals('parents et enfants', $reslabels['ark:/12148/cb11946662b'], "Result for ark:/12148/cb11965628b must be parents et enfants"); + $this->assertEquals('frères et soeurs', $reslabels['ark:/12148/cb11965628b'], "Result for ark:/12148/cb11946662b must be frères et soeurs"); + } + + /** + * resolve parents et enfants and frères et soeurs + * @return void + */ + public function testResolveMultipleFullIdArkUrl() { + + $this->setUpSparqlClient(self::BNF_2_ID_RDF_RESP); + + $reslabels = $this->bnfResolver->getLabels(['http://ark.bnf.fr/ark:/12148/cb11946662b','http://ark.bnf.fr/ark:/12148/cb11965628b']); + + $this->assertCount(2, $reslabels, "Must have 2 results"); + $this->assertArrayHasKey('http://ark.bnf.fr/ark:/12148/cb11946662b', $reslabels); + $this->assertArrayHasKey('http://ark.bnf.fr/ark:/12148/cb11965628b', $reslabels); + + $this->assertEquals('parents et enfants', $reslabels['http://ark.bnf.fr/ark:/12148/cb11946662b'], "Result for http://ark.bnf.fr/ark:/12148/cb11946662b must be parents et enfants"); + $this->assertEquals('frères et soeurs', $reslabels['http://ark.bnf.fr/ark:/12148/cb11965628b'], "Result for http://ark.bnf.fr/ark:/12148/cb11965628b must be frères et soeurs"); + } + + /** + * resolve parents et enfants and frères et soeurs + * @return void + */ + public function testResolveMultipleFullIdDataUrl() { + + $this->setUpSparqlClient(self::BNF_2_ID_RDF_RESP); + + $reslabels = $this->bnfResolver->getLabels(['http://data.bnf.fr/ark:/12148/cb11946662b','http://data.bnf.fr/ark:/12148/cb11965628b']); + + $this->assertCount(2, $reslabels, "Must have 2 results"); + $this->assertArrayHasKey('http://data.bnf.fr/ark:/12148/cb11946662b', $reslabels); + $this->assertArrayHasKey('http://data.bnf.fr/ark:/12148/cb11965628b', $reslabels); + + $this->assertEquals('parents et enfants', $reslabels['http://data.bnf.fr/ark:/12148/cb11946662b'], "Result for http://data.bnf.fr/ark:/12148/cb11946662b must be parents et enfants"); + $this->assertEquals('frères et soeurs', $reslabels['http://data.bnf.fr/ark:/12148/cb11965628b'], "Result for http://data.bnf.fr/ark:/12148/cb11965628b must be frères et soeurs"); + } + + /** + * check query + * @return void + */ + public function testQuery() { + + $expectedUri = config('corpusparole.bnf_query_url')."?query=SELECT+%3Fs+%3Fo+WHERE+%7B%7B+%3Chttp%3A%2F%2Fdata.bnf.fr%2Fark%3A%2F12148%2Fcb11946662b%3E+%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23prefLabel%3E+%3Fo.+%3Fs+%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23prefLabel%3E+%3Fo.+FILTER%28%3Fs+%3D+%3Chttp%3A%2F%2Fdata.bnf.fr%2Fark%3A%2F12148%2Fcb11946662b%3E+%26%26+lang%28%3Fo%29+%3D+%22fr%22%29%7D+UNION+%7B+%3Chttp%3A%2F%2Fdata.bnf.fr%2Fark%3A%2F12148%2Fcb11965628b%3E+%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23prefLabel%3E+%3Fo.+%3Fs+%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23prefLabel%3E+%3Fo.+FILTER%28%3Fs+%3D+%3Chttp%3A%2F%2Fdata.bnf.fr%2Fark%3A%2F12148%2Fcb11965628b%3E+%26%26+lang%28%3Fo%29+%3D+%22fr%22%29%7D%7D"; + $response = m::mock('EasyRdf\Http\Response') + ->shouldReceive('isSuccessful')->andReturn(true) + ->shouldReceive('getBody')->andReturn(self::BNF_EMPTY_RDF_RESP) //result not important + ->shouldReceive('getStatus')->andReturn(200) + ->shouldReceive('getHeader')->andReturn('application/sparql-results+json;charset=UTF-8') + ->mock(); + $this->httpClient = m::mock('EasyRdf\Http\Client') + ->shouldReceive('setConfig') + ->shouldReceive('resetParameters') + ->shouldReceive('setHeaders')//->with(m::on(function($headers) { print("HEADER => $headers\n"); return true;}),m::on(function($value) { print("VALUE => $value\n"); return true;})) + ->shouldReceive('setMethod') + ->shouldReceive('setUri') + ->with($expectedUri) + //->with(m::on(function($uri) { print($uri."\n"); return true;})) + ->shouldReceive('request')->andReturn($response) + ->mock(); + Http::setDefaultHttpClient($this->httpClient); + $this->bnfResolver = $this->app->make('CorpusParole\Services\BnfResolverInterface'); + + $reslabels = $this->bnfResolver->getLabels(['ark:/12148/cb11946662b','ark:/12148/cb11965628b']); + + } + +}