--- a/cms/app-client/mirage/config.js Wed Sep 28 17:24:02 2016 +0200
+++ b/cms/app-client/mirage/config.js Fri Sep 30 00:43:04 2016 +0200
@@ -38,6 +38,8 @@
this.get('/stats/discourses', 'discourses');
+ this.get('/stats/datestats', 'datestats');
+
this.get('/resolvers/lexvo/:ids', ({lexvos}, request) => {
var langIds = decodeURIComponent(request.params.ids);
var resMap = _.reduce(langIds.split(','), function(res, id) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cms/app-client/mirage/fixtures/datestats.js Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,62 @@
+export default [
+ { 'id': "1948", "count": 3 },
+ { 'id': "1957", "count": 29 },
+ { 'id': "1958", "count": 39 },
+ { 'id': "1959", "count": 19 },
+ { 'id': "1960", "count": 3 },
+ { 'id': "1961", "count": 25 },
+ { 'id': "1962", "count": 34 },
+ { 'id': "1963", "count": 22 },
+ { 'id': "1964", "count": 35 },
+ { 'id': "1965", "count": 110 },
+ { 'id': "1966", "count": 11 },
+ { 'id': "1967", "count": 40 },
+ { 'id': "1968", "count": 43 },
+ { 'id': "1969", "count": 228 },
+ { 'id': "1970", "count": 405 },
+ { 'id': "1971", "count": 62 },
+ { 'id': "1972", "count": 89 },
+ { 'id': "1973", "count": 36 },
+ { 'id': "1974", "count": 18 },
+ { 'id': "1975", "count": 21 },
+ { 'id': "1976", "count": 34 },
+ { 'id': "1977", "count": 11 },
+ { 'id': "1978", "count": 15 },
+ { 'id': "1979", "count": 7 },
+ { 'id': "1980", "count": 551 },
+ { 'id': "1981", "count": 19 },
+ { 'id': "1982", "count": 13 },
+ { 'id': "1983", "count": 18 },
+ { 'id': "1984", "count": 20 },
+ { 'id': "1985", "count": 30 },
+ { 'id': "1986", "count": 14 },
+ { 'id': "1987", "count": 12 },
+ { 'id': "1988", "count": 17 },
+ { 'id': "1989", "count": 65 },
+ { 'id': "1990", "count": 37 },
+ { 'id': "1991", "count": 14 },
+ { 'id': "1992", "count": 46 },
+ { 'id': "1993", "count": 43 },
+ { 'id': "1994", "count": 39 },
+ { 'id': "1995", "count": 95 },
+ { 'id': "1996", "count": 37 },
+ { 'id': "1997", "count": 126 },
+ { 'id': "1998", "count": 42 },
+ { 'id': "1999", "count": 33 },
+ { 'id': "2000", "count": 13 },
+ { 'id': "2001", "count": 65 },
+ { 'id': "2002", "count": 31 },
+ { 'id': "2003", "count": 24 },
+ { 'id': "2004", "count": 29 },
+ { 'id': "2005", "count": 88 },
+ { 'id': "2006", "count": 17 },
+ { 'id': "2007", "count": 17 },
+ { 'id': "2008", "count": 74 },
+ { 'id': "2009", "count": 10 },
+ { 'id': "2010", "count": 162 },
+ { 'id': "2011", "count": 69 },
+ { 'id': "2012", "count": 26 },
+ { 'id': "2013", "count": 5 },
+ { 'id': "2014", "count": 1 },
+ { 'id': "2015", "count": 1 }
+];
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cms/app-client/mirage/models/datestat.js Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,4 @@
+import { Model } from 'ember-cli-mirage';
+
+export default Model.extend({
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cms/app-client/mirage/serializers/datestat.js Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,9 @@
+import { JSONAPISerializer } from 'ember-cli-mirage';
+
+import _ from 'lodash/lodash';
+
+export default JSONAPISerializer.extend({
+ serialize(response) {
+ return _(response.models).map((dateinfo) => { return [dateinfo.id, dateinfo.count];}).object().value();
+ }
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/server/src/app/Http/Controllers/Api/DateStatsController.php Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,126 @@
+<?php
+
+namespace CorpusParole\Http\Controllers\Api;
+
+// use CorpusParole\Http\Requests;
+use Illuminate\Http\Request;
+use Log;
+
+use CorpusParole\Libraries\Sparql\SparqlClient;
+
+use CorpusParole\Http\Controllers\Controller;
+
+
+class DateStatsController extends Controller
+{
+ private $sparqlClient = null;
+
+ public function __construct(SparqlClient $sparqlClient) {
+ $this->sparqlClient = $sparqlClient;
+ }
+
+ /**
+ * Display the specified resource.
+ *
+ * @return \Illuminate\Http\Response
+ */
+ public function index(Request $request)
+ {
+ $query = preg_replace('/\s+/', ' ', "SELECT (?d as ?date) (COUNT(?d) AS ?count)
+ WHERE {
+ ?_ a <http://www.europeana.eu/schemas/edm/ProvidedCHO>.
+ ?_ <http://purl.org/dc/terms/created> ?d
+ }
+ GROUP BY ?d
+ ORDER BY ?d");
+
+ $res = $this->sparqlClient->query($query);
+
+ $dates = [];
+
+ foreach ($res as $row) {
+
+ $count = intval($row->count->getValue());
+ $date = $row->date;
+ $dateType = $date->getDatatypeUri();
+
+ $processedDates = [];
+ if($dateType === "http://purl.org/dc/terms/Period") {
+ $processedDates = $this->processPeriod($date->getValue(), $count);
+ }
+ elseif($dateType === "http://purl.org/dc/terms/W3CDTF") {
+ $processedDates = $this->processDate($date->getValue(), $count);
+ }
+
+ $dates = array_reduce(array_keys($processedDates), function($datesArray, $item) use ($processedDates) {
+ if(!isset($datesArray[$item])) {
+ $datesArray[$item] = 0;
+ }
+ $datesArray[$item] += $processedDates[$item];
+ return $datesArray;
+ }, $dates);
+ }
+
+ ksort($dates);
+
+ return response()->json(['datestats' => $dates ]);
+ }
+
+ private function extractYear($dateStr) {
+ if(preg_match("/^\\d{4}$/", $dateStr) === 1) {
+ $dateStr = "$dateStr-1-1";
+ }
+ $date = date_create($dateStr);
+ if($date === false ) {
+ Log::warning("DateStatsController:extractYear bad format for date $dateStr");
+ }
+ return $date?$date->format("Y"):false;
+ }
+
+ private function processPeriod($periodStr, $count) {
+ $start = null;
+ $end = null;
+ foreach(explode(";", $periodStr) as $elem) {
+ $elem = trim($elem);
+ if(strpos($elem, 'start=') === 0) {
+ $start = intval($this->extractYear(trim(substr($elem, 6))));
+ if($start === false) {
+ return [];
+ }
+ } elseif(strpos($elem, 'end=') === 0) {
+ $end = intval($this->extractYear(trim(substr($elem, 4))));
+ if($end === false) {
+ return [];
+ }
+ }
+ }
+
+ if(is_null($start) || is_null($end) || $start>$end ) {
+ // TODO: log problem
+ return [];
+ }
+
+ $res = [];
+ $mean = (int)($count/($end+1-$start));
+ $remains = $count%($end+1-$start);
+ for($d=$start; $d<=$end; $d++) {
+ $nb = $mean + ((($remains--)>0)?1:0);
+ if($nb !== 0) {
+ $res[strval($d)] = $nb;
+ }
+ }
+
+ return $res;
+ }
+
+ private function processDate($dateStr, $count) {
+ $date = $this->extractYear($dateStr);
+ if($date === false) {
+ return [];
+ } else {
+ return [ $this->extractYear($dateStr) => $count ];
+ }
+ }
+
+
+}
--- a/server/src/app/Http/Controllers/Api/DiscourseController.php Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Http/Controllers/Api/DiscourseController.php Fri Sep 30 00:43:04 2016 +0200
@@ -2,7 +2,6 @@
namespace CorpusParole\Http\Controllers\Api;
-// use Illuminate\Http\Request;
// use CorpusParole\Http\Requests;
use CorpusParole\Http\Controllers\Controller;
@@ -31,10 +30,10 @@
public function index(Request $request)
{
- $query = preg_replace('/\s+/', ' ', "select (?o as ?res) (COUNT(?s) as ?count) where {
+ $query = preg_replace('/\s+/', ' ', "SELECT (?o AS ?res) (COUNT(?s) AS ?count) WHERE {
?s a <http://www.europeana.eu/schemas/edm/ProvidedCHO>.
?s <http://purl.org/dc/elements/1.1/type> ?o.
- filter(uri(?o) in (<".implode('>,<', array_keys(config('corpusparole.corpus_discourse_type'))).">))
+ FILTER(uri(?o) in (<".implode('>,<', array_keys(config('corpusparole.corpus_discourse_type'))).">))
}
GROUP BY ?o
ORDER BY DESC(?count)");
--- a/server/src/app/Libraries/RdfModel/RdfModelDelta.php Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Libraries/RdfModel/RdfModelDelta.php Fri Sep 30 00:43:04 2016 +0200
@@ -31,7 +31,7 @@
return $this->deleteWhere;
}
- public function addDeleteWhere(string $value) {
+ public function addDeleteWhere($value) {
array_push($this->deleteWhere, $value);
return $this;
}
--- a/server/src/app/Libraries/Sparql/SparqlClient.php Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Libraries/Sparql/SparqlClient.php Fri Sep 30 00:43:04 2016 +0200
@@ -128,7 +128,7 @@
return $this->updateData('DELETE', $graph);
}
- public function deleteWhere($whereClauses, string $graphUri = null) {
+ public function deleteWhere($whereClauses, $graphUri = null) {
if(empty($whereClauses)) {
return;
--- a/server/src/app/Libraries/Transcript/TranscriptConverterBase.php Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Libraries/Transcript/TranscriptConverterBase.php Fri Sep 30 00:43:04 2016 +0200
@@ -9,7 +9,7 @@
abstract class TranscriptConverterBase implements Transcriptconverterinterface {
- public function __construct(Document $document, string $source, string $creationDate = null) {
+ public function __construct(Document $document, $source, $creationDate = null) {
$this->resJSON = [];
$this->document = $document;
--- a/server/src/routes/api.php Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/routes/api.php Fri Sep 30 00:43:04 2016 +0200
@@ -38,5 +38,7 @@
['only' => ['index']]);
Route::resource('discourses', 'Api\DiscourseController',
['only' => ['index']]);
+ Route::resource('datestats', 'Api\DateStatsController',
+ ['only' => ['index']]);
});
-});
\ No newline at end of file
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/server/src/tests/Controllers/DateStatsControllerTest.php Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,268 @@
+<?php
+
+use Mockery as m;
+
+use EasyRdf\Literal;
+
+class DateStatsControllerTest extends TestCase
+{
+ private $sparqlClient;
+
+ public function setUp() {
+
+ parent::setup();
+
+ // create a mock of the post repository interface and inject it into the
+ // IoC container
+ $this->sparqlClient = m::mock('CorpusParole\Libraries\Sparql\SparqlClient');
+ $this->app->instance('CorpusParole\Libraries\Sparql\SparqlClient', $this->sparqlClient);
+ }
+
+ public function tearDown() {
+ m::close();
+ parent::tearDown();
+ }
+
+
+ public function testIndexQuery() {
+
+ $query = preg_replace('/\s+/', ' ', "SELECT (?d as ?date) (COUNT(?d) AS ?count)
+ WHERE {
+ ?_ a <http://www.europeana.eu/schemas/edm/ProvidedCHO>.
+ ?_ <http://purl.org/dc/terms/created> ?d
+ }
+ GROUP BY ?d
+ ORDER BY ?d");
+
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->with($query)
+ ->once()
+ ->andReturn(new \ArrayIterator([]));
+ $this->get('/api/v1/stats/datestats/');
+ $this->seeJsonEquals(["datestats" => []]);
+ }
+
+ public function testIndexMultiple() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('1975', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ (object)['date'=>new Literal('1965', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(2)],
+ (object)['date'=>new Literal('1955', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(3)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 3,
+ "1965" => 2,
+ "1975" => 1,
+ ]]);
+ }
+
+ public function testIndexSimple() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('1955', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ (object)['date'=>new Literal('1965', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ (object)['date'=>new Literal('1975', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 1,
+ "1965" => 1,
+ "1975" => 1,
+ ]]);
+ }
+
+ public function testIndexPeriod() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 1,
+ "1956" => 1,
+ "1957" => 1,
+ "1958" => 1,
+ "1959" => 1,
+ "1960" => 1,
+ "1961" => 1,
+ "1962" => 1,
+ "1963" => 1,
+ "1964" => 1,
+ "1965" => 1,
+ ]]);
+ }
+
+ public function testIndexPeriodRemainMore() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(15)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 2,
+ "1956" => 2,
+ "1957" => 2,
+ "1958" => 2,
+ "1959" => 1,
+ "1960" => 1,
+ "1961" => 1,
+ "1962" => 1,
+ "1963" => 1,
+ "1964" => 1,
+ "1965" => 1,
+ ]]);
+ }
+
+ public function testIndexPeriodRemainLess() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(10)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 1,
+ "1956" => 1,
+ "1957" => 1,
+ "1958" => 1,
+ "1959" => 1,
+ "1960" => 1,
+ "1961" => 1,
+ "1962" => 1,
+ "1963" => 1,
+ "1964" => 1,
+ ]]);
+ }
+
+ public function testIndexMix() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+ (object)['date'=>new Literal('1960', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(2)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 1,
+ "1956" => 1,
+ "1957" => 1,
+ "1958" => 1,
+ "1959" => 1,
+ "1960" => 3,
+ "1961" => 1,
+ "1962" => 1,
+ "1963" => 1,
+ "1964" => 1,
+ "1965" => 1,
+ ]]);
+ }
+
+ public function testIndexBadDate() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('1955', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ (object)['date'=>new Literal('HELLO', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ (object)['date'=>new Literal('1975', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 1,
+ "1975" => 1,
+ ]]);
+ }
+
+ public function testIndexBadPeriod() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955; end=FOO', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ ]]);
+ }
+
+ public function testIndexBadPeriodMissing() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ ]]);
+ }
+
+ public function testIndexFullPeriod() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('start=1955; end=1965; scheme=v3; name=v4;', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1955" => 1,
+ "1956" => 1,
+ "1957" => 1,
+ "1958" => 1,
+ "1959" => 1,
+ "1960" => 1,
+ "1961" => 1,
+ "1962" => 1,
+ "1963" => 1,
+ "1964" => 1,
+ "1965" => 1,
+ ]]);
+ }
+
+ public function testIndexMultipleFormat() {
+
+ $this->sparqlClient
+ ->shouldReceive('query')
+ ->once()
+ ->andReturn(new \ArrayIterator([
+ (object)['date'=>new Literal('1975-02-05', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+ (object)['date'=>new Literal('1965-03', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(2)],
+ (object)['date'=>new Literal('1955-02-12T08:30:00+00:00', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(3)],
+ (object)['date'=>new Literal('1950-08-18T08:30:00Z', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(4)],
+ ]));
+ $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+ $this->seeJsonEquals(["datestats" => [
+ "1950" => 4,
+ "1955" => 3,
+ "1965" => 2,
+ "1975" => 1,
+ ]]);
+ }
+
+
+
+}