# HG changeset patch
# User ymh <ymh.work@gmail.com>
# Date 1475188984 -7200
# Node ID 07b44a378ad8279218cfd437703f8a60ae7c4db3
# Parent  3fccf43160a71d84f911f1c973bb095d5f545631
Add the datestats api

diff -r 3fccf43160a7 -r 07b44a378ad8 cms/app-client/mirage/config.js
--- a/cms/app-client/mirage/config.js	Wed Sep 28 17:24:02 2016 +0200
+++ b/cms/app-client/mirage/config.js	Fri Sep 30 00:43:04 2016 +0200
@@ -38,6 +38,8 @@
 
     this.get('/stats/discourses', 'discourses');
 
+    this.get('/stats/datestats', 'datestats');
+
     this.get('/resolvers/lexvo/:ids', ({lexvos}, request) => {
         var langIds = decodeURIComponent(request.params.ids);
         var resMap = _.reduce(langIds.split(','), function(res, id) {
diff -r 3fccf43160a7 -r 07b44a378ad8 cms/app-client/mirage/fixtures/datestats.js
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cms/app-client/mirage/fixtures/datestats.js	Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,62 @@
+export default [
+  { 'id': "1948", "count": 3 },
+  { 'id': "1957", "count": 29 },
+  { 'id': "1958", "count": 39 },
+  { 'id': "1959", "count": 19 },
+  { 'id': "1960", "count": 3 },
+  { 'id': "1961", "count": 25 },
+  { 'id': "1962", "count": 34 },
+  { 'id': "1963", "count": 22 },
+  { 'id': "1964", "count": 35 },
+  { 'id': "1965", "count": 110 },
+  { 'id': "1966", "count": 11 },
+  { 'id': "1967", "count": 40 },
+  { 'id': "1968", "count": 43 },
+  { 'id': "1969", "count": 228 },
+  { 'id': "1970", "count": 405 },
+  { 'id': "1971", "count": 62 },
+  { 'id': "1972", "count": 89 },
+  { 'id': "1973", "count": 36 },
+  { 'id': "1974", "count": 18 },
+  { 'id': "1975", "count": 21 },
+  { 'id': "1976", "count": 34 },
+  { 'id': "1977", "count": 11 },
+  { 'id': "1978", "count": 15 },
+  { 'id': "1979", "count": 7 },
+  { 'id': "1980", "count": 551 },
+  { 'id': "1981", "count": 19 },
+  { 'id': "1982", "count": 13 },
+  { 'id': "1983", "count": 18 },
+  { 'id': "1984", "count": 20 },
+  { 'id': "1985", "count": 30 },
+  { 'id': "1986", "count": 14 },
+  { 'id': "1987", "count": 12 },
+  { 'id': "1988", "count": 17 },
+  { 'id': "1989", "count": 65 },
+  { 'id': "1990", "count": 37 },
+  { 'id': "1991", "count": 14 },
+  { 'id': "1992", "count": 46 },
+  { 'id': "1993", "count": 43 },
+  { 'id': "1994", "count": 39 },
+  { 'id': "1995", "count": 95 },
+  { 'id': "1996", "count": 37 },
+  { 'id': "1997", "count": 126 },
+  { 'id': "1998", "count": 42 },
+  { 'id': "1999", "count": 33 },
+  { 'id': "2000", "count": 13 },
+  { 'id': "2001", "count": 65 },
+  { 'id': "2002", "count": 31 },
+  { 'id': "2003", "count": 24 },
+  { 'id': "2004", "count": 29 },
+  { 'id': "2005", "count": 88 },
+  { 'id': "2006", "count": 17 },
+  { 'id': "2007", "count": 17 },
+  { 'id': "2008", "count": 74 },
+  { 'id': "2009", "count": 10 },
+  { 'id': "2010", "count": 162 },
+  { 'id': "2011", "count": 69 },
+  { 'id': "2012", "count": 26 },
+  { 'id': "2013", "count": 5 },
+  { 'id': "2014", "count": 1 },
+  { 'id': "2015", "count": 1 }
+];
diff -r 3fccf43160a7 -r 07b44a378ad8 cms/app-client/mirage/models/datestat.js
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cms/app-client/mirage/models/datestat.js	Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,4 @@
+import { Model } from 'ember-cli-mirage';
+
+export default Model.extend({
+});
diff -r 3fccf43160a7 -r 07b44a378ad8 cms/app-client/mirage/serializers/datestat.js
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cms/app-client/mirage/serializers/datestat.js	Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,9 @@
+import { JSONAPISerializer } from 'ember-cli-mirage';
+
+import _ from 'lodash/lodash';
+
+export default JSONAPISerializer.extend({
+    serialize(response) {
+        return _(response.models).map((dateinfo) => { return [dateinfo.id, dateinfo.count];}).object().value();
+    }
+});
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/app/Http/Controllers/Api/DateStatsController.php
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/src/app/Http/Controllers/Api/DateStatsController.php	Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,126 @@
+<?php
+
+namespace CorpusParole\Http\Controllers\Api;
+
+// use CorpusParole\Http\Requests;
+use Illuminate\Http\Request;
+use Log;
+
+use CorpusParole\Libraries\Sparql\SparqlClient;
+
+use CorpusParole\Http\Controllers\Controller;
+
+
+class DateStatsController extends Controller
+{
+    private $sparqlClient = null;
+
+    public function __construct(SparqlClient $sparqlClient) {
+        $this->sparqlClient = $sparqlClient;
+    }
+
+    /**
+     * Display the specified resource.
+     *
+     * @return \Illuminate\Http\Response
+     */
+    public function index(Request $request)
+    {
+        $query =  preg_replace('/\s+/', ' ', "SELECT (?d as ?date) (COUNT(?d) AS ?count)
+            WHERE {
+                ?_ a <http://www.europeana.eu/schemas/edm/ProvidedCHO>.
+                ?_ <http://purl.org/dc/terms/created> ?d
+            }
+            GROUP BY ?d
+            ORDER BY ?d");
+
+        $res = $this->sparqlClient->query($query);
+
+        $dates = [];
+
+        foreach ($res as $row) {
+
+            $count = intval($row->count->getValue());
+            $date = $row->date;
+            $dateType = $date->getDatatypeUri();
+
+            $processedDates = [];
+            if($dateType === "http://purl.org/dc/terms/Period") {
+                $processedDates = $this->processPeriod($date->getValue(), $count);
+            }
+            elseif($dateType === "http://purl.org/dc/terms/W3CDTF") {
+                $processedDates = $this->processDate($date->getValue(), $count);
+            }
+
+            $dates = array_reduce(array_keys($processedDates), function($datesArray, $item) use ($processedDates) {
+                if(!isset($datesArray[$item])) {
+                    $datesArray[$item] = 0;
+                }
+                $datesArray[$item] += $processedDates[$item];
+                return $datesArray;
+            }, $dates);
+        }
+
+        ksort($dates);
+
+        return response()->json(['datestats' => $dates ]);
+    }
+
+    private function extractYear($dateStr) {
+        if(preg_match("/^\\d{4}$/", $dateStr) === 1) {
+            $dateStr = "$dateStr-1-1";
+        }
+        $date = date_create($dateStr);
+        if($date === false ) {
+            Log::warning("DateStatsController:extractYear bad format for date $dateStr");
+        }
+        return $date?$date->format("Y"):false;
+    }
+
+    private function processPeriod($periodStr, $count) {
+        $start = null;
+        $end = null;
+        foreach(explode(";", $periodStr) as $elem) {
+            $elem = trim($elem);
+            if(strpos($elem, 'start=') === 0) {
+                $start = intval($this->extractYear(trim(substr($elem, 6))));
+                if($start === false) {
+                    return [];
+                }
+            } elseif(strpos($elem, 'end=') === 0) {
+                $end = intval($this->extractYear(trim(substr($elem, 4))));
+                if($end === false) {
+                    return [];
+                }
+            }
+        }
+
+        if(is_null($start) || is_null($end) || $start>$end ) {
+            // TODO: log problem
+            return [];
+        }
+
+        $res = [];
+        $mean = (int)($count/($end+1-$start));
+        $remains = $count%($end+1-$start);
+        for($d=$start; $d<=$end; $d++) {
+            $nb = $mean + ((($remains--)>0)?1:0);
+            if($nb !== 0) {
+                $res[strval($d)] = $nb;
+            }
+        }
+
+        return $res;
+    }
+
+    private function processDate($dateStr, $count) {
+        $date = $this->extractYear($dateStr);
+        if($date === false)  {
+            return [];
+        } else {
+            return [ $this->extractYear($dateStr) => $count ];
+        }
+    }
+
+
+}
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/app/Http/Controllers/Api/DiscourseController.php
--- a/server/src/app/Http/Controllers/Api/DiscourseController.php	Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Http/Controllers/Api/DiscourseController.php	Fri Sep 30 00:43:04 2016 +0200
@@ -2,7 +2,6 @@
 
 namespace CorpusParole\Http\Controllers\Api;
 
-// use Illuminate\Http\Request;
 // use CorpusParole\Http\Requests;
 use CorpusParole\Http\Controllers\Controller;
 
@@ -31,10 +30,10 @@
     public function index(Request $request)
     {
 
-        $query =  preg_replace('/\s+/', ' ', "select (?o as ?res) (COUNT(?s) as ?count) where {
+        $query =  preg_replace('/\s+/', ' ', "SELECT (?o AS ?res) (COUNT(?s) AS ?count) WHERE {
             ?s a <http://www.europeana.eu/schemas/edm/ProvidedCHO>.
             ?s <http://purl.org/dc/elements/1.1/type> ?o.
-            filter(uri(?o) in (<".implode('>,<', array_keys(config('corpusparole.corpus_discourse_type'))).">))
+            FILTER(uri(?o) in (<".implode('>,<', array_keys(config('corpusparole.corpus_discourse_type'))).">))
         }
         GROUP BY ?o
         ORDER BY DESC(?count)");
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/app/Libraries/RdfModel/RdfModelDelta.php
--- a/server/src/app/Libraries/RdfModel/RdfModelDelta.php	Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Libraries/RdfModel/RdfModelDelta.php	Fri Sep 30 00:43:04 2016 +0200
@@ -31,7 +31,7 @@
         return $this->deleteWhere;
     }
 
-    public function addDeleteWhere(string $value) {
+    public function addDeleteWhere($value) {
         array_push($this->deleteWhere, $value);
         return $this;
     }
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/app/Libraries/Sparql/SparqlClient.php
--- a/server/src/app/Libraries/Sparql/SparqlClient.php	Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Libraries/Sparql/SparqlClient.php	Fri Sep 30 00:43:04 2016 +0200
@@ -128,7 +128,7 @@
         return $this->updateData('DELETE', $graph);
     }
 
-    public function deleteWhere($whereClauses, string $graphUri = null) {
+    public function deleteWhere($whereClauses, $graphUri = null) {
 
         if(empty($whereClauses)) {
             return;
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/app/Libraries/Transcript/TranscriptConverterBase.php
--- a/server/src/app/Libraries/Transcript/TranscriptConverterBase.php	Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/app/Libraries/Transcript/TranscriptConverterBase.php	Fri Sep 30 00:43:04 2016 +0200
@@ -9,7 +9,7 @@
 
 abstract class TranscriptConverterBase implements Transcriptconverterinterface {
 
-    public function __construct(Document $document, string $source, string $creationDate = null) {
+    public function __construct(Document $document, $source, $creationDate = null) {
         $this->resJSON = [];
         $this->document = $document;
 
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/routes/api.php
--- a/server/src/routes/api.php	Wed Sep 28 17:24:02 2016 +0200
+++ b/server/src/routes/api.php	Fri Sep 30 00:43:04 2016 +0200
@@ -38,5 +38,7 @@
                         ['only' => ['index']]);
         Route::resource('discourses', 'Api\DiscourseController',
                         ['only' => ['index']]);
+        Route::resource('datestats', 'Api\DateStatsController',
+                        ['only' => ['index']]);
     });
-});
\ No newline at end of file
+});
diff -r 3fccf43160a7 -r 07b44a378ad8 server/src/tests/Controllers/DateStatsControllerTest.php
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/src/tests/Controllers/DateStatsControllerTest.php	Fri Sep 30 00:43:04 2016 +0200
@@ -0,0 +1,268 @@
+<?php
+
+use Mockery as m;
+
+use EasyRdf\Literal;
+
+class DateStatsControllerTest extends TestCase
+{
+    private $sparqlClient;
+
+    public function setUp() {
+
+        parent::setup();
+
+        // create a mock of the post repository interface and inject it into the
+        // IoC container
+        $this->sparqlClient = m::mock('CorpusParole\Libraries\Sparql\SparqlClient');
+        $this->app->instance('CorpusParole\Libraries\Sparql\SparqlClient', $this->sparqlClient);
+    }
+
+    public function tearDown() {
+        m::close();
+        parent::tearDown();
+    }
+
+
+    public function testIndexQuery() {
+
+        $query =  preg_replace('/\s+/', ' ', "SELECT (?d as ?date) (COUNT(?d) AS ?count)
+            WHERE {
+                ?_ a <http://www.europeana.eu/schemas/edm/ProvidedCHO>.
+                ?_ <http://purl.org/dc/terms/created> ?d
+            }
+            GROUP BY ?d
+            ORDER BY ?d");
+
+
+        $this->sparqlClient
+            ->shouldReceive('query')
+            ->with($query)
+            ->once()
+            ->andReturn(new \ArrayIterator([]));
+        $this->get('/api/v1/stats/datestats/');
+        $this->seeJsonEquals(["datestats" => []]);
+    }
+
+    public function testIndexMultiple() {
+
+         $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('1975', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+                 (object)['date'=>new Literal('1965', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(2)],
+                 (object)['date'=>new Literal('1955', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(3)],
+             ]));
+         $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+         $this->seeJsonEquals(["datestats" => [
+             "1955" => 3,
+             "1965" => 2,
+             "1975" => 1,
+         ]]);
+    }
+
+    public function testIndexSimple() {
+
+         $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('1955', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+                 (object)['date'=>new Literal('1965', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+                 (object)['date'=>new Literal('1975', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+             ]));
+         $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+         $this->seeJsonEquals(["datestats" => [
+             "1955" => 1,
+             "1965" => 1,
+             "1975" => 1,
+         ]]);
+    }
+
+    public function testIndexPeriod() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+            "1955" => 1,
+            "1956" => 1,
+            "1957" => 1,
+            "1958" => 1,
+            "1959" => 1,
+            "1960" => 1,
+            "1961" => 1,
+            "1962" => 1,
+            "1963" => 1,
+            "1964" => 1,
+            "1965" => 1,
+        ]]);
+    }
+
+    public function testIndexPeriodRemainMore() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(15)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+            "1955" => 2,
+            "1956" => 2,
+            "1957" => 2,
+            "1958" => 2,
+            "1959" => 1,
+            "1960" => 1,
+            "1961" => 1,
+            "1962" => 1,
+            "1963" => 1,
+            "1964" => 1,
+            "1965" => 1,
+        ]]);
+    }
+
+    public function testIndexPeriodRemainLess() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(10)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+            "1955" => 1,
+            "1956" => 1,
+            "1957" => 1,
+            "1958" => 1,
+            "1959" => 1,
+            "1960" => 1,
+            "1961" => 1,
+            "1962" => 1,
+            "1963" => 1,
+            "1964" => 1,
+        ]]);
+    }
+
+    public function testIndexMix() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955; end=1965', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+                 (object)['date'=>new Literal('1960', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(2)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+            "1955" => 1,
+            "1956" => 1,
+            "1957" => 1,
+            "1958" => 1,
+            "1959" => 1,
+            "1960" => 3,
+            "1961" => 1,
+            "1962" => 1,
+            "1963" => 1,
+            "1964" => 1,
+            "1965" => 1,
+        ]]);
+    }
+
+    public function testIndexBadDate() {
+
+         $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('1955', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+                 (object)['date'=>new Literal('HELLO', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+                 (object)['date'=>new Literal('1975', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+             ]));
+         $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+         $this->seeJsonEquals(["datestats" => [
+             "1955" => 1,
+             "1975" => 1,
+         ]]);
+    }
+
+    public function testIndexBadPeriod() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955; end=FOO', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+        ]]);
+    }
+
+    public function testIndexBadPeriodMissing() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+        ]]);
+    }
+
+    public function testIndexFullPeriod() {
+
+        $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('start=1955; end=1965; scheme=v3; name=v4;', null, "http://purl.org/dc/terms/Period"), 'count' => Literal::create(11)],
+             ]));
+        $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+        $this->seeJsonEquals(["datestats" => [
+            "1955" => 1,
+            "1956" => 1,
+            "1957" => 1,
+            "1958" => 1,
+            "1959" => 1,
+            "1960" => 1,
+            "1961" => 1,
+            "1962" => 1,
+            "1963" => 1,
+            "1964" => 1,
+            "1965" => 1,
+        ]]);
+    }
+
+    public function testIndexMultipleFormat() {
+
+         $this->sparqlClient
+             ->shouldReceive('query')
+             ->once()
+             ->andReturn(new \ArrayIterator([
+                 (object)['date'=>new Literal('1975-02-05', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(1)],
+                 (object)['date'=>new Literal('1965-03', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(2)],
+                 (object)['date'=>new Literal('1955-02-12T08:30:00+00:00', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(3)],
+                 (object)['date'=>new Literal('1950-08-18T08:30:00Z', null, "http://purl.org/dc/terms/W3CDTF"), 'count' => Literal::create(4)],
+             ]));
+         $this->get('/api/v1/stats/datestats/')->assertTrue($this->response->isOk(), $this->response->content());
+         $this->seeJsonEquals(["datestats" => [
+             "1950" => 4,
+             "1955" => 3,
+             "1965" => 2,
+             "1975" => 1,
+         ]]);
+    }
+
+
+
+}