add a way to build rpm for puppet files, correct elasticsearch provisioning, correct error on elasticsearch queries + tests
--- a/.hgignore Wed Nov 09 15:05:41 2016 +0100
+++ b/.hgignore Wed Nov 09 23:41:15 2016 +0100
@@ -53,7 +53,8 @@
^.*\.DS_Store
^build/\.vagrant
^build/root
-^build/.*\.rpm
+^build/root_puppet
+^build/dist/.*\.rpm
^dev2/
^common/corpus-common-addon/dist
^common/corpus-common-addon/tmp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/build/build_puppet.sh Wed Nov 09 23:41:15 2016 +0100
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash -l
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+VAGRANT_STARTED=false
+
+green="\x1B[0;32m" # '\e[1;32m' is too bright for white bg.
+blue="\x1B[1;34m"
+endColor="\x1B[0m"
+
+function echoblue() {
+ echo -e "${blue}$1${endColor}"
+}
+
+function install() {
+ pushd "$DIR"
+
+ echoblue "---> checking vagrant"
+ if vagrant status | grep -q -v "running"; then
+ echoblue "---> starting vagrant"
+ # if which virtualenvwrapper.sh > /dev/null 2>&1 ; then
+ # echoblue "---> activating ansible"
+ # source `which virtualenvwrapper.sh`
+ # workon ansible
+ # fi
+ vagrant up
+ VAGRANT_STARTED=true
+ # if type -t deactivate > /dev/null 2>&1 ; then
+ # deactivate
+ # fi
+ fi
+
+ echoblue "---> done"
+
+ popd > /dev/null
+
+ return 0
+}
+
+function usage() {
+ cat <<EOF
+Usage: $0 [-I] [-h]
+ -I : do not run install
+ -h : print this message
+EOF
+}
+
+environment=""
+do_install=true
+
+
+while getopts "Ih" opt; do
+ case $opt in
+ I) do_install=false ;;
+ h) usage; exit 0 ;;
+ \?) usage >&2; exit 1 ;;
+ esac
+done
+shift $((OPTIND-1))
+
+echo "do_install: $do_install"
+[[ "$do_install" == true ]] && echoblue "DO INSTALL" && install;
+
+pushd "$DIR"
+
+echoblue "---> cleaning build folder"
+rm -fr root-puppet
+
+echoblue "---> creating build folder"
+mkdir -p root-puppet/var/lib/puppet/provision
+mkdir -p root-puppet/etc/puppet/hiera
+
+echoblue "---> copying provision files"
+rsync --exclude='.git' --exclude='.hg*' --exclude='custom.yaml' -aviuPz ../dev/provisioning/ root-puppet/var/lib/puppet/provision/
+echoblue "---> copying provision files done"
+
+echoblue "---> copying template file"
+cp -a ../dev/provisioning/custom.yaml.tmpl root-puppet/etc/puppet/hiera
+echoblue "---> copying template files done"
+
+# :datadir: "."
+echoblue "---> add path in root-puppet/var/lib/puppet/provision/hiera.yaml"
+sed -i "" -E 's/^([[:space:]]+\:datadir\:).*$/\1 \"\/etc\/puppet\/hiera\"/' root-puppet/var/lib/puppet/provision/hiera.yaml
+echoblue "---> add path in root-puppet/var/lib/puppet/provision/hiera.yaml done"
+
+
+echoblue "---> building package"
+vagrant ssh -c "/vagrant/build_rpm_puppet.sh"
+echoblue "---> building package done"
+
+if [ "$VAGRANT_STARTED" = true ]; then
+ echoblue "---> Stopping vagrant"
+ vagrant halt
+ echoblue "---> Stopping vagrant done"
+fi
+
+popd > /dev/null
+
+echoblue "---> done"
+
--- a/build/build_rpm.sh Wed Nov 09 15:05:41 2016 +0100
+++ b/build/build_rpm.sh Wed Nov 09 23:41:15 2016 +0100
@@ -20,7 +20,7 @@
mkdir -p /vagrant/dist
pushd /vagrant/dist > /dev/null
echo "---> Launching packaging"
-rm -f *.rpm
+rm -f corpusdelaparole*.rpm
version=$(grep -oP "\'version\'\s*=>\s*\'\K[\.0-9]+(?=\')" /vagrant/root/var/www/corpusdelaparole/corpus-back/config/version.php)
version=${version:-0.0.0}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/build/build_rpm_puppet.sh Wed Nov 09 23:41:15 2016 +0100
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+mkdir -p /vagrant/dist
+pushd /vagrant/dist > /dev/null
+echo "---> Launching packaging for puppet"
+rm -f puppet-corpusdelaparole*.rpm
+rm -f bootstrap-puppet.sh
+
+version=$(grep -oP "\'version\'\s*=>\s*\'\K[\.0-9]+(?=\')" /vagrant/root/var/www/corpusdelaparole/corpus-back/config/version.php)
+version=${version:-0.0.0}
+
+fpm \
+ --verbose \
+ -a all \
+ -s dir \
+ -t rpm \
+ -C /vagrant/root-puppet \
+ -d puppet \
+ -d puppetlabs-release \
+ --name puppet-corpusdelaparole \
+ --version "$version" \
+ --iteration $(date +%s) \
+ --description "corpus de la Parole : RPM package for puppet provisioning files" \
+ --url "http://corpusdelaparole.huma-num.fr" \
+ --rpm-user puppet \
+ --rpm-group puppet \
+ --directories /var/lib/puppet/provision \
+ .
+
+echo "---> Packaging for puppet done"
+echo "---> Copy puppet bootstrap"
+cp /vagrant/root-puppet/var/lib/puppet/provision/bootstrap-puppet.sh .
+
+popd > /dev/null
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dev/doc_agg_date.txt Wed Nov 09 23:41:15 2016 +0100
@@ -0,0 +1,29 @@
+{
+ "size": 0,
+ "query": { "match_all": [] },
+ "aggs": {
+ "datestats": {
+ "nested": {
+ "path": "creation_years"
+ },
+ "aggs": {
+ "years": {
+ "terms": {
+ "field": "creation_years.year",
+ "size": 0,
+ "order": {
+ "_term": "asc"
+ }
+ },
+ "aggs": {
+ "year_count": {
+ "sum": {
+ "field": "creation_years.weight"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
--- a/dev/provisioning/modules/sysconfig/manifests/elasticsearch.pp Wed Nov 09 15:05:41 2016 +0100
+++ b/dev/provisioning/modules/sysconfig/manifests/elasticsearch.pp Wed Nov 09 23:41:15 2016 +0100
@@ -7,6 +7,11 @@
notify {'elasticsearch': name => "\$es_host : ${es_host}, \$es_port : ${es_port}, \$es_instance : ${es_instance}", withpath => true }
+ exec { "systemd_restart_${es_instance}":
+ command => '/bin/systemctl restart elasticsearch-${es_instance}',
+ refreshonly => true,
+ }
+
class { '::elasticsearch':
manage_repo => true,
autoupgrade => true,
@@ -39,14 +44,12 @@
mode => '0750'
} ->
- file { '$es_instance service file':
+ file { '$es_instance service file':
path => "/usr/lib/systemd/system/elasticsearch-${es_instance}.service",
source => "puppet:///modules/sysconfig/elasticsearch/elasticsearch-${es_instance}.service",
owner => 'root',
group => 'root',
- mode => '0750'
-
- } ~>
- Exec['systemctl-daemon-reload']
-
+ mode => '0750',
+ notify => [Exec['systemctl-daemon-reload'], Exec["systemd_restart_${es_instance}"]]
+ }
}
--- a/readme.md Wed Nov 09 15:05:41 2016 +0100
+++ b/readme.md Wed Nov 09 23:41:15 2016 +0100
@@ -54,9 +54,16 @@
## Creation de version
- mettre à jour le numéro de version. Pour cela lancer le script `build/set-version.sh "x.y.z"`, avec "x.y.z" le numéro de version
+
+### Création du RPM contenant les fichiers `puppet`
+- lancer le script `build/build_puppet.sh`.
+- le fichier rpm se trouvera dans `/build/dist` sous le nom `puppet-corpusdelaparole-<version>-<iteration>.rpm`
+
+### Création du RPM de l'application
- lancer le script `build/build.sh`. options:
- `-d`: build debug
- `-p`: build de production
+- le fichier rpm se trouvera dans `/build/dist` sous le nom `corpusdelaparole-<version>-<iteration>.rpm`
- mettre à jour le fichier version `server/src/config/config/version.php`
- mettre à jour le fichier version `cms/app-client/package.json`
@@ -74,8 +81,9 @@
- `vagrant ssh -c "/vagrant/build_rpm.sh"`
- le fichier rpm se trouvera dans `/build/dist`
+
## installation rpm sur serveur
-- 1ère installation: `rpm -i /path/to/corpusdelaparole-<version>-<build>.rpm`
-- Mise à jour: `rpm -U /path/to/corpusdelaparole-<version>-<build>.rpm`
-- ensuite : création diu fichier `/etc/www/corpus_env.conf`
+- 1ère installation: `rpm -i /path/to/corpusdelaparole-<version>-<iteration>.rpm`
+- Mise à jour: `rpm -U /path/to/corpusdelaparole-<version>-<iteration>.rpm`
+- ensuite : création du fichier `/etc/www/corpus_env.conf`
- après un update : drush cc all
--- a/server/src/app/Console/Commands/IndexDocuments.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Console/Commands/IndexDocuments.php Wed Nov 09 23:41:15 2016 +0100
@@ -616,7 +616,7 @@
$progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% - %message%');
$page = 0;
- $lastPage = PHP_INT_MAX;
+ $lastPage = 2147483647;
$docIds = [];
while($page++<$lastPage) {
--- a/server/src/app/Http/Controllers/Api/DateStatsController.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Http/Controllers/Api/DateStatsController.php Wed Nov 09 23:41:15 2016 +0100
@@ -42,7 +42,7 @@
"years" => [
"terms"=> [
"field" => "creation_years.year",
- "size" => 0,
+ "size" => 2147483647,
"order" => [
"_term" => "asc"
]
--- a/server/src/app/Http/Controllers/Api/DiscourseController.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Http/Controllers/Api/DiscourseController.php Wed Nov 09 23:41:15 2016 +0100
@@ -36,7 +36,7 @@
"query" => $query,
"aggs" => [
"discourses" => [
- "terms" => [ "field" => "discourse_types", "order" => [ "_count" => "desc" ], "size" => 0 ]
+ "terms" => [ "field" => "discourse_types", "order" => [ "_count" => "desc" ], "size" => 2147483647 ]
]
]
]
--- a/server/src/app/Http/Controllers/Api/GeoStatsController.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Http/Controllers/Api/GeoStatsController.php Wed Nov 09 23:41:15 2016 +0100
@@ -47,7 +47,7 @@
"aggs" => [
"geos" => [
"terms" => [
- "size" => 0,
+ "size" => 2147483647,
"field" => "geonames_hierarchy"
]
]
--- a/server/src/app/Http/Controllers/Api/LanguageController.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Http/Controllers/Api/LanguageController.php Wed Nov 09 23:41:15 2016 +0100
@@ -31,7 +31,7 @@
"query" => $query,
"aggs" => [
"languages" => [
- "terms" => [ "field" => "language", "order" => [ "_count" => "desc" ], "size" => 0 ]
+ "terms" => [ "field" => "language", "order" => [ "_count" => "desc" ], "size" => 2147483647 ]
]
]
]
--- a/server/src/app/Http/Controllers/Api/ThemeController.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Http/Controllers/Api/ThemeController.php Wed Nov 09 23:41:15 2016 +0100
@@ -77,7 +77,7 @@
$qFilterParts = $filterManager->buildESFilters($filters);
$query = $filterManager->buildQuery($qFilterParts);
-
+ $termSize = $limit * ($index+1);
$esQuery = [
'index' => config('elasticsearch.index'),
'body' => [
@@ -90,7 +90,7 @@
"subjects" => [
"terms" => [
"field" => "subject.label_code",
- "size" => $limit * ($index+1),
+ "size" => ($termSize==0)?2147483647:$termSize,
"order" => [ $order_field => $order_dir ]
]
]
--- a/server/src/app/Libraries/Filters/CorpusFilterManager.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/app/Libraries/Filters/CorpusFilterManager.php Wed Nov 09 23:41:15 2016 +0100
@@ -328,7 +328,7 @@
public function buildQuery($qFilterParts) {
if(empty($qFilterParts)) {
- return [ "match_all" => []];
+ return [ "match_all" => (object) null ];
} else {
return ['constant_score' => [
'filter' => [
--- a/server/src/tests/Controllers/DateStatsControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/DateStatsControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -6,29 +6,34 @@
class DateStatsControllerTest extends TestCase
{
- const ES_QUERY = [
- 'index' => 'corpus',
- 'body' => [
- "size" => 0,
- "query" => [ "match_all" => [] ],
- "aggs" => [
- "datestats" => [
- "nested"=> [
- "path" => "creation_years"
- ],
- "aggs" => [
- "years" => [
- "terms"=> [
- "field" => "creation_years.year",
- "size" => 0,
- "order" => [
- "_term" => "asc"
- ]
- ],
- "aggs" => [
- "year_count" => [
- "sum" => [
- "field" => "creation_years.weight"
+ private $ES_QUERY;
+
+ public function setUp() {
+ parent::setup();
+ $this->ES_QUERY = [
+ 'index' => 'corpus',
+ 'body' => [
+ "size" => 0,
+ "query" => [ "match_all" => (object) null ],
+ "aggs" => [
+ "datestats" => [
+ "nested"=> [
+ "path" => "creation_years"
+ ],
+ "aggs" => [
+ "years" => [
+ "terms"=> [
+ "field" => "creation_years.year",
+ "size" => 2147483647,
+ "order" => [
+ "_term" => "asc"
+ ]
+ ],
+ "aggs" => [
+ "year_count" => [
+ "sum" => [
+ "field" => "creation_years.weight"
+ ]
]
]
]
@@ -36,12 +41,7 @@
]
]
]
- ]
- ];
-
- public function setUp() {
-
- parent::setup();
+ ];
}
public function tearDown() {
@@ -54,7 +54,7 @@
Es::shouldReceive('search')
->once()
- ->with(self::ES_QUERY)
+ ->with($this->ES_QUERY)
->andReturn(json_decode('{
"took" : 132,
"timed_out" : false,
@@ -87,7 +87,7 @@
Es::shouldReceive('search')
->once()
- ->with(self::ES_QUERY)
+ ->with($this->ES_QUERY)
->andReturn(json_decode('{
"took" : 132,
"timed_out" : false,
--- a/server/src/tests/Controllers/DiscourseControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/DiscourseControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -7,21 +7,21 @@
*/
class DiscourseControllerTest extends TestCase {
- const ES_QUERY = [
- 'index' => 'corpus',
- 'body' => [
- "size" => 0,
- "query" => [ "match_all" => [] ],
- "aggs" => [
- "discourses" => [
- "terms" => [ "field" => "discourse_types", "order" => [ "_count" => "desc" ], "size" => 0 ]
+ private $ES_QUERY;
+
+ public function setUp() {
+ $this->ES_QUERY = [
+ 'index' => 'corpus',
+ 'body' => [
+ "size" => 0,
+ "query" => [ "match_all" => (object) null ],
+ "aggs" => [
+ "discourses" => [
+ "terms" => [ "field" => "discourse_types", "order" => [ "_count" => "desc" ], "size" => 2147483647 ]
+ ]
]
]
- ]
- ];
-
- public function setUp() {
-
+ ];
parent::setup();
}
@@ -35,7 +35,7 @@
Es::shouldReceive('search')
->once()
- ->with(self::ES_QUERY)
+ ->with($this->ES_QUERY)
->andReturn(json_decode('{
"took" : 116,
"timed_out" : false,
@@ -80,7 +80,7 @@
public function testIndexQuery() {
Es::shouldReceive('search')
->once()
- ->with(self::ES_QUERY)
+ ->with($this->ES_QUERY)
->andReturn(json_decode('{
"took" : 116,
"timed_out" : false,
--- a/server/src/tests/Controllers/GeoStatsControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/GeoStatsControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -9,12 +9,12 @@
'body' => [
"size" => 0,
"query" => [
- 'match_all' => []
+ 'match_all' => (object) null
],
"aggs" => [
"geos" => [
"terms" => [
- "size" => 0,
+ "size" => 2147483647,
"field" => "geonames_hierarchy"
]
]
@@ -72,12 +72,12 @@
'body' => [
"size" => 0,
"query" => [
- 'match_all' => []
+ 'match_all' => (object) null
],
"aggs" => [
"geos" => [
"terms" => [
- "size" => 0,
+ "size" => 2147483647,
"field" => "geonames_hierarchy"
]
]
@@ -135,7 +135,7 @@
"aggs" => [
"geos" => [
"terms" => [
- "size" => 0,
+ "size" => 2147483647,
"field" => "geonames_hierarchy"
]
]
--- a/server/src/tests/Controllers/GeonamesControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/GeonamesControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -70,10 +70,10 @@
->shouldReceive('getLabels')
->with(['abcdef','ghij'])
->once()
- ->andThrow('CorpusParole\Services\GeonamesResolverException', "GeonamesId not in correct format", 400);
+ ->andThrow('CorpusParole\Services\GeonamesResolverException', "GeonamesId not in correct format", 500);
$response = $this->get('/api/v1/resolvers/geonames/abcdef,ghij');
- $this->assertResponseStatus(400);
+ $this->assertResponseStatus(500);
}
}
--- a/server/src/tests/Controllers/LanguageControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/LanguageControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -11,20 +11,21 @@
*/
class LanguageControllerTest extends TestCase {
- const ES_QUERY = [
- 'index' => 'corpus',
- 'body' => [
- "size" => 0,
- "query" => [ "match_all" => [] ],
- "aggs" => [
- "languages" => [
- "terms" => [ "field" => "language", "order" => [ "_count" => "desc" ], "size" => 0 ]
+ private $ES_QUERY;
+
+ public function setUp() {
+ $this->ES_QUERY = [
+ 'index' => 'corpus',
+ 'body' => [
+ "size" => 0,
+ "query" => [ "match_all" => (object) null ],
+ "aggs" => [
+ "languages" => [
+ "terms" => [ "field" => "language", "order" => [ "_count" => "desc" ], "size" => 2147483647 ]
+ ]
]
]
- ]
- ];
-
- public function setUp() {
+ ];
parent::setup();
}
@@ -37,7 +38,7 @@
Es::shouldReceive('search')
->once()
- ->with(self::ES_QUERY)
+ ->with($this->ES_QUERY)
->andReturn(json_decode('{
"took": 92,
"timed_out": false,
--- a/server/src/tests/Controllers/ThemeControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/ThemeControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -28,7 +28,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
@@ -120,7 +120,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
@@ -219,7 +219,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
@@ -286,7 +286,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
@@ -385,7 +385,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
@@ -473,7 +473,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
@@ -561,7 +561,7 @@
'body' => [
'size' => 0,
'query' =>[
- 'match_all' => []
+ 'match_all' => (object) null
],
'aggs' => [
"subjects" => [
--- a/server/src/tests/Controllers/ViafControllerTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Controllers/ViafControllerTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -75,10 +75,10 @@
->shouldReceive('getNames')
->with(['abcdef','ghij'])
->once()
- ->andThrow('CorpusParole\Services\ViafResolverException', "ViafId not in correct format", 400);
+ ->andThrow('CorpusParole\Services\ViafResolverException', "ViafId not in correct format", 500);
$response = $this->get('/api/v1/resolvers/viaf/abcdef,ghij');
- $this->assertResponseStatus(400);
+ $this->assertResponseStatus(500);
}
}
--- a/server/src/tests/Libraries/Sparql/SparqlClientTest.php Wed Nov 09 15:05:41 2016 +0100
+++ b/server/src/tests/Libraries/Sparql/SparqlClientTest.php Wed Nov 09 23:41:15 2016 +0100
@@ -86,7 +86,7 @@
$this->assertCount(1, $container, 'One request');
$req = $container[0]['request'];
- $this->assertEquals("http:{$this->rdf4jRepository}/transactions?isolation-level=http%3A%2F%2Fwww.openrdf.org%2Fschema%2Fsesame%23SNAPSHOT_READ", (string)$req->getUri(), "url must be ok");
+ $this->assertEquals("{$this->rdf4jRepository}/transactions?isolation-level=http%3A%2F%2Fwww.openrdf.org%2Fschema%2Fsesame%23SNAPSHOT_READ", (string)$req->getUri(), "url must be ok");
$this->assertEquals('POST', $container[0]['request']->getMethod(), "methos is POST");
}