server/src/app/Console/Commands/IndexDocuments.php
author ymh <ymh.work@gmail.com>
Sun, 09 Oct 2016 11:44:18 +0200
changeset 325 31a4987f6017
parent 323 47f0611cc57d
child 326 226d5b17a119
permissions -rw-r--r--
Add fields to document index
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
     1
<?php
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
     2
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
     3
namespace CorpusParole\Console\Commands;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
     4
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
     5
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
     6
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
     7
use Illuminate\Console\Command;
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
     8
use EasyRdf\Resource;
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
     9
use EasyRdf\Literal;
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    10
use EasyRdf\Graph;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    11
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    12
use Carbon\Carbon;
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
    13
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    14
use GuzzleHttp\Client;
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    15
use GuzzleHttp\Exception\TransferException;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    16
use GuzzleHttp\Psr7;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    17
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
    18
use CorpusParole\Libraries\Utils;
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    19
use CorpusParole\Repositories\DocumentRepository;
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    20
use CorpusParole\Libraries\CocoonUtils;
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    21
use CorpusParole\Models\GeonamesHierarchy;
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
    22
use CorpusParole\Services\BnfResolverInterface;
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    23
use CorpusParole\Services\LexvoResolverInterface;
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    24
use Es;
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    25
use Log;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
    26
use Cache;
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    27
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    28
class IndexDocuments extends Command
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    29
{
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    30
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    31
    /**
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    32
     * The name and signature of the console command.
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    33
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    34
     * @var string
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    35
     */
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    36
    protected $signature = 'corpus-parole:indexDocuments
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    37
                          {--limit=0 : index only the first n documents, 0 (default) means index everything }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    38
                          {--no-bulk : index documents one by one instead of using ElasticSearch bulk indexing }
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    39
                          {--step-size=100 : number of documents to retrieve from repository at a time before indexing}
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
    40
                          {--reset-geo-cache : reset geo cache befr indexing}';
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    41
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    42
    /**
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    43
     * The console command description.
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    44
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    45
     * @var string
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    46
     */
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    47
    protected $description = 'Index documents into ElasticSearch.';
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    48
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    49
    /**
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    50
     * Create a new command instance.
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    51
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    52
     * @return void
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    53
     */
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    54
    public function __construct(
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    55
        DocumentRepository $documentRepository,
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    56
        Client $httpClient,
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    57
        BnfResolverInterface $bnfResolver,
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    58
        LexvoResolverInterface $lexvoResolver)
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    59
    {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    60
        $this->documentRepository = $documentRepository;
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
    61
        $this->bnfResolver = $bnfResolver;
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
    62
        $this->lexvoResolver = $lexvoResolver;
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    63
        $this->httpClient = $httpClient;
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    64
        parent::__construct();
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    65
    }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    66
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    67
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    68
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    69
    /**
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    70
     * Reset Elasticsearch index
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    71
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    72
     * @return int (1 if sucess, 0 if error)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    73
     */
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
    74
    private function resetIndex()
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    75
    {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    76
        $indexParams = [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    77
            'index' => env('ELASTICSEARCH_INDEX')
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    78
        ];
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    79
        if(Es::indices()->exists($indexParams)){
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    80
            $response = Es::indices()->delete($indexParams);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    81
            if($response['acknowledged']!=1){
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    82
                return 0;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    83
            }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    84
        }
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    85
        // Note: removed the "'store' => True" parameters on fields and use _source on record instead
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    86
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    87
        $indexParams['body'] = [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    88
            'settings' => [
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    89
                'number_of_shards' => config('elasticsearch.shards'),
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
    90
                'number_of_replicas' => config('elasticsearch.replicas'),
320
0fce13da58af filter geostat by area + tests
ymh <ymh.work@gmail.com>
parents: 308
diff changeset
    91
                'index.mapping.ignore_malformed' => True,
0fce13da58af filter geostat by area + tests
ymh <ymh.work@gmail.com>
parents: 308
diff changeset
    92
                'index.requests.cache.enable' => True
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    93
            ],
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    94
            'mappings' => [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    95
                'document' => [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    96
                    'properties' => [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    97
                        'title' => [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    98
                            'type' => 'string',
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
    99
                            'fields' => [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   100
                                'raw' => [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   101
                                    'type' => 'string',
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   102
                                    'index' => 'not_analyzed'
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   103
                                ]
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   104
                            ]
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   105
                        ],
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   106
                        'date' => [ 'type' => 'date' ],
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   107
                        'geonames_hyerarchy' => [ 'type' => 'string' ],
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   108
                        'location' => [ 'type' => 'geo_point' ],
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   109
                        'creation_date' => ['type' => 'date'],
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   110
                        'language' => ['type' => 'string'],
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   111
                        'discourse_types' => ['type' => 'string'],
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   112
                        'subject' => [
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   113
                            'type' => 'nested',
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   114
                            'properties' => [
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   115
                                'label' => [ 'type' => 'string', 'index' => 'not_analyzed'],
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   116
                                'code' => [ 'type' => 'string', 'index' => 'not_analyzed'],
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   117
                                'label_code' => [ 'type' => 'string', 'index' => 'not_analyzed']
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   118
                            ]
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   119
                        ]
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   120
                        // TODO: add location information
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   121
                    ]
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   122
                ]
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   123
            ]
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   124
        ];
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   125
        $response = Es::indices()->create($indexParams);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   126
        if($response['acknowledged']!=1){
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   127
            return 0;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   128
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   129
        return 1;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   130
    }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   131
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   132
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   133
    private function getGeonamesHierarchyArray($geonamesid) {
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   134
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   135
        $hcache = GeonamesHierarchy::where('geonamesid', $geonamesid)->first();
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   136
        if(is_null($hcache)) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   137
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   138
            // TODO: add delay to respect geonames 2k request/hour
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   139
            // TODO: manage errors
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   140
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   141
            $apiBody = $this->httpClient->get(
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   142
                config('corpusparole.geonames_hierarchy_webservice_url'),
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   143
                [ 'query' =>
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   144
                    [ 'geonameId' => $geonamesid,
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   145
                      'username' => config('corpusparole.geonames_username') ],
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   146
                  'accept' => 'application/json' // TODO: check this
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   147
                ]
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   148
            )->getBody();
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   149
            $hjson = json_decode($apiBody);
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   150
            $hcache = new GeonamesHierarchy();
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   151
            $hcache->geonamesid = $geonamesid;
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   152
            $hcache->hierarchy = $hjson;
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   153
            $hcache->save();
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   154
        }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   155
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   156
        $res = [];
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   157
        foreach($hcache->hierarchy['geonames'] as $hierarchyElem) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   158
            if(in_array($hierarchyElem['fcode'], ['CONT','PCLI', 'PCL','PCLD', 'PCLF', 'PCLH', 'PCLIX', 'PCLIS', 'ADM1'])) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   159
                array_push($res, $hierarchyElem['geonameId']);
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   160
            }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   161
        }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   162
        return $res;
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   163
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   164
    }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   165
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   166
    /**
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   167
     * get geonames hierarchy data.
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   168
     * @return array list of geonames ids
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   169
     */
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   170
    private function getGeonamesHierarchy($doc) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   171
        $geoRes = $doc->getGeoInfo();
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   172
        if(is_null($geoRes)) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   173
            return [];
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   174
        }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   175
        // aggregate hierachy list from geonames results
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   176
        $res = [];
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   177
        foreach($geoRes->getGeonamesLocs() as $gurl) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   178
            $geonamesId = CocoonUtils::getGeonamesidFromUrl($gurl);
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   179
            if(is_null($geonamesId)) {
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   180
                continue;
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   181
            }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   182
            $hierarchyIds = $this->getGeonamesHierarchyArray($geonamesId);
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   183
            $res = array_unique(array_merge($res, $hierarchyIds));
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   184
        }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   185
        return $res;
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   186
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   187
    }
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   188
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   189
    /**
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   190
     * get subjects as { 'label': label, 'code': code } objects
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   191
     * Takes only into account the bnf subjects
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   192
     */
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   193
    private function getSubjects($doc) {
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   194
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   195
        $sres = array_reduce($doc->getSubjects(), function($res, $s) {
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   196
            $mBnf = [];
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   197
            $mLexvo = [];
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   198
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   199
            if($s instanceof Resource && preg_match(config('corpusparole.bnf_ark_url_regexp'), $s->getUri(), $mBnf) === 1) {
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   200
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   201
                array_push($res, [
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   202
                    'uri' => $mBnf[0],
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   203
                    'code' => $mBnf[1],
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   204
                    'type' => 'bnf'
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   205
                ]);
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   206
            } elseif($s instanceof Resource && preg_match(config('corpusparole.lexvo_url_regexp'), $s->getUri(), $mLexvo) === 1) {
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   207
                array_push($res, [
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   208
                    'uri' => $mLexvo[0],
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   209
                    'code' => $mLexvo[1],
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   210
                    'type' => 'lxv'
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   211
                ]);
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   212
            } elseif($s instanceof Literal && strpos($s->getDatatypeUri(), config('corpusparole.olac_base_url')) === 0 ) {
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   213
                array_push($res, [
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   214
                    'uri' => $s->getValue(),
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   215
                    'code' => $s->getValue(),
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   216
                    'type' => 'olac'
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   217
                ]);
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   218
            } elseif($s instanceof Literal) {
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   219
                array_push($res, [
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   220
                    'uri' => $s->getValue(),
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   221
                    'code' => $s->getValue(),
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   222
                    'type' => 'txt'
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   223
                ]);
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   224
            }
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   225
            return $res;
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   226
        }, []);
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   227
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   228
        $labelsBnf = $this->bnfResolver->getLabels(
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   229
            array_unique(array_reduce(
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   230
                $sres,
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   231
                function($r, $so) {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   232
                    if($so['type'] === 'bnf') {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   233
                        array_push($r, $so['uri']);
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   234
                    }
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   235
                    return $r;
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   236
                },[]
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   237
            ))
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   238
        );
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   239
        $labelsLexvo = $this->lexvoResolver->getNames(
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   240
            array_unique(array_reduce(
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   241
                $sres,
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   242
                function($r, $so) {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   243
                    if($so['type'] === 'lxv') {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   244
                        array_push($r, $so['uri']);
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   245
                    }
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   246
                    return $r;
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   247
                },[]
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   248
            ))
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   249
        );
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   250
322
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   251
        return array_map(function($so) use ($labelsBnf, $labelsLexvo) {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   252
            $label = $so['uri'];
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   253
            if($so['type'] === 'bnf') {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   254
                $label = $labelsBnf[$label];
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   255
            } elseif ($so['type'] === 'lxv') {
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   256
                $label = $labelsLexvo[$label];
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   257
            }
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   258
            return [ 'label' => $label, 'code' => $so['code'], 'label_code' =>  $label."|".$so['type']."|".$so['code'] ]; }, $sres
084aae09edf4 correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents: 321
diff changeset
   259
        );
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   260
    }
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   261
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   262
    private function graphResolvCoordinate($loc, $graph) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   263
        $latLit = $graph->getLiteral($loc, "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   264
        if(is_null($latLit) || empty($latLit->getValue())) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   265
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   266
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   267
        $lat = $latLit->getValue();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   268
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   269
        $longLit = $graph->getLiteral($loc, "<http://www.w3.org/2003/01/geo/wgs84_pos#long>");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   270
        if(is_null($longLit) || empty($longLit->getValue())) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   271
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   272
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   273
        $long = $longLit->getValue();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   274
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   275
        return [ $lat, $long ];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   276
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   277
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   278
    private function loadGraph($url, $type) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   279
        try {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   280
            $r = $this->httpClient->get($url);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   281
        } catch (TransferException $e) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   282
            $this->error("loadGraph : Error Loading $url");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   283
            Log::error("loadGraph : Error Loading $url");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   284
            Log::error("loadGraph : Error request " . Psr7\str($e->getRequest()));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   285
            if ($e->hasResponse()) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   286
                $this->error("loadGraph : Error response " . Psr7\str($e->getResponse()));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   287
                Log::error("loadGraph : Error response " . Psr7\str($e->getResponse()));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   288
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   289
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   290
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   291
        try {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   292
            $message = (string)$r->getBody();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   293
            $graph = new Graph($url, $message, $type);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   294
            return $graph;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   295
        } catch (EasyRdf\Exception $e) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   296
            $this->error("loadGraph : Error parsing $url");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   297
            Log::error("loadGraph : Error parsing $url");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   298
            if($e instanceof EasyRdf\Parser\Exception) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   299
                Log::error("loadGraph : Error exception line ".$e->getLine().", column: ".$e->getColumn());
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   300
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   301
            $this->error("loadGraph : Error exception message ".$e->getMessage());
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   302
            Log::error("loadGraph : Error exception message ".$e->getMessage());
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   303
            Log::error("loadGraph : Error content $message");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   304
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   305
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   306
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   307
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   308
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   309
    private function geonamesResolveCoordinates($loc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   310
        $coords = cache("corpus.geonames.coord.$loc");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   311
        if(is_null($coords)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   312
            $graph = $this->loadGraph("{$loc}about.rdf", 'rdfxml');
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   313
            $coords = is_null($graph)?null:$this->graphResolvCoordinate($loc, $graph);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   314
            cache(["corpus.geonames.coord.$loc" => is_null($coords)?false:$coords], Carbon::now()->addMinutes(20));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   315
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   316
        return ($coords===false)?null:$coords;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   317
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   318
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   319
    private function dbpediaResolveCoordinates($loc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   320
        $coords = cache("corpus.dbpedia.coord.$loc");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   321
        if(is_null($coords)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   322
            $graph = $this->loadGraph("$loc.rdf", 'rdfxml');
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   323
            $coords = is_null($graph)?null:$this->graphResolvCoordinate($loc, $graph);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   324
            cache(["corpus.dbpedia.coord.$loc"=> is_null($coords)?false:$coords], Carbon::now()->addMinutes(20));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   325
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   326
        return ($coords===false)?null:$coords;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   327
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   328
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   329
    private function getLocation($doc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   330
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   331
        $geoRes = $doc->getGeoInfo();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   332
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   333
        if(is_null($geoRes)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   334
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   335
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   336
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   337
        $locUrls = [];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   338
        foreach($geoRes->getRefLocs() as $loc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   339
            if(preg_match(config('corpusparole.geonames_url_regexp'), $loc, $m) === 1) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   340
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   341
                if(!array_key_exists('geonames', $locUrls)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   342
                    $locUrls['geonames'] = [];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   343
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   344
                array_push($locUrls['geonames'], "http://sws.geonames.org/$m[1]/");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   345
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   346
            } elseif(preg_match(config('corpusparole.dbpedia_url_regexp'), $loc, $md) === 1) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   347
                if(!array_key_exists('dbpedia', $locUrls)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   348
                    $locUrls['dbpedia'] = [];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   349
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   350
                //$this->line("DBPEDIA MATCH $loc ".print_r($md,true));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   351
                array_push($locUrls['dbpedia'], "http://$md[1]/data/$md[4]");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   352
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   353
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   354
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   355
        $coordinates = null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   356
        foreach($locUrls as $locType => $locList) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   357
            foreach($locList as $locationUrl) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   358
                $coordinates = call_user_func([$this, "${locType}ResolveCoordinates"], $locationUrl);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   359
                if(!is_null($coordinates)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   360
                    break;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   361
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   362
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   363
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   364
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   365
        if(is_null($coordinates)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   366
            $coordinates = [$geoRes->getLatitudeValue(), $geoRes->getLongitudeValue()];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   367
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   368
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   369
        if(empty($coordinates[0]) || empty($coordinates[1])) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   370
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   371
        } else {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   372
            return [floatval($coordinates[0]), floatval($coordinates[1])];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   373
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   374
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   375
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   376
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   377
    private function getCreationDate($doc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   378
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   379
        $created = $doc->getCreated();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   380
        if(is_null($created)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   381
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   382
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   383
        $dateType = $created->getDatatypeUri();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   384
        $res = null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   385
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   386
        if($dateType === "http://purl.org/dc/terms/Period") {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   387
            $res = $this->processPeriod($created->getValue());
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   388
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   389
        elseif($dateType === "http://purl.org/dc/terms/W3CDTF") {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   390
            $res = $this->processDate($created->getValue());
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   391
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   392
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   393
        return $res;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   394
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   395
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   396
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   397
    private function extractDate($dateStr) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   398
        if(preg_match("/^\\d{4}$/", $dateStr) === 1) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   399
            $dateStr = "$dateStr-1-1";
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   400
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   401
        $date = date_create($dateStr);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   402
        if($date === false ) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   403
            Log::warning("DateStatsController:extractYear bad format for date $dateStr");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   404
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   405
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   406
        return $date;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   407
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   408
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   409
    private function processPeriod($periodStr) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   410
        $start = null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   411
        $end = null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   412
        foreach(explode(";", $periodStr) as $elem) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   413
            $elem = trim($elem);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   414
            if(strpos($elem, 'start=') === 0) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   415
                $startDate = $this->extractDate(trim(substr($elem, 6)));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   416
                if(is_null($startDate)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   417
                    return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   418
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   419
                $start = intval($startDate->format("Y"));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   420
                if($start === false) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   421
                    return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   422
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   423
            } elseif(strpos($elem, 'end=') === 0) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   424
                $endDate = $this->extractDate(trim(substr($elem, 4)));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   425
                if(is_null($endDate)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   426
                    return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   427
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   428
                $end = intval($endDate->format("Y"));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   429
                if($end === false) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   430
                    return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   431
                }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   432
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   433
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   434
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   435
        if(is_null($start) || is_null($end) || $start>$end ) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   436
            Log::warning("Bad format for $periodStr");
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   437
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   438
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   439
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   440
        return array_map(function($y) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   441
            return \DateTime::createFromFormat("Y", "$y")->format(\DateTime::W3C);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   442
        }, range($start, $end));
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   443
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   444
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   445
    private function processDate($dateStr) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   446
        $date = $this->extractDate($dateStr);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   447
        if(is_null($date))  {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   448
            return null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   449
        } else {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   450
            return $date->format(\DateTime::W3C);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   451
        }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   452
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   453
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   454
    private function getDiscourseTypes($doc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   455
        return array_reduce($doc->getDiscourseTypes(), function($res, $d) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   456
            $val = null;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   457
            if($d instanceof Resource) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   458
                $val = $d->getUri();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   459
            } elseif($d instanceof Literal) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   460
                $datatype = $d->getDatatypeURI();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   461
                $val = (!empty($datatype)?"$datatype#":"").$d->getValue();
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   462
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   463
            if(!empty($val)) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   464
                array_push($res,$val);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   465
            }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   466
            return $res;
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   467
        }, []);
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   468
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   469
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   470
    private function getDocBody($doc) {
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   471
        return [
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   472
            'title' => (string)$doc->getTitle(),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   473
            'date' => (string)$doc->getModified(),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   474
            'location' => $this->getLocation($doc),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   475
            'creation_date' => $this->getCreationDate($doc),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   476
            'language' => $doc->getLanguageValue(),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   477
            'discourse_types' => $this->getDiscourseTypes($doc),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   478
            'geonames_hierarchy' => $this->getGeonamesHierarchy($doc),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   479
            'subject' => $this->getSubjects($doc),
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   480
        ];
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   481
    }
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   482
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   483
    /**
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   484
     * Index one document into Elasticsearch
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   485
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   486
     * @return int (1 if sucess, 0 if error)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   487
     */
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   488
    private function indexOne($resultDoc)
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   489
    {
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   490
        $doc = $this->documentRepository->get($resultDoc->getId());
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   491
        $query_data = [
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   492
            'index' => config('elasticsearch.index'),
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   493
            'type' => 'document',
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   494
            'id' => (string)$doc->getId(),
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   495
            'body' => $this->getDocBody($doc)
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   496
        ];
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   497
        Es::index($query_data);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   498
    }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   499
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   500
    /**
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   501
     * Index multiple document into Elasticsearch
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   502
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   503
     * @return int (1 if sucess, 0 if error)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   504
     */
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   505
     private function indexBulk($docs)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   506
     {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   507
          $query_data = ['body' => []];
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   508
          foreach($docs as $resultDoc){
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   509
              $doc = $this->documentRepository->get($resultDoc->getId());
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   510
              $query_data['body'][] = [
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   511
                  'index' => [
308
e032d686d88e add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents: 25
diff changeset
   512
                      '_index' => config('elasticsearch.index'),
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   513
                      '_type' => 'document',
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   514
                      '_id' => (string)$doc->getId()
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   515
                  ]
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   516
              ];
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   517
              $query_data['body'][] = $this->getDocBody($doc);
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   518
          }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   519
          Es::bulk($query_data);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   520
     }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   521
    /**
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   522
     * Execute the console command.
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   523
     *
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   524
     * @return mixed
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   525
     */
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   526
    public function handle()
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   527
    {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   528
        $this->info('Options:');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   529
        $noBulk = $this->option('no-bulk');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   530
        if ($noBulk)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   531
        {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   532
            $this->comment(' - Indexing without bulk insert');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   533
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   534
        else
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   535
        {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   536
            $this->comment(' - Indexing using bulk insert');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   537
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   538
        $limit = $this->option('limit');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   539
        if ($limit>0) {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   540
            $this->comment(' - Indexing only the first '.$limit.' documents');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   541
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   542
        $stepSize = $this->option('step-size');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   543
        $this->comment(' - Indexing with step size of '.$stepSize);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   544
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   545
        if($this->option('reset-geo-cache', false)) {
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   546
            // delete all rows in GeonamesHierarchy
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   547
            GeonamesHierarchy::getQuery()->delete();
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   548
            $this->comment('Geonames cache reset!');
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   549
        }
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   550
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   551
        $this->info('Resetting index...');
323
47f0611cc57d correct theme controller
ymh <ymh.work@gmail.com>
parents: 322
diff changeset
   552
        $success = $this->resetIndex();
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   553
        if($success==1){
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   554
            $this->comment('Index reset!');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   555
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   556
        else{
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   557
            $this->error('Error resetting index ' . env('ELASTICSEARCH_INDEX'));
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   558
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   559
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   560
        $this->info('Indexing documents...');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   561
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   562
        if ($limit<=0) {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   563
            $lastPage = $this->documentRepository->paginateAll($stepSize, 'page')->lastPage();
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   564
            $total = $this->documentRepository->getCount();
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   565
            $lastPageEntryCount = $stepSize+1;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   566
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   567
        else {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   568
            $lastPage = min((int)($limit/$stepSize)+1, $this->documentRepository->paginateAll($stepSize, 'page')->lastPage());
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   569
            $total = $limit;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   570
            $lastPageEntryCount = $limit % $stepSize;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   571
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   572
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   573
        if ($noBulk)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   574
        {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   575
            $progressBar = $this->output->createProgressBar($total);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   576
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   577
        else
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   578
        {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   579
            $progressBar = $this->output->createProgressBar($lastPage);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   580
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   581
        $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% - %message%');
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   582
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   583
        for ($page=1;$page<=$lastPage;$page++)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   584
        {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   585
            $docs = $this->documentRepository->paginateAll($stepSize, 'page', $page);
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   586
            if ($noBulk)
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   587
            {
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   588
                foreach ($docs as $i=>$doc){
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   589
                    if ($page==$lastPage && $i>=$lastPageEntryCount){
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   590
                        break;
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   591
                    }
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   592
                    $progressBar->setMessage($doc->getId());
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   593
                    $progressBar->advance();
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   594
                    $this->indexOne($doc);
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   595
                }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   596
            }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   597
            else
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   598
            {
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   599
                $progressBar->setMessage('Page '.$page);
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   600
                $progressBar->advance();
325
31a4987f6017 Add fields to document index
ymh <ymh.work@gmail.com>
parents: 323
diff changeset
   601
                $this->indexBulk($docs);
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   602
            }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   603
        }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   604
        $progressBar->finish();
321
aefaad270b9b reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents: 320
diff changeset
   605
        $this->info("\nIndexing completed");
24
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   606
    }
de47e8f66e8b Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff changeset
   607
}