author | ymh <ymh.work@gmail.com> |
Sun, 09 Oct 2016 11:44:18 +0200 | |
changeset 325 | 31a4987f6017 |
parent 323 | 47f0611cc57d |
child 326 | 226d5b17a119 |
permissions | -rw-r--r-- |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
1 |
<?php |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
2 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
3 |
namespace CorpusParole\Console\Commands; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
4 |
|
325 | 5 |
|
6 |
||
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
7 |
use Illuminate\Console\Command; |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
8 |
use EasyRdf\Resource; |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
9 |
use EasyRdf\Literal; |
325 | 10 |
use EasyRdf\Graph; |
11 |
||
12 |
use Carbon\Carbon; |
|
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
13 |
|
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
14 |
use GuzzleHttp\Client; |
325 | 15 |
use GuzzleHttp\Exception\TransferException; |
16 |
use GuzzleHttp\Psr7; |
|
17 |
||
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
18 |
use CorpusParole\Libraries\Utils; |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
19 |
use CorpusParole\Repositories\DocumentRepository; |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
20 |
use CorpusParole\Libraries\CocoonUtils; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
21 |
use CorpusParole\Models\GeonamesHierarchy; |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
22 |
use CorpusParole\Services\BnfResolverInterface; |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
23 |
use CorpusParole\Services\LexvoResolverInterface; |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
24 |
use Es; |
325 | 25 |
use Log; |
26 |
use Cache; |
|
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
27 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
28 |
class IndexDocuments extends Command |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
29 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
30 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
31 |
/** |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
32 |
* The name and signature of the console command. |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
33 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
34 |
* @var string |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
35 |
*/ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
36 |
protected $signature = 'corpus-parole:indexDocuments |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
37 |
{--limit=0 : index only the first n documents, 0 (default) means index everything } |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
38 |
{--no-bulk : index documents one by one instead of using ElasticSearch bulk indexing } |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
39 |
{--step-size=100 : number of documents to retrieve from repository at a time before indexing} |
323 | 40 |
{--reset-geo-cache : reset geo cache befr indexing}'; |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
41 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
42 |
/** |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
43 |
* The console command description. |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
44 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
45 |
* @var string |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
46 |
*/ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
47 |
protected $description = 'Index documents into ElasticSearch.'; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
48 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
49 |
/** |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
50 |
* Create a new command instance. |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
51 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
52 |
* @return void |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
53 |
*/ |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
54 |
public function __construct( |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
55 |
DocumentRepository $documentRepository, |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
56 |
Client $httpClient, |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
57 |
BnfResolverInterface $bnfResolver, |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
58 |
LexvoResolverInterface $lexvoResolver) |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
59 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
60 |
$this->documentRepository = $documentRepository; |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
61 |
$this->bnfResolver = $bnfResolver; |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
62 |
$this->lexvoResolver = $lexvoResolver; |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
63 |
$this->httpClient = $httpClient; |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
64 |
parent::__construct(); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
65 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
66 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
67 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
68 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
69 |
/** |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
70 |
* Reset Elasticsearch index |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
71 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
72 |
* @return int (1 if sucess, 0 if error) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
73 |
*/ |
323 | 74 |
private function resetIndex() |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
75 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
76 |
$indexParams = [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
77 |
'index' => env('ELASTICSEARCH_INDEX') |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
78 |
]; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
79 |
if(Es::indices()->exists($indexParams)){ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
80 |
$response = Es::indices()->delete($indexParams); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
81 |
if($response['acknowledged']!=1){ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
82 |
return 0; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
83 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
84 |
} |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
85 |
// Note: removed the "'store' => True" parameters on fields and use _source on record instead |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
86 |
|
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
87 |
$indexParams['body'] = [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
88 |
'settings' => [ |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
89 |
'number_of_shards' => config('elasticsearch.shards'), |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
90 |
'number_of_replicas' => config('elasticsearch.replicas'), |
320 | 91 |
'index.mapping.ignore_malformed' => True, |
92 |
'index.requests.cache.enable' => True |
|
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
93 |
], |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
94 |
'mappings' => [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
95 |
'document' => [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
96 |
'properties' => [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
97 |
'title' => [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
98 |
'type' => 'string', |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
99 |
'fields' => [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
100 |
'raw' => [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
101 |
'type' => 'string', |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
102 |
'index' => 'not_analyzed' |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
103 |
] |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
104 |
] |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
105 |
], |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
106 |
'date' => [ 'type' => 'date' ], |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
107 |
'geonames_hyerarchy' => [ 'type' => 'string' ], |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
108 |
'location' => [ 'type' => 'geo_point' ], |
325 | 109 |
'creation_date' => ['type' => 'date'], |
110 |
'language' => ['type' => 'string'], |
|
111 |
'discourse_types' => ['type' => 'string'], |
|
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
112 |
'subject' => [ |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
113 |
'type' => 'nested', |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
114 |
'properties' => [ |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
115 |
'label' => [ 'type' => 'string', 'index' => 'not_analyzed'], |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
116 |
'code' => [ 'type' => 'string', 'index' => 'not_analyzed'], |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
117 |
'label_code' => [ 'type' => 'string', 'index' => 'not_analyzed'] |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
118 |
] |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
119 |
] |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
120 |
// TODO: add location information |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
121 |
] |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
122 |
] |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
123 |
] |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
124 |
]; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
125 |
$response = Es::indices()->create($indexParams); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
126 |
if($response['acknowledged']!=1){ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
127 |
return 0; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
128 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
129 |
return 1; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
130 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
131 |
|
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
132 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
133 |
private function getGeonamesHierarchyArray($geonamesid) { |
325 | 134 |
|
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
135 |
$hcache = GeonamesHierarchy::where('geonamesid', $geonamesid)->first(); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
136 |
if(is_null($hcache)) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
137 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
138 |
// TODO: add delay to respect geonames 2k request/hour |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
139 |
// TODO: manage errors |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
140 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
141 |
$apiBody = $this->httpClient->get( |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
142 |
config('corpusparole.geonames_hierarchy_webservice_url'), |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
143 |
[ 'query' => |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
144 |
[ 'geonameId' => $geonamesid, |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
145 |
'username' => config('corpusparole.geonames_username') ], |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
146 |
'accept' => 'application/json' // TODO: check this |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
147 |
] |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
148 |
)->getBody(); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
149 |
$hjson = json_decode($apiBody); |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
150 |
$hcache = new GeonamesHierarchy(); |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
151 |
$hcache->geonamesid = $geonamesid; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
152 |
$hcache->hierarchy = $hjson; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
153 |
$hcache->save(); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
154 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
155 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
156 |
$res = []; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
157 |
foreach($hcache->hierarchy['geonames'] as $hierarchyElem) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
158 |
if(in_array($hierarchyElem['fcode'], ['CONT','PCLI', 'PCL','PCLD', 'PCLF', 'PCLH', 'PCLIX', 'PCLIS', 'ADM1'])) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
159 |
array_push($res, $hierarchyElem['geonameId']); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
160 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
161 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
162 |
return $res; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
163 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
164 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
165 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
166 |
/** |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
167 |
* get geonames hierarchy data. |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
168 |
* @return array list of geonames ids |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
169 |
*/ |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
170 |
private function getGeonamesHierarchy($doc) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
171 |
$geoRes = $doc->getGeoInfo(); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
172 |
if(is_null($geoRes)) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
173 |
return []; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
174 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
175 |
// aggregate hierachy list from geonames results |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
176 |
$res = []; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
177 |
foreach($geoRes->getGeonamesLocs() as $gurl) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
178 |
$geonamesId = CocoonUtils::getGeonamesidFromUrl($gurl); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
179 |
if(is_null($geonamesId)) { |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
180 |
continue; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
181 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
182 |
$hierarchyIds = $this->getGeonamesHierarchyArray($geonamesId); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
183 |
$res = array_unique(array_merge($res, $hierarchyIds)); |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
184 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
185 |
return $res; |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
186 |
|
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
187 |
} |
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
188 |
|
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
189 |
/** |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
190 |
* get subjects as { 'label': label, 'code': code } objects |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
191 |
* Takes only into account the bnf subjects |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
192 |
*/ |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
193 |
private function getSubjects($doc) { |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
194 |
|
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
195 |
$sres = array_reduce($doc->getSubjects(), function($res, $s) { |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
196 |
$mBnf = []; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
197 |
$mLexvo = []; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
198 |
|
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
199 |
if($s instanceof Resource && preg_match(config('corpusparole.bnf_ark_url_regexp'), $s->getUri(), $mBnf) === 1) { |
323 | 200 |
|
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
201 |
array_push($res, [ |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
202 |
'uri' => $mBnf[0], |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
203 |
'code' => $mBnf[1], |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
204 |
'type' => 'bnf' |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
205 |
]); |
323 | 206 |
} elseif($s instanceof Resource && preg_match(config('corpusparole.lexvo_url_regexp'), $s->getUri(), $mLexvo) === 1) { |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
207 |
array_push($res, [ |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
208 |
'uri' => $mLexvo[0], |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
209 |
'code' => $mLexvo[1], |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
210 |
'type' => 'lxv' |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
211 |
]); |
323 | 212 |
} elseif($s instanceof Literal && strpos($s->getDatatypeUri(), config('corpusparole.olac_base_url')) === 0 ) { |
213 |
array_push($res, [ |
|
214 |
'uri' => $s->getValue(), |
|
215 |
'code' => $s->getValue(), |
|
216 |
'type' => 'olac' |
|
217 |
]); |
|
218 |
} elseif($s instanceof Literal) { |
|
219 |
array_push($res, [ |
|
220 |
'uri' => $s->getValue(), |
|
221 |
'code' => $s->getValue(), |
|
222 |
'type' => 'txt' |
|
223 |
]); |
|
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
224 |
} |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
225 |
return $res; |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
226 |
}, []); |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
227 |
|
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
228 |
$labelsBnf = $this->bnfResolver->getLabels( |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
229 |
array_unique(array_reduce( |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
230 |
$sres, |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
231 |
function($r, $so) { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
232 |
if($so['type'] === 'bnf') { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
233 |
array_push($r, $so['uri']); |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
234 |
} |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
235 |
return $r; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
236 |
},[] |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
237 |
)) |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
238 |
); |
323 | 239 |
$labelsLexvo = $this->lexvoResolver->getNames( |
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
240 |
array_unique(array_reduce( |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
241 |
$sres, |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
242 |
function($r, $so) { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
243 |
if($so['type'] === 'lxv') { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
244 |
array_push($r, $so['uri']); |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
245 |
} |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
246 |
return $r; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
247 |
},[] |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
248 |
)) |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
249 |
); |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
250 |
|
322
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
251 |
return array_map(function($so) use ($labelsBnf, $labelsLexvo) { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
252 |
$label = $so['uri']; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
253 |
if($so['type'] === 'bnf') { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
254 |
$label = $labelsBnf[$label]; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
255 |
} elseif ($so['type'] === 'lxv') { |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
256 |
$label = $labelsLexvo[$label]; |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
257 |
} |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
258 |
return [ 'label' => $label, 'code' => $so['code'], 'label_code' => $label."|".$so['type']."|".$so['code'] ]; }, $sres |
084aae09edf4
correction on importRDF documents + evolution theme controller
ymh <ymh.work@gmail.com>
parents:
321
diff
changeset
|
259 |
); |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
260 |
} |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
261 |
|
325 | 262 |
private function graphResolvCoordinate($loc, $graph) { |
263 |
$latLit = $graph->getLiteral($loc, "<http://www.w3.org/2003/01/geo/wgs84_pos#lat>"); |
|
264 |
if(is_null($latLit) || empty($latLit->getValue())) { |
|
265 |
return null; |
|
266 |
} |
|
267 |
$lat = $latLit->getValue(); |
|
268 |
||
269 |
$longLit = $graph->getLiteral($loc, "<http://www.w3.org/2003/01/geo/wgs84_pos#long>"); |
|
270 |
if(is_null($longLit) || empty($longLit->getValue())) { |
|
271 |
return null; |
|
272 |
} |
|
273 |
$long = $longLit->getValue(); |
|
274 |
||
275 |
return [ $lat, $long ]; |
|
276 |
} |
|
277 |
||
278 |
private function loadGraph($url, $type) { |
|
279 |
try { |
|
280 |
$r = $this->httpClient->get($url); |
|
281 |
} catch (TransferException $e) { |
|
282 |
$this->error("loadGraph : Error Loading $url"); |
|
283 |
Log::error("loadGraph : Error Loading $url"); |
|
284 |
Log::error("loadGraph : Error request " . Psr7\str($e->getRequest())); |
|
285 |
if ($e->hasResponse()) { |
|
286 |
$this->error("loadGraph : Error response " . Psr7\str($e->getResponse())); |
|
287 |
Log::error("loadGraph : Error response " . Psr7\str($e->getResponse())); |
|
288 |
} |
|
289 |
return null; |
|
290 |
} |
|
291 |
try { |
|
292 |
$message = (string)$r->getBody(); |
|
293 |
$graph = new Graph($url, $message, $type); |
|
294 |
return $graph; |
|
295 |
} catch (EasyRdf\Exception $e) { |
|
296 |
$this->error("loadGraph : Error parsing $url"); |
|
297 |
Log::error("loadGraph : Error parsing $url"); |
|
298 |
if($e instanceof EasyRdf\Parser\Exception) { |
|
299 |
Log::error("loadGraph : Error exception line ".$e->getLine().", column: ".$e->getColumn()); |
|
300 |
} |
|
301 |
$this->error("loadGraph : Error exception message ".$e->getMessage()); |
|
302 |
Log::error("loadGraph : Error exception message ".$e->getMessage()); |
|
303 |
Log::error("loadGraph : Error content $message"); |
|
304 |
return null; |
|
305 |
} |
|
306 |
||
307 |
} |
|
308 |
||
309 |
private function geonamesResolveCoordinates($loc) { |
|
310 |
$coords = cache("corpus.geonames.coord.$loc"); |
|
311 |
if(is_null($coords)) { |
|
312 |
$graph = $this->loadGraph("{$loc}about.rdf", 'rdfxml'); |
|
313 |
$coords = is_null($graph)?null:$this->graphResolvCoordinate($loc, $graph); |
|
314 |
cache(["corpus.geonames.coord.$loc" => is_null($coords)?false:$coords], Carbon::now()->addMinutes(20)); |
|
315 |
} |
|
316 |
return ($coords===false)?null:$coords; |
|
317 |
} |
|
318 |
||
319 |
private function dbpediaResolveCoordinates($loc) { |
|
320 |
$coords = cache("corpus.dbpedia.coord.$loc"); |
|
321 |
if(is_null($coords)) { |
|
322 |
$graph = $this->loadGraph("$loc.rdf", 'rdfxml'); |
|
323 |
$coords = is_null($graph)?null:$this->graphResolvCoordinate($loc, $graph); |
|
324 |
cache(["corpus.dbpedia.coord.$loc"=> is_null($coords)?false:$coords], Carbon::now()->addMinutes(20)); |
|
325 |
} |
|
326 |
return ($coords===false)?null:$coords; |
|
327 |
} |
|
328 |
||
329 |
private function getLocation($doc) { |
|
330 |
||
331 |
$geoRes = $doc->getGeoInfo(); |
|
332 |
||
333 |
if(is_null($geoRes)) { |
|
334 |
return null; |
|
335 |
} |
|
336 |
||
337 |
$locUrls = []; |
|
338 |
foreach($geoRes->getRefLocs() as $loc) { |
|
339 |
if(preg_match(config('corpusparole.geonames_url_regexp'), $loc, $m) === 1) { |
|
340 |
||
341 |
if(!array_key_exists('geonames', $locUrls)) { |
|
342 |
$locUrls['geonames'] = []; |
|
343 |
} |
|
344 |
array_push($locUrls['geonames'], "http://sws.geonames.org/$m[1]/"); |
|
345 |
||
346 |
} elseif(preg_match(config('corpusparole.dbpedia_url_regexp'), $loc, $md) === 1) { |
|
347 |
if(!array_key_exists('dbpedia', $locUrls)) { |
|
348 |
$locUrls['dbpedia'] = []; |
|
349 |
} |
|
350 |
//$this->line("DBPEDIA MATCH $loc ".print_r($md,true)); |
|
351 |
array_push($locUrls['dbpedia'], "http://$md[1]/data/$md[4]"); |
|
352 |
} |
|
353 |
} |
|
354 |
||
355 |
$coordinates = null; |
|
356 |
foreach($locUrls as $locType => $locList) { |
|
357 |
foreach($locList as $locationUrl) { |
|
358 |
$coordinates = call_user_func([$this, "${locType}ResolveCoordinates"], $locationUrl); |
|
359 |
if(!is_null($coordinates)) { |
|
360 |
break; |
|
361 |
} |
|
362 |
} |
|
363 |
} |
|
364 |
||
365 |
if(is_null($coordinates)) { |
|
366 |
$coordinates = [$geoRes->getLatitudeValue(), $geoRes->getLongitudeValue()]; |
|
367 |
} |
|
368 |
||
369 |
if(empty($coordinates[0]) || empty($coordinates[1])) { |
|
370 |
return null; |
|
371 |
} else { |
|
372 |
return [floatval($coordinates[0]), floatval($coordinates[1])]; |
|
373 |
} |
|
374 |
||
375 |
} |
|
376 |
||
377 |
private function getCreationDate($doc) { |
|
378 |
||
379 |
$created = $doc->getCreated(); |
|
380 |
if(is_null($created)) { |
|
381 |
return null; |
|
382 |
} |
|
383 |
$dateType = $created->getDatatypeUri(); |
|
384 |
$res = null; |
|
385 |
||
386 |
if($dateType === "http://purl.org/dc/terms/Period") { |
|
387 |
$res = $this->processPeriod($created->getValue()); |
|
388 |
} |
|
389 |
elseif($dateType === "http://purl.org/dc/terms/W3CDTF") { |
|
390 |
$res = $this->processDate($created->getValue()); |
|
391 |
} |
|
392 |
||
393 |
return $res; |
|
394 |
||
395 |
} |
|
396 |
||
397 |
private function extractDate($dateStr) { |
|
398 |
if(preg_match("/^\\d{4}$/", $dateStr) === 1) { |
|
399 |
$dateStr = "$dateStr-1-1"; |
|
400 |
} |
|
401 |
$date = date_create($dateStr); |
|
402 |
if($date === false ) { |
|
403 |
Log::warning("DateStatsController:extractYear bad format for date $dateStr"); |
|
404 |
return null; |
|
405 |
} |
|
406 |
return $date; |
|
407 |
} |
|
408 |
||
409 |
private function processPeriod($periodStr) { |
|
410 |
$start = null; |
|
411 |
$end = null; |
|
412 |
foreach(explode(";", $periodStr) as $elem) { |
|
413 |
$elem = trim($elem); |
|
414 |
if(strpos($elem, 'start=') === 0) { |
|
415 |
$startDate = $this->extractDate(trim(substr($elem, 6))); |
|
416 |
if(is_null($startDate)) { |
|
417 |
return null; |
|
418 |
} |
|
419 |
$start = intval($startDate->format("Y")); |
|
420 |
if($start === false) { |
|
421 |
return null; |
|
422 |
} |
|
423 |
} elseif(strpos($elem, 'end=') === 0) { |
|
424 |
$endDate = $this->extractDate(trim(substr($elem, 4))); |
|
425 |
if(is_null($endDate)) { |
|
426 |
return null; |
|
427 |
} |
|
428 |
$end = intval($endDate->format("Y")); |
|
429 |
if($end === false) { |
|
430 |
return null; |
|
431 |
} |
|
432 |
} |
|
433 |
} |
|
434 |
||
435 |
if(is_null($start) || is_null($end) || $start>$end ) { |
|
436 |
Log::warning("Bad format for $periodStr"); |
|
437 |
return null; |
|
438 |
} |
|
439 |
||
440 |
return array_map(function($y) { |
|
441 |
return \DateTime::createFromFormat("Y", "$y")->format(\DateTime::W3C); |
|
442 |
}, range($start, $end)); |
|
443 |
} |
|
444 |
||
445 |
private function processDate($dateStr) { |
|
446 |
$date = $this->extractDate($dateStr); |
|
447 |
if(is_null($date)) { |
|
448 |
return null; |
|
449 |
} else { |
|
450 |
return $date->format(\DateTime::W3C); |
|
451 |
} |
|
452 |
} |
|
453 |
||
454 |
private function getDiscourseTypes($doc) { |
|
455 |
return array_reduce($doc->getDiscourseTypes(), function($res, $d) { |
|
456 |
$val = null; |
|
457 |
if($d instanceof Resource) { |
|
458 |
$val = $d->getUri(); |
|
459 |
} elseif($d instanceof Literal) { |
|
460 |
$datatype = $d->getDatatypeURI(); |
|
461 |
$val = (!empty($datatype)?"$datatype#":"").$d->getValue(); |
|
462 |
} |
|
463 |
if(!empty($val)) { |
|
464 |
array_push($res,$val); |
|
465 |
} |
|
466 |
return $res; |
|
467 |
}, []); |
|
468 |
} |
|
469 |
||
470 |
private function getDocBody($doc) { |
|
471 |
return [ |
|
472 |
'title' => (string)$doc->getTitle(), |
|
473 |
'date' => (string)$doc->getModified(), |
|
474 |
'location' => $this->getLocation($doc), |
|
475 |
'creation_date' => $this->getCreationDate($doc), |
|
476 |
'language' => $doc->getLanguageValue(), |
|
477 |
'discourse_types' => $this->getDiscourseTypes($doc), |
|
478 |
'geonames_hierarchy' => $this->getGeonamesHierarchy($doc), |
|
479 |
'subject' => $this->getSubjects($doc), |
|
480 |
]; |
|
481 |
} |
|
482 |
||
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
483 |
/** |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
484 |
* Index one document into Elasticsearch |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
485 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
486 |
* @return int (1 if sucess, 0 if error) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
487 |
*/ |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
488 |
private function indexOne($resultDoc) |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
489 |
{ |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
490 |
$doc = $this->documentRepository->get($resultDoc->getId()); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
491 |
$query_data = [ |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
492 |
'index' => config('elasticsearch.index'), |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
493 |
'type' => 'document', |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
494 |
'id' => (string)$doc->getId(), |
325 | 495 |
'body' => $this->getDocBody($doc) |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
496 |
]; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
497 |
Es::index($query_data); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
498 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
499 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
500 |
/** |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
501 |
* Index multiple document into Elasticsearch |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
502 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
503 |
* @return int (1 if sucess, 0 if error) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
504 |
*/ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
505 |
private function indexBulk($docs) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
506 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
507 |
$query_data = ['body' => []]; |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
508 |
foreach($docs as $resultDoc){ |
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
509 |
$doc = $this->documentRepository->get($resultDoc->getId()); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
510 |
$query_data['body'][] = [ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
511 |
'index' => [ |
308
e032d686d88e
add hierarchy info in document indexation + geostats api controllers + add some keys to geonames resolver
ymh <ymh.work@gmail.com>
parents:
25
diff
changeset
|
512 |
'_index' => config('elasticsearch.index'), |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
513 |
'_type' => 'document', |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
514 |
'_id' => (string)$doc->getId() |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
515 |
] |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
516 |
]; |
325 | 517 |
$query_data['body'][] = $this->getDocBody($doc); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
518 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
519 |
Es::bulk($query_data); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
520 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
521 |
/** |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
522 |
* Execute the console command. |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
523 |
* |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
524 |
* @return mixed |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
525 |
*/ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
526 |
public function handle() |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
527 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
528 |
$this->info('Options:'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
529 |
$noBulk = $this->option('no-bulk'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
530 |
if ($noBulk) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
531 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
532 |
$this->comment(' - Indexing without bulk insert'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
533 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
534 |
else |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
535 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
536 |
$this->comment(' - Indexing using bulk insert'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
537 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
538 |
$limit = $this->option('limit'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
539 |
if ($limit>0) { |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
540 |
$this->comment(' - Indexing only the first '.$limit.' documents'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
541 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
542 |
$stepSize = $this->option('step-size'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
543 |
$this->comment(' - Indexing with step size of '.$stepSize); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
544 |
|
323 | 545 |
if($this->option('reset-geo-cache', false)) { |
546 |
// delete all rows in GeonamesHierarchy |
|
547 |
GeonamesHierarchy::getQuery()->delete(); |
|
548 |
$this->comment('Geonames cache reset!'); |
|
549 |
} |
|
550 |
||
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
551 |
$this->info('Resetting index...'); |
323 | 552 |
$success = $this->resetIndex(); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
553 |
if($success==1){ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
554 |
$this->comment('Index reset!'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
555 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
556 |
else{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
557 |
$this->error('Error resetting index ' . env('ELASTICSEARCH_INDEX')); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
558 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
559 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
560 |
$this->info('Indexing documents...'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
561 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
562 |
if ($limit<=0) { |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
563 |
$lastPage = $this->documentRepository->paginateAll($stepSize, 'page')->lastPage(); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
564 |
$total = $this->documentRepository->getCount(); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
565 |
$lastPageEntryCount = $stepSize+1; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
566 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
567 |
else { |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
568 |
$lastPage = min((int)($limit/$stepSize)+1, $this->documentRepository->paginateAll($stepSize, 'page')->lastPage()); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
569 |
$total = $limit; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
570 |
$lastPageEntryCount = $limit % $stepSize; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
571 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
572 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
573 |
if ($noBulk) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
574 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
575 |
$progressBar = $this->output->createProgressBar($total); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
576 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
577 |
else |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
578 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
579 |
$progressBar = $this->output->createProgressBar($lastPage); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
580 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
581 |
$progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% - %message%'); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
582 |
|
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
583 |
for ($page=1;$page<=$lastPage;$page++) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
584 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
585 |
$docs = $this->documentRepository->paginateAll($stepSize, 'page', $page); |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
586 |
if ($noBulk) |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
587 |
{ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
588 |
foreach ($docs as $i=>$doc){ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
589 |
if ($page==$lastPage && $i>=$lastPageEntryCount){ |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
590 |
break; |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
591 |
} |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
592 |
$progressBar->setMessage($doc->getId()); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
593 |
$progressBar->advance(); |
325 | 594 |
$this->indexOne($doc); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
595 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
596 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
597 |
else |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
598 |
{ |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
599 |
$progressBar->setMessage('Page '.$page); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
600 |
$progressBar->advance(); |
325 | 601 |
$this->indexBulk($docs); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
602 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
603 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
604 |
$progressBar->finish(); |
321
aefaad270b9b
reimplement ThemeController using ES requests to be able to sort by label
ymh <ymh.work@gmail.com>
parents:
320
diff
changeset
|
605 |
$this->info("\nIndexing completed"); |
24
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
606 |
} |
de47e8f66e8b
Added command "corpus-parole:indexDocuments" to index documents into ElasticSearch
durandn
parents:
diff
changeset
|
607 |
} |