equal
deleted
inserted
replaced
248 ]); |
248 ]); |
249 } |
249 } |
250 return $res; |
250 return $res; |
251 }, []); |
251 }, []); |
252 |
252 |
253 $labelsBnf = $this->bnfResolver->getLabels( |
253 $labelsResolved = false; |
254 array_unique(array_reduce( |
254 $timeoutRetries = 0; |
255 $sres, |
255 while(!$labelsResolved && $timeoutRetries < config('corpusparole.bnf_max_retries', 3)) { |
256 function($r, $so) { |
256 try { |
257 if($so['type'] === Utils::SUBJECT_TYPE_BNF) { |
257 $labelsBnf = $this->bnfResolver->getLabels( |
258 array_push($r, $so['uri']); |
258 array_unique(array_reduce( |
259 } |
259 $sres, |
260 return $r; |
260 function($r, $so) { |
261 },[] |
261 if($so['type'] === Utils::SUBJECT_TYPE_BNF) { |
262 )) |
262 array_push($r, $so['uri']); |
263 ); |
263 } |
|
264 return $r; |
|
265 },[] |
|
266 )) |
|
267 ); |
|
268 $labelsResolved = true; |
|
269 } catch(BnfResolverTimeoutException $e) { |
|
270 Log::warning('IndexDocument: Resolve label timeout, will retry'); |
|
271 $timeoutRetries++; |
|
272 continue; |
|
273 } |
|
274 } |
|
275 if(!$labelsResolved) { |
|
276 Log::error("IndexDocument: Some bnf labels not resolved (retry timeout: $timeoutRetries)."); |
|
277 $this->error("\nError resolving bnf labels (retry timeout: $timeoutRetries). Stopping"); |
|
278 throw new \Exception("Error resolving bnf labels (retry timeout: $timeoutRetries). Stopping"); |
|
279 } |
264 $labelsLexvo = $this->lexvoResolver->getNames( |
280 $labelsLexvo = $this->lexvoResolver->getNames( |
265 array_unique(array_reduce( |
281 array_unique(array_reduce( |
266 $sres, |
282 $sres, |
267 function($r, $so) { |
283 function($r, $so) { |
268 if($so['type'] === Utils::SUBJECT_TYPE_LEXVO) { |
284 if($so['type'] === Utils::SUBJECT_TYPE_LEXVO) { |
584 /** |
600 /** |
585 * Index multiple document into Elasticsearch |
601 * Index multiple document into Elasticsearch |
586 * |
602 * |
587 * @return int (1 if sucess, 0 if error) |
603 * @return int (1 if sucess, 0 if error) |
588 */ |
604 */ |
589 private function indexBulk($docBodies) |
605 private function indexBulk($docBodies) { |
590 { |
606 if(empty($docBodies)) { |
591 $query_data = ['body' => []]; |
607 return; |
|
608 } |
|
609 $query_data = ['body' => []]; |
592 foreach($docBodies as $docId => $docBody){ |
610 foreach($docBodies as $docId => $docBody){ |
593 $query_data['body'][] = [ |
611 $query_data['body'][] = [ |
594 'index' => [ |
612 'index' => [ |
595 '_index' => config('elasticsearch.index'), |
613 '_index' => config('elasticsearch.index'), |
596 '_type' => 'document', |
614 '_type' => 'document', |
663 foreach($docsPaginator as $docResult) { |
681 foreach($docsPaginator as $docResult) { |
664 $docId = (string)$docResult->getId(); |
682 $docId = (string)$docResult->getId(); |
665 $progressBar->setMessage($docId); |
683 $progressBar->setMessage($docId); |
666 $progressBar->advance(); |
684 $progressBar->advance(); |
667 $doc = $this->documentRepository->get($docId); |
685 $doc = $this->documentRepository->get($docId); |
|
686 if(is_null($doc)) { |
|
687 continue; |
|
688 } |
668 $docBody = $this->getDocBody($doc); |
689 $docBody = $this->getDocBody($doc); |
669 if($noBulk) { |
690 if($noBulk) { |
670 $this->indexOne($docId, $docBody); |
691 $this->indexOne($docId, $docBody); |
671 } else { |
692 } else { |
672 $docsBodies[$docId] = $docBody; |
693 $docsBodies[$docId] = $docBody; |