equal
deleted
inserted
replaced
9 use Symfony\Component\Console\Input\InputArgument; |
9 use Symfony\Component\Console\Input\InputArgument; |
10 use Phpoaipmh\Client; |
10 use Phpoaipmh\Client; |
11 use Phpoaipmh\Endpoint; |
11 use Phpoaipmh\Endpoint; |
12 |
12 |
13 class ImportCocoonRDF extends Command { |
13 class ImportCocoonRDF extends Command { |
|
14 |
|
15 const INSERT_TIMEOUT_RETRY = 5; |
14 |
16 |
15 /** |
17 /** |
16 * The console command description. |
18 * The console command description. |
17 * |
19 * |
18 * @var string |
20 * @var string |
56 |
58 |
57 //TODO : treat timeout exceptions |
59 //TODO : treat timeout exceptions |
58 $progressBar = $this->output->createProgressBar($recs->getTotalRecordsInCollection()); |
60 $progressBar = $this->output->createProgressBar($recs->getTotalRecordsInCollection()); |
59 $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% - %message%'); |
61 $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% - %message%'); |
60 |
62 |
|
63 $insertTimeouts = 0; |
|
64 |
61 foreach ($recs as $item) { |
65 foreach ($recs as $item) { |
62 |
66 |
63 $identifier = (string) $item->xpath('/record/header/identifier')[0]; |
67 $identifier = (string) $item->xpath('/record/header/identifier')[0]; |
64 $docRdfUrl = Config::get('corpusparole.cocoon_rdf_base_uri').substr($identifier, strlen(Config::get('corpusparole.cocoon_doc_id_base'))); |
68 $docRdfUrl = Config::get('corpusparole.cocoon_rdf_base_uri').substr($identifier, strlen(Config::get('corpusparole.cocoon_doc_id_base'))); |
65 $message = "$identifier : $docRdfUrl"; |
69 $message = "$identifier : $docRdfUrl"; |
70 } |
74 } |
71 $progressBar->setMessage($message); |
75 $progressBar->setMessage($message); |
72 $progressBar->advance(); |
76 $progressBar->advance(); |
73 |
77 |
74 $docUri = config('corpusparole.cocoon_doc_id_base_uri').substr($identifier, strlen(Config::get('corpusparole.cocoon_doc_id_base'))); |
78 $docUri = config('corpusparole.cocoon_doc_id_base_uri').substr($identifier, strlen(Config::get('corpusparole.cocoon_doc_id_base'))); |
|
79 |
75 $resDocs = $gs->query("ASK WHERE { GRAPH <$docUri> { ?s ?p ?o }}"); |
80 $resDocs = $gs->query("ASK WHERE { GRAPH <$docUri> { ?s ?p ?o }}"); |
76 if(!$resDocs->getBoolean()) { |
81 if(!$resDocs->getBoolean()) { |
77 $docLoaded = false; |
82 $docLoaded = false; |
78 $loadRetry = 0; |
83 $loadRetry = 0; |
79 while(!$docLoaded && $loadRetry < config('corpusparole.max_load_retry', 3)) { |
84 while(!$docLoaded && $loadRetry < config('corpusparole.max_load_retry', 3)) { |
106 continue; |
111 continue; |
107 } |
112 } |
108 //TODO: treat errors |
113 //TODO: treat errors |
109 $subjects = $doc->resources(); |
114 $subjects = $doc->resources(); |
110 $subject = reset($subjects)->getUri(); |
115 $subject = reset($subjects)->getUri(); |
111 //TODO: exceptions ? but if pb on insert probably we have to fail anyway |
116 try { |
112 $gs->insert($doc, $subject); |
117 $gs->insert($doc, $subject); |
|
118 } |
|
119 catch(\Exception $e) { |
|
120 // just log not much we can do here... |
|
121 $this->error("\nError on insert $identifier ($docRdfUrl) : $e"); |
|
122 Log::error("Error on insert $identifier ($docRdfUrl) : $e"); |
|
123 $code = $e->getCode(); |
|
124 $message = $e->getMessage(); |
|
125 if($e instanceof EasyRdf\Exception && stripos($message, 'timed out')>=0 && $insertTimeout<= ImportCocoonRDF::INSERT_TIMEOUT_RETRY) { |
|
126 $this->info("\nThis is a timeout, we continue."); |
|
127 Log::info("This is a timeout, we continue."); |
|
128 $insertTimeouts++; |
|
129 continue; |
|
130 } |
|
131 throw $e; |
|
132 } |
113 } |
133 } |
114 } |
134 } |
115 $progressBar->setMessage("finished"); |
135 $progressBar->setMessage("finished"); |
116 $progressBar->finish(); |
136 $progressBar->finish(); |
117 } |
137 } |