|
83
|
1 |
<?php |
|
|
2 |
/* |
|
|
3 |
* This file is part of the WikiTagBundle package. |
|
|
4 |
* |
|
|
5 |
* (c) IRI <http://www.iri.centrepompidou.fr/> |
|
|
6 |
* |
|
|
7 |
* For the full copyright and license information, please view the LICENSE |
|
|
8 |
* file that was distributed with this source code. |
|
|
9 |
*/ |
|
|
10 |
|
|
|
11 |
namespace IRI\Bundle\WikiTagBundle\Command; |
|
|
12 |
|
|
|
13 |
|
|
|
14 |
use IRI\Bundle\WikiTagBundle\Entity\DocumentTag; |
|
|
15 |
|
|
|
16 |
use IRI\Bundle\WikiTagBundle\Entity\Tag; |
|
|
17 |
|
|
|
18 |
use IRI\Bundle\WikiTagBundle\Entity\Category; |
|
|
19 |
|
|
|
20 |
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; |
|
|
21 |
use Symfony\Component\Console\Input\InputArgument; |
|
|
22 |
use Symfony\Component\Console\Input\InputInterface; |
|
|
23 |
use Symfony\Component\Console\Input\InputOption; |
|
|
24 |
use Symfony\Component\Console\Output\OutputInterface; |
|
|
25 |
use Mandango\Mondator\Definition\Definition; |
|
|
26 |
use Mandango\Mondator\Definition\Property; |
|
|
27 |
use Mandango\Mondator\Definition\Method; |
|
|
28 |
use Mandango\Mondator\Dumper; |
|
|
29 |
|
|
|
30 |
/** |
|
|
31 |
* |
|
|
32 |
* This class implements a command to load categories, tags, and document_tags fixtures from a yaml file. |
|
|
33 |
* |
|
|
34 |
* @author ymh |
|
|
35 |
* @author tc |
|
|
36 |
* |
|
|
37 |
*/ |
|
85
|
38 |
class LoadFixturesCommand extends ProgressContainerAwareCommand |
|
83
|
39 |
{ |
|
|
40 |
protected function configure() |
|
|
41 |
{ |
|
|
42 |
parent::configure(); |
|
|
43 |
|
|
|
44 |
$this |
|
85
|
45 |
->setName('wikitag:load-fixtures') |
|
83
|
46 |
->setDescription('Load categories, tags, document_tags in JSON format. We suppose SyncDocuments command was executed.') |
|
|
47 |
->addArgument('path', InputArgument::REQUIRED, 'path to the yaml file.') |
|
|
48 |
->addOption("simulate","S",InputOption::VALUE_NONE, "Load and parse file, create objects but does not save them") |
|
|
49 |
->addOption("categories","C",InputOption::VALUE_NONE, "Will save the categories and not the tags/doctags if options are not set (useful to avoid memory problems)") |
|
|
50 |
->addOption("tags","T",InputOption::VALUE_NONE, "Will save the tags and not the categories/doctags if options are not set (useful to avoid memory problems)") |
|
|
51 |
->addOption("doctags","D",InputOption::VALUE_NONE, "Will save the document_tags and not the categories/tags if options are not set (useful to avoid memory problems)") |
|
|
52 |
->addOption("ibegin","B",InputOption::VALUE_OPTIONAL, "Iteration begin, to save part of the datas (useful to avoid memory problems)") |
|
|
53 |
->addOption("iend","E",InputOption::VALUE_OPTIONAL, "Iteration end, to save part of the datas (useful to avoid memory problems)"); |
|
|
54 |
} |
|
|
55 |
|
|
|
56 |
protected function execute(InputInterface $input, OutputInterface $output) |
|
|
57 |
{ |
|
|
58 |
// Default values |
|
|
59 |
$do_cat = FALSE; |
|
|
60 |
$do_tags = FALSE; |
|
|
61 |
$do_doctags = FALSE; |
|
|
62 |
// Get options |
|
|
63 |
$file_path = $input->getArgument('path'); |
|
|
64 |
$simulate = $input->getOption('simulate'); |
|
|
65 |
$do_cat = $input->getOption('categories'); |
|
|
66 |
$do_tags = $input->getOption('tags'); |
|
|
67 |
$do_doctags = $input->getOption('doctags'); |
|
|
68 |
// If no options were set, we load the 3 objects |
|
|
69 |
if(!$do_cat && !$do_tags && !$do_doctags){ |
|
|
70 |
$do_cat = TRUE; |
|
|
71 |
$do_tags = TRUE; |
|
|
72 |
$do_doctags = TRUE; |
|
|
73 |
} |
|
|
74 |
// Set iBegin and iEnd |
|
|
75 |
$iBegin = intval($input->getOption('ibegin')); |
|
|
76 |
$iEnd = intval($input->getOption('iend')); |
|
|
77 |
if($iEnd==0){ |
|
|
78 |
$iEnd = INF; |
|
|
79 |
} |
|
|
80 |
$output->writeln("simulate = ".((!$simulate)?"false":"true").", file path = $file_path" |
|
|
81 |
.", load cat = ".(($do_cat)?"true":"false").", load tags = ".(($do_tags)?"true":"false") |
|
|
82 |
.", load doc_tags = ".(($do_doctags)?"true":"false").", ibegin = ".$iBegin.", iend = ".$iEnd.", min = ".min(10, $iEnd)); |
|
|
83 |
|
|
|
84 |
|
|
|
85 |
// Parse json file and build the data array. |
|
|
86 |
$value = NULL; |
|
|
87 |
try { |
|
|
88 |
$content = file_get_contents($file_path); |
|
|
89 |
$value = json_decode($content, TRUE); |
|
|
90 |
} catch (Exception $e) { |
|
|
91 |
printf("Unable to parse the JSON string: %s", $e->getMessage()); |
|
|
92 |
} |
|
|
93 |
|
|
|
94 |
// We build the 3 data arrays. |
|
|
95 |
$categories = array(); |
|
|
96 |
$tags = array(); |
|
|
97 |
$doctags = array(); |
|
|
98 |
foreach ($value as $ar){ |
|
|
99 |
if($ar["model"]=="hdabo.tagcategory"){ |
|
|
100 |
array_push($categories,$ar); |
|
|
101 |
} |
|
|
102 |
else if($ar["model"]=="hdabo.tag"){ |
|
|
103 |
array_push($tags,$ar); |
|
|
104 |
} |
|
|
105 |
else if($ar["model"]=="hdabo.taggedsheet"){ |
|
|
106 |
array_push($doctags,$ar); |
|
|
107 |
} |
|
|
108 |
} |
|
|
109 |
$content = NULL; |
|
|
110 |
$value = NULL; |
|
|
111 |
|
|
|
112 |
// Get the entity manager |
|
|
113 |
$em = $this->getContainer()->get('doctrine')->getEntityManager(); |
|
|
114 |
|
|
|
115 |
if($do_cat){ |
|
|
116 |
// We save the categories, and force their id to be coherent with the tag's data. |
|
|
117 |
$nb = count($categories); |
|
|
118 |
if($nb>0){ |
|
|
119 |
$output->writeln("Saving $nb categories..."); |
|
|
120 |
$i = 0; |
|
|
121 |
foreach ($categories as $cat_ar){ |
|
|
122 |
$cat = new Category(); |
|
|
123 |
$cat->setId($cat_ar["pk"]); |
|
|
124 |
$f = $cat_ar["fields"]; |
|
|
125 |
$cat->setLabel($f["label"]); |
|
|
126 |
if(!$simulate){ |
|
|
127 |
$em->persist($cat); |
|
|
128 |
// Temporary stop doctrine2 auto increment |
|
|
129 |
$metadata = $em->getClassMetaData(get_class($cat)); |
|
|
130 |
$metadata->setIdGeneratorType(\Doctrine\ORM\Mapping\ClassMetadata::GENERATOR_TYPE_NONE); |
|
|
131 |
} |
|
|
132 |
$i++; |
|
|
133 |
$this->showProgress($output, $i, $nb, "label : ".$f["label"], 50); |
|
|
134 |
} |
|
|
135 |
// We flush here because we need to call the category object when we save the tag's datas. |
|
|
136 |
if(!$simulate){ |
|
|
137 |
$em->flush(); |
|
|
138 |
$em->clear(); |
|
|
139 |
$output->writeln("Categories have been saved"); |
|
|
140 |
} |
|
|
141 |
} |
|
|
142 |
} |
|
|
143 |
|
|
|
144 |
if($do_tags){ |
|
|
145 |
$nb = count($tags); |
|
|
146 |
if($nb>0){ |
|
|
147 |
$output->writeln("Saving $nb tags..."); |
|
|
148 |
$cat_repo = $this->getContainer()->get('doctrine')->getRepository('WikiTagBundle:Category'); |
|
|
149 |
$stop = min($iEnd+1,$nb); |
|
|
150 |
$output->writeln("Saving from $iBegin to ".($stop-1)." on $nb tags..."); |
|
|
151 |
$nbToSave = $stop - $iBegin; |
|
|
152 |
for($i=$iBegin;$i<$stop;$i++){ |
|
|
153 |
$tag = new Tag(); |
|
|
154 |
$tag->setId($tags[$i]["pk"]); |
|
|
155 |
$f = $tags[$i]["fields"]; |
|
|
156 |
$tag->setLabel($f["label"]); |
|
|
157 |
$tag->setNormalizedLabel($f["normalized_label"]); |
|
|
158 |
$tag->setOriginalLabel($f["original_label"]); |
|
|
159 |
$tag->setAlternativeLabel($f["alternative_label"]); |
|
|
160 |
$tag->setAlias($f["alias"]); |
|
|
161 |
$tag->setWikipediaUrl($f["wikipedia_url"]); |
|
|
162 |
$tag->setAlternativeWikipediaUrl($f["alternative_wikipedia_url"]); |
|
|
163 |
$tag->setWikipediaPageId($f["wikipedia_pageid"]); |
|
|
164 |
$tag->setAlternativeWikipediaPageId($f["alternative_wikipedia_pageid"]); |
|
|
165 |
$tag->setUrlStatus($f["url_status"]); // smallint |
|
|
166 |
$tag->setDbpediaUri($f["dbpedia_uri"]); |
|
|
167 |
$tag->setPopularity($f["popularity"]); |
|
|
168 |
$cat = $f["category"]; |
|
|
169 |
if($cat!=NULL){ |
|
|
170 |
$tag->setCategory($cat_repo->findOneById($cat)); // !! OBJECT CATEGORY |
|
|
171 |
} |
|
|
172 |
$tag->setCreatedAt(new \DateTime($f["created_at"], new \DateTimeZone('UTC'))); |
|
|
173 |
if(!$simulate){ |
|
|
174 |
$em->persist($tag); |
|
|
175 |
// Temporary stop doctrine2 auto increment |
|
|
176 |
$metadata = $em->getClassMetaData(get_class($tag)); |
|
|
177 |
$metadata->setIdGeneratorType(\Doctrine\ORM\Mapping\ClassMetadata::GENERATOR_TYPE_NONE); |
|
|
178 |
} |
|
|
179 |
// We save every 10 tags to avoid symfony's UnitOfWork crash |
|
|
180 |
if((!$simulate) && ($i%10==0)){ |
|
|
181 |
// We flush here because we need to call the tag object when we save the document_tag's datas. |
|
|
182 |
$em->flush(); |
|
|
183 |
$em->clear(); |
|
|
184 |
$em = $this->getContainer()->get('doctrine')->getEntityManager(); |
|
|
185 |
} |
|
|
186 |
$memory = "mem: ".strval(memory_get_usage(true)); |
|
|
187 |
$this->showProgress($output, (($i-$iBegin)+1), $nbToSave, "$memory - label : ".$f["label"], 50); |
|
|
188 |
} |
|
|
189 |
// We flush one last time. |
|
|
190 |
if(!$simulate){ |
|
|
191 |
$em->flush(); |
|
|
192 |
$em->clear(); |
|
|
193 |
$em = $this->getContainer()->get('doctrine')->getEntityManager(); |
|
|
194 |
$output->writeln("Tags have been saved"); |
|
|
195 |
} |
|
|
196 |
} |
|
|
197 |
} |
|
|
198 |
|
|
|
199 |
if($do_doctags){ |
|
|
200 |
$categories = NULL; |
|
|
201 |
$tags = NULL; |
|
|
202 |
$nb = count($doctags); |
|
|
203 |
if($nb>0){ |
|
|
204 |
$tag_repo = $repository = $this->getContainer()->get('doctrine')->getRepository('WikiTagBundle:Tag'); |
|
|
205 |
$doc_repo = $repository = $this->getContainer()->get('doctrine')->getRepository('WikiTagBundle:Document'); |
|
|
206 |
$stop = min($iEnd,$nb); |
|
|
207 |
$output->writeln("Saving from $iBegin to $stop on $nb document_tags..."); |
|
|
208 |
$nbToSave = $stop - $iBegin; |
|
|
209 |
for($i=$iBegin;$i<$stop;$i++){ |
|
|
210 |
$f = $doctags[$i]["fields"]; |
|
|
211 |
$dt = new DocumentTag(); |
|
|
212 |
$dt->setOriginalOrder($f["original_order"]); |
|
|
213 |
$dt->setTagOrder($f["order"]); |
|
|
214 |
$dt->setIndexNote($f["index_note"]); |
|
|
215 |
$dt->setWikipediaRevisionId($f["wikipedia_revision_id"]); |
|
|
216 |
$dt->setTag($tag_repo->findOneById($f["tag"])); |
|
|
217 |
$doc = $doc_repo->findOneByExternalId($f["datasheet"]); |
|
|
218 |
// We save if the document exists |
|
|
219 |
if($doc!=NULL){ |
|
|
220 |
$dt->setDocument($doc); |
|
|
221 |
$dt->setCreatedAt(new \DateTime($f["created_at"], new \DateTimeZone('UTC'))); |
|
|
222 |
if(!$simulate){ |
|
|
223 |
$em->persist($dt); |
|
|
224 |
} |
|
|
225 |
} |
|
|
226 |
else{ |
|
|
227 |
$output->writeln("Document with external id ".$f["datasheet"]." was not found !"); |
|
|
228 |
} |
|
|
229 |
// We save every 10 tags to avoid symfony's UnitOfWork crash |
|
|
230 |
if((!$simulate) && ($i%10==0)){ |
|
|
231 |
// We flush here because we need to call the tag object when we save the document_tag's datas. |
|
|
232 |
$em->flush(); |
|
|
233 |
$em->clear(); |
|
|
234 |
$em = $this->getContainer()->get('doctrine')->getEntityManager(); |
|
|
235 |
} |
|
|
236 |
$memory = "mem: ".strval(memory_get_usage(true)); |
|
|
237 |
$this->showProgress($output, (($i-$iBegin)+1), $nbToSave, $memory, 50); |
|
|
238 |
} |
|
|
239 |
if(!$simulate){ |
|
|
240 |
$em->flush(); |
|
|
241 |
$em->clear(); |
|
|
242 |
$output->writeln("DocumentTags have been saved"); |
|
|
243 |
} |
|
|
244 |
} |
|
|
245 |
} |
|
|
246 |
|
|
|
247 |
|
|
|
248 |
if($simulate) |
|
|
249 |
{ |
|
|
250 |
$output->writeln("you simulated."); |
|
|
251 |
} |
|
|
252 |
else |
|
|
253 |
{ |
|
|
254 |
//$output->writeln("file $file_path loaded."); |
|
|
255 |
//$output->writeln(var_dump($value)); |
|
|
256 |
//$output->writeln("s = ".$s); |
|
|
257 |
$output->writeln("End of fixture loading"); |
|
|
258 |
} |
|
|
259 |
} |
|
85
|
260 |
} |