--- a/app/DoctrineMigrations/Version20140129151724.php Fri Jul 04 17:01:57 2014 +0200
+++ b/app/DoctrineMigrations/Version20140129151724.php Mon Jul 07 17:19:01 2014 +0200
@@ -43,27 +43,32 @@
// First step : we populate the dbpedia uris thanks to the dbpedia-owl:wikiPageID
echo "\nFIRST STEP";
- $query = $em->createQuery('SELECT t FROM WikiTagBundle:Tag t WHERE t.wikipediaPageId IS NOT NULL ORDER BY t.label ASC');//->setMaxResults(240)->setFirstResult(500);
- $tags = $query->getResult();
- $i = 1;
- $nb_set = 0;
- echo "\n".count($tags)." tags to search.";
- foreach($tags as $tag){
- $l = $tag->getLabel();
- $uri = WikiTagUtils::getDbpediaUri($tag->getWikipediaPageId(), array(), false, "pageid");
- $tag->setDbpediaUri($uri);
- $em->persist($tag);
- if($uri!=NULL && $uri!=""){
- $nb_set++;
+ $query = $em->createQuery('SELECT count(t) FROM WikiTagBundle:Tag t WHERE t.wikipediaPageId IS NOT NULL');//->setMaxResults(240)->setFirstResult(500);
+ $nb = $query->getResult();
+ $nb = $nb[0][1];
+ $nb_batch = 50;
+ $nb_loops = ceil($nb / $nb_batch);
+ $i = 1;
+ $nb_set = 0;
+ echo "\n".$nb." tags to search.";
+ for($il=0; $il<$nb_loops; $il++){
+ $query = $em->createQuery('SELECT t FROM WikiTagBundle:Tag t WHERE t.wikipediaPageId IS NOT NULL ORDER BY t.id ASC')->setMaxResults($nb_batch)->setFirstResult($il*$nb_batch);
+ $tags = $query->getResult();
+ foreach($tags as $tag){
+ $l = $tag->getLabel();
+ $uri = WikiTagUtils::getDbpediaUri($tag->getWikipediaPageId(), array(), false, "pageid");
+ $tag->setDbpediaUri($uri);
+ $em->persist($tag);
+ if($uri!=NULL && $uri!=""){
+ $nb_set++;
+ }
+ $i++;
+ echo "\n$i : $l \t\t: $uri";
}
- if( $i % 50 == 0 ){
- $em->flush();
- echo "\n FLUSH";
- }
- $i++;
- echo "\n$i : $l \t\t: $uri";
- }
- $em->flush();
+ echo "\n FLUSH";
+ $em->flush();
+ $em->clear();
+ }
echo "\nFIRST STEP : $nb_set uris found";
@@ -117,6 +122,7 @@
}
$em->flush();
echo "\nTHIRD STEP : $nb_set uris found";
+ echo "\n\nTHIS IS THE END";
}
public function down(Schema $schema)