DoctrineMigrations/Version20140129151724.php
author cavaliet
Mon, 07 Jul 2014 17:23:23 +0200
changeset 131 b39513a56cd0
parent 129 65c12455ce74
permissions -rwxr-xr-x
adapt code for php's great memory management
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
116
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     1
<?php
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     2
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     3
namespace Application\Migrations;
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     4
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     5
use Doctrine\DBAL\Migrations\AbstractMigration,
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     6
    Doctrine\DBAL\Schema\Schema,
123
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
     7
	IRI\Bundle\WikiTagBundle\Utils\WikiTagUtils,
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
     8
	IRI\Bundle\WikiTagBundle\Listener\DocumentListener;
116
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
     9
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    10
/**
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    11
 * Migration for WikiTagBundle <= V00.13
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    12
 * This migration takes every tag label and search the REAL dbpedia uri associated to this label.
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    13
 * Before, the dbpedia uri was manually generated by http://dbpedia.org/resource/ + english_label.
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    14
 * Now we get the dbpedia uri by requesting http://LANG_CODE.dbpedia.org/sparql with the current label.
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    15
 * 
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    16
 */
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    17
class Version20140129151724 extends AbstractMigration
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    18
{
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    19
    public function up(Schema $schema)
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    20
    {
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    21
        // this up() migration is autogenerated, please modify it to your needs
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    22
        $this->abortIf($this->connection->getDatabasePlatform()->getName() != "mysql");
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    23
        
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    24
        
116
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    25
        // First we get all tags.
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
    26
        $em = $GLOBALS["kernel"]->getContainer()->get( 'doctrine.orm.entity_manager' );
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    27
        // Avoid php annoying memory leaks
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    28
        $em->getConnection()->getConfiguration()->setSQLLogger(null);
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    29
        
123
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    30
        // remove event listener to avoid useless sql queries. Only WikiTag's Tags are modified
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    31
        $listenerInst = null;
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    32
        foreach ($em->getEventManager()->getListeners() as $event => $listeners) {
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    33
        	foreach ($listeners as $hash => $listener) {
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    34
        		if ($listener instanceof DocumentListener) {
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    35
        			$listenerInst = $listener;
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    36
        			break 2;
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    37
        		}
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    38
        	}
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    39
        }
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    40
        $listenerInst || die('Listener is not registered in the event manager');
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    41
        $evm = $em->getEventManager();
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    42
        $evm->removeEventListener(array('onFlush', 'preRemove', 'postPersist', 'postUpdate',  'postRemove'), $listenerInst);
d48100e2fead enhance fr.dbpedia migration for production
cavaliet
parents: 118
diff changeset
    43
        
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    44
        // First step : we populate the dbpedia uris thanks to the dbpedia-owl:wikiPageID
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    45
        echo "\nFIRST STEP";
131
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    46
        $query = $em->createQuery('SELECT count(t) FROM WikiTagBundle:Tag t WHERE t.wikipediaPageId IS NOT NULL');//->setMaxResults(240)->setFirstResult(500);
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    47
        $nb = $query->getResult();
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    48
        $nb = $nb[0][1];
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    49
        $nb_batch = 50;
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    50
        $nb_loops = ceil($nb / $nb_batch);
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    51
	    $i = 1;
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    52
	    $nb_set = 0;
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    53
	    echo "\n".$nb." tags to search.";
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    54
        for($il=0; $il<$nb_loops; $il++){
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    55
	        $query = $em->createQuery('SELECT t FROM WikiTagBundle:Tag t WHERE t.wikipediaPageId IS NOT NULL ORDER BY t.id ASC')->setMaxResults($nb_batch)->setFirstResult($il*$nb_batch);
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    56
	        $tags = $query->getResult();
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    57
	        foreach($tags as $tag){
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    58
	        	$l = $tag->getLabel();
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    59
	        	$uri = WikiTagUtils::getDbpediaUri($tag->getWikipediaPageId(), array(), false, "pageid");
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    60
	        	$tag->setDbpediaUri($uri);
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    61
	        	$em->persist($tag);
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    62
		        if($uri!=NULL && $uri!=""){
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    63
		            $nb_set++;
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    64
		        }
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    65
	        	$i++;
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    66
	        	echo "\n$i : $l \t\t: $uri";
129
65c12455ce74 adapt code to be compatible with dinosaur php 5.3
cavaliet
parents: 123
diff changeset
    67
	        }
131
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    68
	        echo "\n    FLUSH";
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    69
	        $em->flush();
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    70
	        $em->clear();
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
    71
    	}
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    72
        echo "\nFIRST STEP : $nb_set uris found";
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    73
        
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    74
        
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    75
        // Second step : we populate the dbpedia uris not found thanks to the foaf:isPrimaryTopicOf
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    76
        echo "\nSECOND STEP";
118
e9c1a6f4b83e enhance migration for dbpedia uri
cavaliet
parents: 117
diff changeset
    77
        $query = $em->createQuery('SELECT t FROM WikiTagBundle:Tag t WHERE (t.wikipediaUrl!=\'\' AND t.wikipediaUrl IS NOT NULL AND (t.dbpediaUri=\'\' OR t.dbpediaUri IS NULL)) ORDER BY t.label ASC');//->setMaxResults(240);
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    78
        $tags = $query->getResult();
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    79
        $i = 1;
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    80
        $nb_set = 0;
118
e9c1a6f4b83e enhance migration for dbpedia uri
cavaliet
parents: 117
diff changeset
    81
        echo "\n".count($tags)." tags to search.";
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    82
        foreach($tags as $tag){
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    83
        	$l = $tag->getLabel();
129
65c12455ce74 adapt code to be compatible with dinosaur php 5.3
cavaliet
parents: 123
diff changeset
    84
        	$uri = WikiTagUtils::getDbpediaUri($tag->getWikipediaUrl(), array(), false, "wikiurl");
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    85
        	$tag->setDbpediaUri($uri);
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    86
        	$em->persist($tag);
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    87
            if($uri!=NULL && $uri!=""){
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    88
                $nb_set++;
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    89
            }
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    90
        	if( $i % 50 == 0 ){
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    91
        		$em->flush();
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    92
        		echo "\n    FLUSH";
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    93
        	}
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    94
        	$i++;
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    95
        	echo "\n$i : $l \t\t: $uri";
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    96
        }
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    97
        $em->flush();
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    98
        echo "\nSECOND STEP : $nb_set uris found";
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
    99
        
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   100
        
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   101
        // Third step : we populate the dbpedia uris not found thanks to the rdfs:label
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   102
        echo "\nTHIRD STEP";
118
e9c1a6f4b83e enhance migration for dbpedia uri
cavaliet
parents: 117
diff changeset
   103
        $query = $em->createQuery('SELECT t FROM WikiTagBundle:Tag t WHERE (t.wikipediaUrl!=\'\' AND t.wikipediaUrl IS NOT NULL AND (t.dbpediaUri=\'\' OR t.dbpediaUri IS NULL)) ORDER BY t.label ASC');//->setMaxResults(240);
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   104
        $tags = $query->getResult();
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   105
        $i = 1;
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   106
        $nb_set = 0;
118
e9c1a6f4b83e enhance migration for dbpedia uri
cavaliet
parents: 117
diff changeset
   107
        echo "\n".count($tags)." tags to search.";
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   108
        foreach($tags as $tag){
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   109
        	$l = $tag->getLabel();
129
65c12455ce74 adapt code to be compatible with dinosaur php 5.3
cavaliet
parents: 123
diff changeset
   110
        	$uri = WikiTagUtils::getDbpediaUri($tag->getLabel(), array(), false);
117
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   111
        	$tag->setDbpediaUri($uri);
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   112
        	$em->persist($tag);
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   113
            if($uri!=NULL && $uri!=""){
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   114
                $nb_set++;
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   115
            }
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   116
        	if( $i % 50 == 0 ){
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   117
        		$em->flush();
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   118
        		echo "\n    FLUSH";
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   119
        	}
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   120
        	$i++;
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   121
        	echo "\n$i : $l \t\t: $uri";
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   122
        }
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   123
        $em->flush();
5771052a647a better migration for dbpedia uri
cavaliet
parents: 116
diff changeset
   124
        echo "\nTHIRD STEP : $nb_set uris found";
131
b39513a56cd0 adapt code for php's great memory management
cavaliet
parents: 129
diff changeset
   125
        echo "\n\nTHIS IS THE END";
116
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   126
    }
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   127
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   128
    public function down(Schema $schema)
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   129
    {
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   130
        // this down() migration is autogenerated, please modify it to your needs
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   131
        $this->abortIf($this->connection->getDatabasePlatform()->getName() != "mysql");
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   132
    }
a023e0185a02 migration to real dbpedia uri
cavaliet
parents:
diff changeset
   133
}