# HG changeset patch
# User ymh <ymh.work@gmail.com>
# Date 1320619477 -3600
# Node ID 8551d844b4f331a6200f668dffc777211a0a4fb3
# Parent  11fd7966637442b665fa51609320c36a9abc51b4
Correct memory problem

diff -r 11fd79666374 -r 8551d844b4f3 Command/SyncDocumentsCommand.php
--- a/Command/SyncDocumentsCommand.php	Fri Nov 04 15:59:49 2011 +0100
+++ b/Command/SyncDocumentsCommand.php	Sun Nov 06 23:44:37 2011 +0100
@@ -27,6 +27,8 @@
         $this
             ->setName('wikitag:sync-doc')
             ->setDescription('Synchronize and index document class')
+            ->addOption('missing', 'm', InputOption::VALUE_NONE, "process missing")
+            ->addOption('tags', 't', InputOption::VALUE_NONE, "update tags")
             ->addOption('clear', 'c', InputOption::VALUE_NONE, "clear all docs");
     }
 
@@ -34,6 +36,7 @@
     {
         $class = $this->getContainer()->getParameter('wiki_tag.document_class');
         $clear = $input->getOption('clear');
+        $missing = $input->getOption('missing');
      
         
         $doctrine = $this->getContainer()->get('doctrine');
@@ -47,15 +50,75 @@
             return ;
         }
         
+        if($input->getOption('tags'))
+        {
+            if(!$missing)
+            {
+                $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc from WikiTagBundle:Document doc");
+                $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT COUNT(doc.id) from WikiTagBundle:Document doc");
+            }
+            else
+            {
+                $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc from WikiTagBundle:Document doc WHERE doc.tagsStr IS NULL");
+                $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT COUNT(doc.id) from WikiTagBundle:Document doc WHERE doc.tagsStr IS NULL");
+            }
+            
+            
+            $total = $doccountquery->getSingleScalarResult();
+            $done = 0;
+            $iterable = $docquery->iterate();
+            $todetach = array();
+            while (($row = $iterable->next()) !== false) {
+                $done++;
+                $memory = ((($done%10)==0)?" - mem: ".strval(memory_get_usage(true)):"");
+                $doc = $row[0];
+                $todetach[] = $doc;
+                $output->writeln("Process doc id ".$doc->getId()." $done/$total ".strval(intval(floatval($done)/floatval($total)*100.0))."%$memory");
+                $docrep->updateTagsStr($doc);
+                if($done%10 == 0)
+                {
+                    $doctrine->getEntityManager()->flush();
+                    foreach($todetach as $obj)
+                    {
+                        $doctrine->getEntityManager()->detach($obj);
+                    }
+                    $todetach = array();
+                }
+            }
+            $doctrine->getEntityManager()->flush();
+                        
+            return;
+            
+        }
+        
         //TODO : check class to implement DocumentInterface
         //TODO : write progress
-        $doclist = $rep->findAll();
-        $total = count($doclist);
+        if($missing)
+        {
+            $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc FROM $class doc WHERE doc.id not in (SELECT wtdoc FROM WikiTagBundle:Document wtdoc)");
+            $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT count(doc.id) FROM $class doc WHERE doc.id not in (SELECT wtdoc FROM WikiTagBundle:Document wtdoc)");
+            //$doclist = $doctrine->getEntityManager()->createQuery("SELECT doc FROM $class doc WHERE doc.id not in (SELECT wtdoc FROM WikiTagBundle:Document wtdoc)")->getResult();
+        }
+        else
+        {
+            $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc FROM $class doc");
+            $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT count(doc.id) FROM $class doc");
+            //$doclist = $rep->findAll();
+        }
+        $total = $doccountquery->getSingleScalarResult();
         $done = 0;
-        foreach ($doclist as $doc) {
+        $iterable = $docquery->iterate();
+        while (($row = $iterable->next()) !== false) {
             $done++;
-            $output->writeln("Process doc id ".$doc->getId()." $done/$total ".strval(intval(floatval($done)/floatval($total)*100.0))."%");
+            $doc = $row[0];
+            $memory = ((($done%10)==0)?" - mem: ".strval(memory_get_usage(true)):"");
+            $output->writeln("Process doc id ".$doc->getId()." $done/$total ".strval(intval(floatval($done)/floatval($total)*100.0))."%$memory");
             $docrep->writeDocument($doc, $this->getContainer()->getParameter('wiki_tag.document_id_column'), $this->getContainer()->getParameter('wiki_tag.fields'));
+            if($done%10 == 0)
+            {
+                $doctrine->getEntityManager()->flush();
+                $doctrine->getEntityManager()->clear();
+            }
         }
         $doctrine->getEntityManager()->flush();
         
diff -r 11fd79666374 -r 8551d844b4f3 Entity/DocumentRepository.php
--- a/Entity/DocumentRepository.php	Fri Nov 04 15:59:49 2011 +0100
+++ b/Entity/DocumentRepository.php	Sun Nov 06 23:44:37 2011 +0100
@@ -185,17 +185,26 @@
         }
     }
     
+    function getTagsStr($document)
+    {
+        $em = $this->getEntityManager();
+        $query = $em->createQuery("SELECT t.label FROM WikiTagBundle:DocumentTag dt JOIN dt.tag t WHERE dt.document = :docid");
+        $query = $query->setParameter("docid", $document);
+        $result = $query->getScalarResult();
+        $tagstr = array();
+        foreach ($result as $res) {
+            $tagstr[] = $res['label'];
+        }
+        return $tagstr;
+    }
+    
     function updateTagsStr($document)
     {
-        $tags_str = array();
-        foreach($document->getTags() as $tag)
-        {
-            $tags_str[] = $tag->getTag()->getLabel();
-        }
+        
+        $tagstr = $this->getTagsStr($document);
         
-        $document->setTagsStr(implode(",",$tags_str));
-        
-        $this->getEntityManager()->flush();
+        $document->setTagsStr(implode(",",$tagstr));
+        $this->getEntityManager()->persist($document);
     }
         
 }
\ No newline at end of file
diff -r 11fd79666374 -r 8551d844b4f3 Model/Document.php
--- a/Model/Document.php	Fri Nov 04 15:59:49 2011 +0100
+++ b/Model/Document.php	Sun Nov 06 23:44:37 2011 +0100
@@ -11,6 +11,7 @@
 namespace IRI\Bundle\WikiTagBundle\Model;
 
 use Doctrine\Common\Collections\ArrayCollection;
+use Doctrine\Common\Util\Debug;
 
 abstract class Document implements DocumentInterface {
     
@@ -117,5 +118,9 @@
         return $this->tagsStr;
     }
     
+    public function __toString()
+    {
+        return print_r(Debug::export($this, 3),true);
+    }
 
 }