--- a/Command/SyncDocumentsCommand.php	Fri Nov 04 15:59:49 2011 +0100
+++ b/Command/SyncDocumentsCommand.php	Sun Nov 06 23:44:37 2011 +0100
@@ -27,6 +27,8 @@
         $this
             ->setName('wikitag:sync-doc')
             ->setDescription('Synchronize and index document class')
+            ->addOption('missing', 'm', InputOption::VALUE_NONE, "process missing")
+            ->addOption('tags', 't', InputOption::VALUE_NONE, "update tags")
             ->addOption('clear', 'c', InputOption::VALUE_NONE, "clear all docs");
     }
 
@@ -34,6 +36,7 @@
     {
         $class = $this->getContainer()->getParameter('wiki_tag.document_class');
         $clear = $input->getOption('clear');
+        $missing = $input->getOption('missing');
      
         
         $doctrine = $this->getContainer()->get('doctrine');
@@ -47,15 +50,75 @@
             return ;
         }
         
+        if($input->getOption('tags'))
+        {
+            if(!$missing)
+            {
+                $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc from WikiTagBundle:Document doc");
+                $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT COUNT(doc.id) from WikiTagBundle:Document doc");
+            }
+            else
+            {
+                $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc from WikiTagBundle:Document doc WHERE doc.tagsStr IS NULL");
+                $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT COUNT(doc.id) from WikiTagBundle:Document doc WHERE doc.tagsStr IS NULL");
+            }
+            
+            
+            $total = $doccountquery->getSingleScalarResult();
+            $done = 0;
+            $iterable = $docquery->iterate();
+            $todetach = array();
+            while (($row = $iterable->next()) !== false) {
+                $done++;
+                $memory = ((($done%10)==0)?" - mem: ".strval(memory_get_usage(true)):"");
+                $doc = $row[0];
+                $todetach[] = $doc;
+                $output->writeln("Process doc id ".$doc->getId()." $done/$total ".strval(intval(floatval($done)/floatval($total)*100.0))."%$memory");
+                $docrep->updateTagsStr($doc);
+                if($done%10 == 0)
+                {
+                    $doctrine->getEntityManager()->flush();
+                    foreach($todetach as $obj)
+                    {
+                        $doctrine->getEntityManager()->detach($obj);
+                    }
+                    $todetach = array();
+                }
+            }
+            $doctrine->getEntityManager()->flush();
+                        
+            return;
+            
+        }
+        
         //TODO : check class to implement DocumentInterface
         //TODO : write progress
-        $doclist = $rep->findAll();
-        $total = count($doclist);
+        if($missing)
+        {
+            $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc FROM $class doc WHERE doc.id not in (SELECT wtdoc FROM WikiTagBundle:Document wtdoc)");
+            $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT count(doc.id) FROM $class doc WHERE doc.id not in (SELECT wtdoc FROM WikiTagBundle:Document wtdoc)");
+            //$doclist = $doctrine->getEntityManager()->createQuery("SELECT doc FROM $class doc WHERE doc.id not in (SELECT wtdoc FROM WikiTagBundle:Document wtdoc)")->getResult();
+        }
+        else
+        {
+            $docquery = $doctrine->getEntityManager()->createQuery("SELECT doc FROM $class doc");
+            $doccountquery = $doctrine->getEntityManager()->createQuery("SELECT count(doc.id) FROM $class doc");
+            //$doclist = $rep->findAll();
+        }
+        $total = $doccountquery->getSingleScalarResult();
         $done = 0;
-        foreach ($doclist as $doc) {
+        $iterable = $docquery->iterate();
+        while (($row = $iterable->next()) !== false) {
             $done++;
-            $output->writeln("Process doc id ".$doc->getId()." $done/$total ".strval(intval(floatval($done)/floatval($total)*100.0))."%");
+            $doc = $row[0];
+            $memory = ((($done%10)==0)?" - mem: ".strval(memory_get_usage(true)):"");
+            $output->writeln("Process doc id ".$doc->getId()." $done/$total ".strval(intval(floatval($done)/floatval($total)*100.0))."%$memory");
             $docrep->writeDocument($doc, $this->getContainer()->getParameter('wiki_tag.document_id_column'), $this->getContainer()->getParameter('wiki_tag.fields'));
+            if($done%10 == 0)
+            {
+                $doctrine->getEntityManager()->flush();
+                $doctrine->getEntityManager()->clear();
+            }
         }
         $doctrine->getEntityManager()->flush();