<?php

/**

 * Zend Framework

 *

 * LICENSE

 *

 * This source file is subject to the new BSD license that is bundled

 * with this package in the file LICENSE.txt.

 * It is also available through the world-wide-web at this URL:

 * http://framework.zend.com/license/new-bsd

 * If you did not receive a copy of the license and are unable to

 * obtain it through the world-wide-web, please send an email

 * to license@zend.com so we can send you a copy immediately.

 *

 * @category   Zend

 * @package    Zend_Search_Lucene

 * @subpackage Index

 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)

 * @license    http://framework.zend.com/license/new-bsd     New BSD License

 * @version    $Id: Writer.php 20096 2010-01-06 02:05:09Z bkarwin $

 */

/** Zend_Search_Lucene_LockManager */

require_once 'Zend/Search/Lucene/LockManager.php';

/**

 * @category   Zend

 * @package    Zend_Search_Lucene

 * @subpackage Index

 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)

 * @license    http://framework.zend.com/license/new-bsd     New BSD License

 */

class Zend_Search_Lucene_Index_Writer

{

    /**

     * @todo Implement Analyzer substitution

     * @todo Implement Zend_Search_Lucene_Storage_DirectoryRAM and Zend_Search_Lucene_Storage_FileRAM to use it for

     *       temporary index files

     * @todo Directory lock processing

     */

    /**

     * Number of documents required before the buffered in-memory

     * documents are written into a new Segment

     *

     * Default value is 10

     *

     * @var integer

     */

    public $maxBufferedDocs = 10;

    /**

     * Largest number of documents ever merged by addDocument().

     * Small values (e.g., less than 10,000) are best for interactive indexing,

     * as this limits the length of pauses while indexing to a few seconds.

     * Larger values are best for batched indexing and speedier searches.

     *

     * Default value is PHP_INT_MAX

     *

     * @var integer

     */

    public $maxMergeDocs = PHP_INT_MAX;

    /**

     * Determines how often segment indices are merged by addDocument().

     *

     * With smaller values, less RAM is used while indexing,

     * and searches on unoptimized indices are faster,

     * but indexing speed is slower.

     *

     * With larger values, more RAM is used during indexing,

     * and while searches on unoptimized indices are slower,

     * indexing is faster.

     *

     * Thus larger values (> 10) are best for batch index creation,

     * and smaller values (< 10) for indices that are interactively maintained.

     *

     * Default value is 10

     *

     * @var integer

     */

    public $mergeFactor = 10;

    /**

     * File system adapter.

     *

     * @var Zend_Search_Lucene_Storage_Directory

     */

    private $_directory = null;

    /**

     * Changes counter.

     *

     * @var integer

     */

    private $_versionUpdate = 0;

    /**

     * List of the segments, created by index writer

     * Array of Zend_Search_Lucene_Index_SegmentInfo objects

     *

     * @var array

     */

    private $_newSegments = array();

    /**

     * List of segments to be deleted on commit

     *

     * @var array

     */

    private $_segmentsToDelete = array();

    /**

     * Current segment to add documents

     *

     * @var Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter

     */

    private $_currentSegment = null;

    /**

     * Array of Zend_Search_Lucene_Index_SegmentInfo objects for this index.

     *

     * It's a reference to the corresponding Zend_Search_Lucene::$_segmentInfos array

     *

     * @var array Zend_Search_Lucene_Index_SegmentInfo

     */

    private $_segmentInfos;

    /**

     * Index target format version

     *

     * @var integer

     */

    private $_targetFormatVersion;

    /**

     * List of indexfiles extensions

     *

     * @var array

     */

    private static $_indexExtensions = array('.cfs' => '.cfs',

                                             '.cfx' => '.cfx',

                                             '.fnm' => '.fnm',

                                             '.fdx' => '.fdx',

                                             '.fdt' => '.fdt',

                                             '.tis' => '.tis',

                                             '.tii' => '.tii',

                                             '.frq' => '.frq',

                                             '.prx' => '.prx',

                                             '.tvx' => '.tvx',

                                             '.tvd' => '.tvd',

                                             '.tvf' => '.tvf',

                                             '.del' => '.del',

                                             '.sti' => '.sti' );

    /**

     * Create empty index

     *

     * @param Zend_Search_Lucene_Storage_Directory $directory

     * @param integer $generation

     * @param integer $nameCount

     */

    public static function createIndex(Zend_Search_Lucene_Storage_Directory $directory, $generation, $nameCount)

    {

        if ($generation == 0) {

            // Create index in pre-2.1 mode

            foreach ($directory->fileList() as $file) {

                if ($file == 'deletable' ||

                    $file == 'segments'  ||

                    isset(self::$_indexExtensions[ substr($file, strlen($file)-4)]) ||

                    preg_match('/\.f\d+$/i', $file) /* matches <segment_name>.f<decimal_nmber> file names */) {

                        $directory->deleteFile($file);

                    }

            }

            $segmentsFile = $directory->createFile('segments');

            $segmentsFile->writeInt((int)0xFFFFFFFF);

            // write version (initialized by current time)

            $segmentsFile->writeLong(round(microtime(true)));

            // write name counter

            $segmentsFile->writeInt($nameCount);

            // write segment counter

            $segmentsFile->writeInt(0);

            $deletableFile = $directory->createFile('deletable');

            // write counter

            $deletableFile->writeInt(0);

        } else {

            $genFile = $directory->createFile('segments.gen');

            $genFile->writeInt((int)0xFFFFFFFE);

            // Write generation two times

            $genFile->writeLong($generation);

            $genFile->writeLong($generation);

            $segmentsFile = $directory->createFile(Zend_Search_Lucene::getSegmentFileName($generation));

            $segmentsFile->writeInt((int)0xFFFFFFFD);

            // write version (initialized by current time)

            $segmentsFile->writeLong(round(microtime(true)));

            // write name counter

            $segmentsFile->writeInt($nameCount);

            // write segment counter

            $segmentsFile->writeInt(0);

        }

    }

    /**

     * Open the index for writing

     *

     * @param Zend_Search_Lucene_Storage_Directory $directory

     * @param array $segmentInfos

     * @param integer $targetFormatVersion

     * @param Zend_Search_Lucene_Storage_File $cleanUpLock

     */

    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, &$segmentInfos, $targetFormatVersion)

    {

        $this->_directory           = $directory;

        $this->_segmentInfos        = &$segmentInfos;

        $this->_targetFormatVersion = $targetFormatVersion;

    }

    /**

     * Adds a document to this index.

     *

     * @param Zend_Search_Lucene_Document $document

     */

    public function addDocument(Zend_Search_Lucene_Document $document)

    {

        /** Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter */

        require_once 'Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php';

        if ($this->_currentSegment === null) {

            $this->_currentSegment =

                new Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter($this->_directory, $this->_newSegmentName());

        }

        $this->_currentSegment->addDocument($document);

        if ($this->_currentSegment->count() >= $this->maxBufferedDocs) {

            $this->commit();

        }

        $this->_maybeMergeSegments();

        $this->_versionUpdate++;

    }

    /**

     * Check if we have anything to merge

     *

     * @return boolean

     */

    private function _hasAnythingToMerge()

    {

        $segmentSizes = array();

        foreach ($this->_segmentInfos as $segName => $segmentInfo) {

            $segmentSizes[$segName] = $segmentInfo->count();

        }

        $mergePool   = array();

        $poolSize    = 0;

        $sizeToMerge = $this->maxBufferedDocs;

        asort($segmentSizes, SORT_NUMERIC);

        foreach ($segmentSizes as $segName => $size) {

            // Check, if segment comes into a new merging block

            while ($size >= $sizeToMerge) {

                // Merge previous block if it's large enough

                if ($poolSize >= $sizeToMerge) {

                    return true;

                }

                $mergePool   = array();

                $poolSize    = 0;

                $sizeToMerge *= $this->mergeFactor;

                if ($sizeToMerge > $this->maxMergeDocs) {

                    return false;

                }

            }

            $mergePool[] = $this->_segmentInfos[$segName];

            $poolSize += $size;

        }

        if ($poolSize >= $sizeToMerge) {

            return true;

        }

        return false;

    }

    /**

     * Merge segments if necessary

     */

    private function _maybeMergeSegments()

    {

        if (Zend_Search_Lucene_LockManager::obtainOptimizationLock($this->_directory) === false) {

            return;

        }

        if (!$this->_hasAnythingToMerge()) {

            Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);

            return;

        }

        // Update segments list to be sure all segments are not merged yet by another process

        //

        // Segment merging functionality is concentrated in this class and surrounded

        // by optimization lock obtaining/releasing.

        // _updateSegments() refreshes segments list from the latest index generation.

        // So only new segments can be added to the index while we are merging some already existing

        // segments.

        // Newly added segments will be also included into the index by the _updateSegments() call

        // either by another process or by the current process with the commit() call at the end of _mergeSegments() method.

        // That's guaranteed by the serialisation of _updateSegments() execution using exclusive locks.

        $this->_updateSegments();

        // Perform standard auto-optimization procedure

        $segmentSizes = array();

        foreach ($this->_segmentInfos as $segName => $segmentInfo) {

            $segmentSizes[$segName] = $segmentInfo->count();

        }

        $mergePool   = array();

        $poolSize    = 0;

        $sizeToMerge = $this->maxBufferedDocs;

        asort($segmentSizes, SORT_NUMERIC);

        foreach ($segmentSizes as $segName => $size) {

            // Check, if segment comes into a new merging block

            while ($size >= $sizeToMerge) {

                // Merge previous block if it's large enough

                if ($poolSize >= $sizeToMerge) {

                    $this->_mergeSegments($mergePool);

                }

                $mergePool   = array();

                $poolSize    = 0;

                $sizeToMerge *= $this->mergeFactor;

                if ($sizeToMerge > $this->maxMergeDocs) {

                    Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);

                    return;

                }

            }

            $mergePool[] = $this->_segmentInfos[$segName];

            $poolSize += $size;

        }

        if ($poolSize >= $sizeToMerge) {

            $this->_mergeSegments($mergePool);

        }

        Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);

    }

    /**

     * Merge specified segments

     *

     * $segments is an array of SegmentInfo objects

     *

     * @param array $segments

     */

    private function _mergeSegments($segments)

    {

        $newName = $this->_newSegmentName();

        /** Zend_Search_Lucene_Index_SegmentMerger */

        require_once 'Zend/Search/Lucene/Index/SegmentMerger.php';

        $merger = new Zend_Search_Lucene_Index_SegmentMerger($this->_directory,

                                                             $newName);

        foreach ($segments as $segmentInfo) {

            $merger->addSource($segmentInfo);

            $this->_segmentsToDelete[$segmentInfo->getName()] = $segmentInfo->getName();

        }

        $newSegment = $merger->merge();

        if ($newSegment !== null) {

            $this->_newSegments[$newSegment->getName()] = $newSegment;

        }

        $this->commit();

    }

    /**

     * Update segments file by adding current segment to a list

     *

     * @throws Zend_Search_Lucene_Exception

     */

    private function _updateSegments()

    {

        // Get an exclusive index lock

        Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory);

        // Write down changes for the segments

        foreach ($this->_segmentInfos as $segInfo) {

            $segInfo->writeChanges();

        }

        $generation = Zend_Search_Lucene::getActualGeneration($this->_directory);

        $segmentsFile   = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false);

        $newSegmentFile = $this->_directory->createFile(Zend_Search_Lucene::getSegmentFileName(++$generation), false);

        try {

            $genFile = $this->_directory->getFileObject('segments.gen', false);

        } catch (Zend_Search_Lucene_Exception $e) {

            if (strpos($e->getMessage(), 'is not readable') !== false) {

                $genFile = $this->_directory->createFile('segments.gen');

            } else {

                throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);

            }

        }

        $genFile->writeInt((int)0xFFFFFFFE);

        // Write generation (first copy)

        $genFile->writeLong($generation);

        try {

            // Write format marker

            if ($this->_targetFormatVersion == Zend_Search_Lucene::FORMAT_2_1) {

                $newSegmentFile->writeInt((int)0xFFFFFFFD);

            } else if ($this->_targetFormatVersion == Zend_Search_Lucene::FORMAT_2_3) {

                $newSegmentFile->writeInt((int)0xFFFFFFFC);

            }

            // Read src file format identifier

            $format = $segmentsFile->readInt();

            if ($format == (int)0xFFFFFFFF) {

                $srcFormat = Zend_Search_Lucene::FORMAT_PRE_2_1;

            } else if ($format == (int)0xFFFFFFFD) {

                $srcFormat = Zend_Search_Lucene::FORMAT_2_1;

            } else if ($format == (int)0xFFFFFFFC) {

                $srcFormat = Zend_Search_Lucene::FORMAT_2_3;

            } else {

                throw new Zend_Search_Lucene_Exception('Unsupported segments file format');