<?php

/**

 * Zend Framework

 *

 * LICENSE

 *

 * This source file is subject to the new BSD license that is bundled

 * with this package in the file LICENSE.txt.

 * It is also available through the world-wide-web at this URL:

 * http://framework.zend.com/license/new-bsd

 * If you did not receive a copy of the license and are unable to

 * obtain it through the world-wide-web, please send an email

 * to license@zend.com so we can send you a copy immediately.

 *

 * @category   Zend

 * @package    Zend_Search_Lucene

 * @subpackage Index

 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)

 * @license    http://framework.zend.com/license/new-bsd     New BSD License

 * @version    $Id: SegmentInfo.php 22987 2010-09-21 10:39:53Z alexander $

 */

/** Zend_Search_Lucene_Index_TermsStream_Interface */

require_once 'Zend/Search/Lucene/Index/TermsStream/Interface.php';

/** Zend_Search_Lucene_Search_Similarity */

require_once 'Zend/Search/Lucene/Search/Similarity.php';

/** Zend_Search_Lucene_Index_FieldInfo */

require_once 'Zend/Search/Lucene/Index/FieldInfo.php';

/** Zend_Search_Lucene_Index_Term */

require_once 'Zend/Search/Lucene/Index/Term.php';

/** Zend_Search_Lucene_Index_TermInfo */

require_once 'Zend/Search/Lucene/Index/TermInfo.php';

/**

 * @category   Zend

 * @package    Zend_Search_Lucene

 * @subpackage Index

 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)

 * @license    http://framework.zend.com/license/new-bsd     New BSD License

 */

class Zend_Search_Lucene_Index_SegmentInfo implements Zend_Search_Lucene_Index_TermsStream_Interface

{

    /**

     * "Full scan vs fetch" boundary.

     *

     * If filter selectivity is less than this value, then full scan is performed

     * (since term entries fetching has some additional overhead).

     */

    const FULL_SCAN_VS_FETCH_BOUNDARY = 5;

    /**

     * Number of docs in a segment

     *

     * @var integer

     */

    private $_docCount;

    /**

     * Segment name

     *

     * @var string

     */

    private $_name;

    /**

     * Term Dictionary Index

     *

     * Array of arrays (Zend_Search_Lucene_Index_Term objects are represented as arrays because

     * of performance considerations)

     * [0] -> $termValue

     * [1] -> $termFieldNum

     *

     * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos

     *

     * @var array

     */

    private $_termDictionary;

    /**

     * Term Dictionary Index TermInfos

     *

     * Array of arrays (Zend_Search_Lucene_Index_TermInfo objects are represented as arrays because

     * of performance considerations)

     * [0] -> $docFreq

     * [1] -> $freqPointer

     * [2] -> $proxPointer

     * [3] -> $skipOffset

     * [4] -> $indexPointer

     *

     * @var array

     */

    private $_termDictionaryInfos;

    /**

     * Segment fields. Array of Zend_Search_Lucene_Index_FieldInfo objects for this segment

     *

     * @var array

     */

    private $_fields;

    /**

     * Field positions in a dictionary.

     * (Term dictionary contains filelds ordered by names)

     *

     * @var array

     */

    private $_fieldsDicPositions;

    /**

     * Associative array where the key is the file name and the value is data offset

     * in a compound segment file (.csf).

     *

     * @var array

     */

    private $_segFiles;

    /**

     * Associative array where the key is the file name and the value is file size (.csf).

     *

     * @var array

     */

    private $_segFileSizes;

    /**

     * Delete file generation number

     *

     * -2 means autodetect latest delete generation

     * -1 means 'there is no delete file'

     *  0 means pre-2.1 format delete file

     *  X specifies used delete file

     *

     * @var integer

     */

    private $_delGen;

    /**

     * Segment has single norms file

     *

     * If true then one .nrm file is used for all fields

     * Otherwise .fN files are used

     *

     * @var boolean

     */

    private $_hasSingleNormFile;

    /**

     * Use compound segment file (*.cfs) to collect all other segment files

     * (excluding .del files)

     *

     * @var boolean

     */

    private $_isCompound;

    /**

     * File system adapter.

     *

     * @var Zend_Search_Lucene_Storage_Directory_Filesystem

     */

    private $_directory;

    /**

     * Normalization factors.

     * An array fieldName => normVector

     * normVector is a binary string.

     * Each byte corresponds to an indexed document in a segment and

     * encodes normalization factor (float value, encoded by

     * Zend_Search_Lucene_Search_Similarity::encodeNorm())

     *

     * @var array

     */

    private $_norms = array();

    /**

     * List of deleted documents.

     * bitset if bitset extension is loaded or array otherwise.

     *

     * @var mixed

     */

    private $_deleted = null;

    /**

     * $this->_deleted update flag

     *

     * @var boolean

     */

    private $_deletedDirty = false;

    /**

     * True if segment uses shared doc store

     *

     * @var boolean

     */

    private $_usesSharedDocStore;

    /*

     * Shared doc store options.

     * It's an assotiative array with the following items:

     * - 'offset'     => $docStoreOffset           The starting document in the shared doc store files where this segment's documents begin

     * - 'segment'    => $docStoreSegment          The name of the segment that has the shared doc store files.

     * - 'isCompound' => $docStoreIsCompoundFile   True, if compound file format is used for the shared doc store files (.cfx file).

     */

    private $_sharedDocStoreOptions;

    /**

     * Zend_Search_Lucene_Index_SegmentInfo constructor

     *

     * @param Zend_Search_Lucene_Storage_Directory $directory

     * @param string     $name

     * @param integer    $docCount

     * @param integer    $delGen

     * @param array|null $docStoreOptions

     * @param boolean    $hasSingleNormFile

     * @param boolean    $isCompound

     */

    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name, $docCount, $delGen = 0, $docStoreOptions = null, $hasSingleNormFile = false, $isCompound = null)

    {

        $this->_directory = $directory;

        $this->_name      = $name;

        $this->_docCount  = $docCount;

        if ($docStoreOptions !== null) {

            $this->_usesSharedDocStore    = true;

            $this->_sharedDocStoreOptions = $docStoreOptions;

            if ($docStoreOptions['isCompound']) {

                $cfxFile       = $this->_directory->getFileObject($docStoreOptions['segment'] . '.cfx');

                $cfxFilesCount = $cfxFile->readVInt();

                $cfxFiles     = array();

                $cfxFileSizes = array();

                for ($count = 0; $count < $cfxFilesCount; $count++) {

                    $dataOffset = $cfxFile->readLong();

                    if ($count != 0) {

                        $cfxFileSizes[$fileName] = $dataOffset - end($cfxFiles);

                    }

                    $fileName            = $cfxFile->readString();

                    $cfxFiles[$fileName] = $dataOffset;

                }

                if ($count != 0) {

                    $cfxFileSizes[$fileName] = $this->_directory->fileLength($docStoreOptions['segment'] . '.cfx') - $dataOffset;

                }

                $this->_sharedDocStoreOptions['files']     = $cfxFiles;

                $this->_sharedDocStoreOptions['fileSizes'] = $cfxFileSizes;

            }

        }

        $this->_hasSingleNormFile = $hasSingleNormFile;

        $this->_delGen            = $delGen;

        $this->_termDictionary    = null;

        if ($isCompound !== null) {

            $this->_isCompound    = $isCompound;

        } else {

            // It's a pre-2.1 segment or isCompound is set to 'unknown'

            // Detect if segment uses compound file

            require_once 'Zend/Search/Lucene/Exception.php';

            try {

                // Try to open compound file

                $this->_directory->getFileObject($name . '.cfs');

                // Compound file is found

                $this->_isCompound = true;

            } catch (Zend_Search_Lucene_Exception $e) {

                if (strpos($e->getMessage(), 'is not readable') !== false) {

                    // Compound file is not found or is not readable

                    $this->_isCompound = false;

                } else {

                    throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);

                }

            }

        }

        $this->_segFiles = array();

        if ($this->_isCompound) {

            $cfsFile = $this->_directory->getFileObject($name . '.cfs');

            $segFilesCount = $cfsFile->readVInt();

            for ($count = 0; $count < $segFilesCount; $count++) {

                $dataOffset = $cfsFile->readLong();

                if ($count != 0) {

                    $this->_segFileSizes[$fileName] = $dataOffset - end($this->_segFiles);

                }

                $fileName = $cfsFile->readString();

                $this->_segFiles[$fileName] = $dataOffset;

            }

            if ($count != 0) {

                $this->_segFileSizes[$fileName] = $this->_directory->fileLength($name . '.cfs') - $dataOffset;

            }

        }

        $fnmFile = $this->openCompoundFile('.fnm');

        $fieldsCount = $fnmFile->readVInt();

        $fieldNames = array();

        $fieldNums  = array();

        $this->_fields = array();

        for ($count=0; $count < $fieldsCount; $count++) {

            $fieldName = $fnmFile->readString();

            $fieldBits = $fnmFile->readByte();

            $this->_fields[$count] = new Zend_Search_Lucene_Index_FieldInfo($fieldName,

                                                                            $fieldBits & 0x01 /* field is indexed */,

                                                                            $count,

                                                                            $fieldBits & 0x02 /* termvectors are stored */,

                                                                            $fieldBits & 0x10 /* norms are omitted */,

                                                                            $fieldBits & 0x20 /* payloads are stored */);

            if ($fieldBits & 0x10) {

                // norms are omitted for the indexed field

                $this->_norms[$count] = str_repeat(chr(Zend_Search_Lucene_Search_Similarity::encodeNorm(1.0)), $docCount);

            }

            $fieldNums[$count]  = $count;

            $fieldNames[$count] = $fieldName;

        }

        array_multisort($fieldNames, SORT_ASC, SORT_REGULAR, $fieldNums);

        $this->_fieldsDicPositions = array_flip($fieldNums);

        if ($this->_delGen == -2) {

            // SegmentInfo constructor is invoked from index writer

            // Autodetect current delete file generation number

            $this->_delGen = $this->_detectLatestDelGen();

        }

        // Load deletions

        $this->_deleted = $this->_loadDelFile();

    }

    /**

     * Load detetions file

     *

     * Returns bitset or an array depending on bitset extension availability

     *

     * @return mixed

     * @throws Zend_Search_Lucene_Exception

     */

    private function _loadDelFile()

    {

        if ($this->_delGen == -1) {

            // There is no delete file for this segment

            return null;

        } else if ($this->_delGen == 0) {

            // It's a segment with pre-2.1 format delete file

            // Try to load deletions file

            return $this->_loadPre21DelFile();

        } else {

            // It's 2.1+ format deleteions file

            return $this->_load21DelFile();

        }

    }

    /**

     * Load pre-2.1 detetions file

     *

     * Returns bitset or an array depending on bitset extension availability

     *

     * @return mixed

     * @throws Zend_Search_Lucene_Exception

     */

    private function _loadPre21DelFile()

    {

        require_once 'Zend/Search/Lucene/Exception.php';

        try {

            // '.del' files always stored in a separate file

            // Segment compound is not used

            $delFile = $this->_directory->getFileObject($this->_name . '.del');

            $byteCount = $delFile->readInt();

            $byteCount = ceil($byteCount/8);

            $bitCount  = $delFile->readInt();

            if ($bitCount == 0) {

                $delBytes = '';

            } else {

                $delBytes = $delFile->readBytes($byteCount);

            }

            if (extension_loaded('bitset')) {

                return $delBytes;

            } else {

                $deletions = array();

                for ($count = 0; $count < $byteCount; $count++) {

                    $byte = ord($delBytes[$count]);

                    for ($bit = 0; $bit < 8; $bit++) {

                        if ($byte & (1<<$bit)) {

                            $deletions[$count*8 + $bit] = 1;

                        }

                    }

                }

                return $deletions;

            }

        } catch(Zend_Search_Lucene_Exception $e) {

            if (strpos($e->getMessage(), 'is not readable') === false) {

                throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);

            }

            // There is no deletion file

            $this->_delGen = -1;

            return null;

        }

    }

    /**

     * Load 2.1+ format detetions file

     *

     * Returns bitset or an array depending on bitset extension availability

     *

     * @return mixed

     */

    private function _load21DelFile()

    {

        $delFile = $this->_directory->getFileObject($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');

        $format = $delFile->readInt();

        if ($format == (int)0xFFFFFFFF) {

            if (extension_loaded('bitset')) {

                $deletions = bitset_empty();

            } else {

                $deletions = array();

            }

            $byteCount = $delFile->readInt();

            $bitCount  = $delFile->readInt();

            $delFileSize = $this->_directory->fileLength($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');

            $byteNum = 0;

            do {

                $dgap = $delFile->readVInt();

                $nonZeroByte = $delFile->readByte();

                $byteNum += $dgap;