diff -r bd595ad770fc -r 1c2f13fd785c web/enmi/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/web/enmi/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php Thu Jan 20 19:30:54 2011 +0100 @@ -0,0 +1,95 @@ +_position = 0; + + if ($this->_input === null) { + return; + } + + // convert input into ascii + if (PHP_OS != 'AIX') { + $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input); + } + $this->_encoding = 'ASCII'; + } + + /** + * Tokenization stream API + * Get next token + * Returns null at the end of stream + * + * @return Zend_Search_Lucene_Analysis_Token|null + */ + public function nextToken() + { + if ($this->_input === null) { + return null; + } + + do { + if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) { + // It covers both cases a) there are no matches (preg_match(...) === 0) + // b) error occured (preg_match(...) === FALSE) + return null; + } + + $str = $match[0][0]; + $pos = $match[0][1]; + $endpos = $pos + strlen($str); + + $this->_position = $endpos; + + $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos)); + } while ($token === null); // try again if token is skipped + + return $token; + } +} +