Query specific matches highlighting

@param Zend_Search_Lucene_Search_Highlighter_Interface $highlighter Highlighter object (also contains doc for highlighting)
356490
protected function _highlightMatches(Zend_Search_Lucene_Search_Highlighter_Interface $highlighter)
    {
        $words = array();

        if ($this->_signs === null) {
            foreach ($this->_terms as $term) {
                $words[] = $term->text;
            }
        } else {
            foreach ($this->_signs as $id => $sign) {
                if ($sign !== false) {
                    $words[] = $this->_terms[$id]->text;
                }
            }
        }

        $highlighter->highlight($words);
    }
280460
public function performSearch()
    {
        try {
            $index = Zend_Search_Lucene::open(self::get_index_location());

            Zend_Search_Lucene::setResultSetLimit(100);
            
            $query = new Zend_Search_Lucene_Search_Query_Boolean();
            $term = Zend_Search_Lucene_Search_QueryParser::parse($this->getQuery());
            $query->addSubquery($term, true);
            
            if ($this->modules) {
                $moduleQuery = new Zend_Search_Lucene_Search_Query_MultiTerm();
                
                foreach ($this->modules as $module) {
                    $moduleQuery->addTerm(new Zend_Search_Lucene_Index_Term($module, 'Entity'));
                }
                
                $query->addSubquery($moduleQuery, true);
            }

            if ($this->versions) {
                $versionQuery = new Zend_Search_Lucene_Search_Query_MultiTerm();
                
                foreach ($this->versions as $version) {
                    $versionQuery->addTerm(new Zend_Search_Lucene_Index_Term($version, 'Version'));
                }
                
                $query->addSubquery($versionQuery, true);
            }
            
            $er = error_reporting();
            error_reporting('E_ALL ^ E_NOTICE');
            $this->results = $index->find($query);
            error_reporting($er);
            $this->totalResults = $index->numDocs();
        } catch (Zend_Search_Lucene_Exception $e) {
            user_error($e .'. Ensure you have run the rebuld task (/dev/tasks/RebuildLuceneDocsIndex)', E_USER_ERROR);
        }
    }
364788
protected function determineMatches(Matcher $matcher, Result $control, array $trials = [])
    {
        foreach ($trials as $trial) {
            if ($matcher->match($control->getValue(), $trial->getValue())) {
                $trial->setMatch(true);
            }
        }
    }
581995
public function termDocs(Zend_Search_Lucene_Index_Term $term, $docsFilter = null)
    {
        $subResults = array();
        $segmentStartDocId = 0;

        foreach ($this->_segmentInfos as $segmentInfo) {
            $subResults[] = $segmentInfo->termDocs($term, $segmentStartDocId, $docsFilter);

            $segmentStartDocId += $segmentInfo->count();
        }

        if (count($subResults) == 0) {
            return array();
        } else if (count($subResults) == 1) {
            // Index is optimized (only one segment)
            // Do not perform array reindexing
            return reset($subResults);
        } else {
            $result = call_user_func_array('array_merge', $subResults);
        }

        return $result;
    }
1039856
public function execute(Zend_Search_Lucene_Interface $reader, $docsFilter = null)
    {
        $this->_resVector = null;

        if (count($this->_terms) == 0) {
            $this->_resVector = array();
        }

        $resVectors      = array();
        $resVectorsSizes = array();
        $resVectorsIds   = array(); // is used to prevent arrays comparison
        foreach ($this->_terms as $termId => $term) {
            $resVectors[]      = array_flip($reader->termDocs($term));
            $resVectorsSizes[] = count(end($resVectors));
            $resVectorsIds[]   = $termId;

            $this->_termsPositions[$termId] = $reader->termPositions($term);
        }
        // sort resvectors in order of subquery cardinality increasing
        array_multisort(
            $resVectorsSizes, SORT_ASC, SORT_NUMERIC,
            $resVectorsIds,   SORT_ASC, SORT_NUMERIC,
            $resVectors
        );

        foreach ($resVectors as $nextResVector) {
            if($this->_resVector === null) {
                $this->_resVector = $nextResVector;
            } else {
                //$this->_resVector = array_intersect_key($this->_resVector, $nextResVector);

                /**
                 * This code is used as workaround for array_intersect_key() slowness problem.
                 */
                $updatedVector = array();
                foreach ($this->_resVector as $id => $value) {
                    if (isset($nextResVector[$id])) {
                        $updatedVector[$id] = $value;
                    }
                }
                $this->_resVector = $updatedVector;
            }

            if (count($this->_resVector) == 0) {
                // Empty result set, we don't need to check other terms
                break;
            }
        }

        // ksort($this->_resVector, SORT_NUMERIC);
        // Docs are returned ordered. Used algorithm doesn't change elements order.

        // Initialize weight if it's not done yet
        $this->_initWeight($reader);
    }
977298
private static function seekToNextType(ArrayIterator $tokenIterator, $type)
    {
        while ($tokenIterator->valid()) {
            if (self::isTokenType($tokenIterator->current(), $type)) {
                return;
            }
            $tokenIterator->next();
        }
    }
1315020
public function highlight($field)
    {
        if (isset($this->highlighted[$field])) {
            return $this->highlighted[$field];
        }

        return false;
    }
247264
public function searchBy(
        $conditions = [],
        $sorts = [],
        $joins = [],
        $pager = null,
        $mode = 'and',
        $expression = null
    ) {
        if (!$expression) {
            return $this->search($conditions, $sorts, $joins, $pager, $mode);
        }

        $queryBuilder = $this->createQueryBuilder('o')->select('o');

        $this->queryBuilderExpressionAndConditions($queryBuilder, $conditions, $expression, $mode);
        $this->queryBuilderSorts($queryBuilder, $sorts);
        $this->queryBuilderJoins($queryBuilder, $joins);
        $this->queryBuilderPager($queryBuilder, $pager);

        return $queryBuilder->getQuery()->getResult();
    }
673852
protected function _highlightMatches(Zend_Search_Lucene_Search_Highlighter_Interface $highlighter)
    {
        /**
 * Skip fields detection. We don't need it, since we expect all fields presented in the HTML body and don't differentiate them 
*/

        /**
 * Skip exact term matching recognition, keyword fields highlighting is not supported 
*/

        // -------------------------------------
        // Recognize wildcard queries

        /**
 * @todo check for PCRE unicode support may be performed through Zend_Environment in some future 
*/
        if (@preg_match('/\pL/u', 'a') == 1) {
            $subPatterns = preg_split('/[*?]/u', iconv($this->_encoding, 'UTF-8', $this->_word));
        } else {
            $subPatterns = preg_split('/[*?]/', $this->_word);
        }
        if (count($subPatterns) > 1) {
            // Do nothing
            return;
        }


        // -------------------------------------
        // Recognize one-term multi-term and "insignificant" queries
        include_once 'Zend/Search/Lucene/Analysis/Analyzer.php';
        $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_word, $this->_encoding);
        if (count($tokens) == 0) {
            // Do nothing
            return;
        }
        if (count($tokens) == 1) {
            include_once 'Zend/Search/Lucene/Index/Term.php';
            $term  = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field);
            include_once 'Zend/Search/Lucene/Search/Query/Fuzzy.php';
            $query = new Zend_Search_Lucene_Search_Query_Fuzzy($term, $this->_minimumSimilarity);

            $query->_highlightMatches($highlighter);
            return;
        }

        // Word is tokenized into several tokens
        // But fuzzy search is supported only for non-multiple word terms
        // Do nothing
    }
1370867
public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) 
    {
        if (array_key_exists($srcToken->getTermText(), $this->_stopSet)) {
            return null;
        } else {
            return $srcToken;
        }
    }
166832
private function getHighlight()
    {
        if (Utils::findKey($this->options, 'highlight', false))
        {
            foreach ($this->fields as $field)
            {
                $this->payload['highlight']['fields'][$field] = new StdClass();
            }

            if ($tag = Utils::findKey($this->options['highlight'], 'tag', false))
            {
                $this->payload['highlight']['pre_tags'] = [$tag];
                $this->payload['highlight']['post_tags'] = [preg_replace('/\A</', '</', $tag)];
            }
        }
    }
1099678
protected function _highlightMatches(Zend_Search_Lucene_Search_Highlighter_Interface $highlighter)
    {
        $words = array();

        include_once 'Zend/Search/Lucene/Index/Term.php';
        $prefix           = Zend_Search_Lucene_Index_Term::getPrefix($this->_term->text, $this->_prefixLength);
        $prefixByteLength = strlen($prefix);
        $prefixUtf8Length = Zend_Search_Lucene_Index_Term::getLength($prefix);

        $termLength       = Zend_Search_Lucene_Index_Term::getLength($this->_term->text);

        $termRest         = substr($this->_term->text, $prefixByteLength);
        // we calculate length of the rest in bytes since levenshtein() is not UTF-8 compatible
        $termRestLength   = strlen($termRest);

        $scaleFactor = 1/(1 - $this->_minimumSimilarity);

        $docBody = $highlighter->getDocument()->getFieldUtf8Value('body');
        include_once 'Zend/Search/Lucene/Analysis/Analyzer.php';
        $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($docBody, 'UTF-8');
        foreach ($tokens as $token) {
            $termText = $token->getTermText();

            if (substr($termText, 0, $prefixByteLength) == $prefix) {
                // Calculate similarity
                $target = substr($termText, $prefixByteLength);

                $maxDistance = isset($this->_maxDistances[strlen($target)])?
                                   $this->_maxDistances[strlen($target)] :
                                   $this->_calculateMaxDistance($prefixUtf8Length, $termRestLength, strlen($target));

                if ($termRestLength == 0) {
                    // we don't have anything to compare.  That means if we just add
                    // the letters for current term we get the new word
                    $similarity = (($prefixUtf8Length == 0)? 0 : 1 - strlen($target)/$prefixUtf8Length);
                } else if (strlen($target) == 0) {
                    $similarity = (($prefixUtf8Length == 0)? 0 : 1 - $termRestLength/$prefixUtf8Length);
                } else if ($maxDistance < abs($termRestLength - strlen($target))) {
                    //just adding the characters of term to target or vice-versa results in too many edits
                    //for example "pre" length is 3 and "prefixes" length is 8.  We can see that
                    //given this optimal circumstance, the edit distance cannot be less than 5.
                    //which is 8-3 or more precisesly abs(3-8).
                    //if our maximum edit distance is 4, then we can discard this word
                    //without looking at it.
                    $similarity = 0;
                } else {
                    $similarity = 1 - levenshtein($termRest, $target)/($prefixUtf8Length + min($termRestLength, strlen($target)));
                }

                if ($similarity > $this->_minimumSimilarity) {
                    $words[] = $termText;
                }
            }
        }

        $highlighter->highlight($words);
    }
536828
public function execute(Zend_Search_Lucene_Interface $reader, $docsFilter = null)
    {
        if ($this->_signs === null) {
            $this->_calculateConjunctionResult($reader);
        } else {
            $this->_calculateNonConjunctionResult($reader);
        }

        // Initialize weight if it's not done yet
        $this->_initWeight($reader);
    }
1030541
public function execute(Zend_Search_Lucene_Interface $reader, $docsFilter = null)
    {
        // Initialize weight if it's not done yet
        $this->_initWeight($reader);

        if ($docsFilter === null) {
            // Create local documents filter if it's not provided by upper query
            include_once 'Zend/Search/Lucene/Index/DocsFilter.php';
            $docsFilter = new Zend_Search_Lucene_Index_DocsFilter();
        }

        foreach ($this->_subqueries as $subqueryId => $subquery) {
            if ($this->_signs == null  ||  $this->_signs[$subqueryId] === true) {
                // Subquery is required
                $subquery->execute($reader, $docsFilter);
            } else {
                $subquery->execute($reader);
            }
        }

        if ($this->_signs === null) {
            $this->_calculateConjunctionResult();
        } else {
            $this->_calculateNonConjunctionResult();
        }
    }
1332971
public function addDocument(Zend_Search_Lucene_Document $document)
    {
        /**
 * Zend_Search_Lucene_Search_Similarity 
*/
        include_once 'Zend/Search/Lucene/Search/Similarity.php';

        $storedFields = array();
        $docNorms     = array();
        $similarity   = Zend_Search_Lucene_Search_Similarity::getDefault();

        foreach ($document->getFieldNames() as $fieldName) {
            $field = $document->getField($fieldName);

            if ($field->storeTermVector) {
                /**
                 * @todo term vector storing support
                 */
                include_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Store term vector functionality is not supported yet.');
            }

            if ($field->isIndexed) {
                if ($field->isTokenized) {
                    /**
 * Zend_Search_Lucene_Analysis_Analyzer 
*/
                    include_once 'Zend/Search/Lucene/Analysis/Analyzer.php';

                    $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault();
                    $analyzer->setInput($field->value, $field->encoding);

                    $position     = 0;
                    $tokenCounter = 0;
                    while (($token = $analyzer->nextToken()) !== null) {
                        $tokenCounter++;

                        $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $field->name);
                        $termKey = $term->key();

                        if (!isset($this->_termDictionary[$termKey])) {
                            // New term
                            $this->_termDictionary[$termKey] = $term;
                            $this->_termDocs[$termKey] = array();
                            $this->_termDocs[$termKey][$this->_docCount] = array();
                        } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) {
                            // Existing term, but new term entry
                            $this->_termDocs[$termKey][$this->_docCount] = array();
                        }
                        $position += $token->getPositionIncrement();
                        $this->_termDocs[$termKey][$this->_docCount][] = $position;
                    }

                    if ($tokenCounter == 0) {
                        // Field contains empty value. Treat it as non-indexed and non-tokenized
                        $field = clone($field);
                        $field->isIndexed = $field->isTokenized = false;
                    } else {
                        $docNorms[$field->name] = chr(
                            $similarity->encodeNorm(
                                $similarity->lengthNorm(
                                    $field->name,
                                    $tokenCounter
                                )*
                                                                               $document->boost*
                                $field->boost 
                            )
                        );
                    }
                } else if (($fieldUtf8Value = $field->getUtf8Value()) == '') {
                    // Field contains empty value. Treat it as non-indexed and non-tokenized
                    $field = clone($field);
                    $field->isIndexed = $field->isTokenized = false;
                } else {
                    $term = new Zend_Search_Lucene_Index_Term($fieldUtf8Value, $field->name);
                    $termKey = $term->key();

                    if (!isset($this->_termDictionary[$termKey])) {
                        // New term
                        $this->_termDictionary[$termKey] = $term;
                        $this->_termDocs[$termKey] = array();
                        $this->_termDocs[$termKey][$this->_docCount] = array();
                    } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) {
                        // Existing term, but new term entry
                        $this->_termDocs[$termKey][$this->_docCount] = array();
                    }
                    $this->_termDocs[$termKey][$this->_docCount][] = 0; // position

                    $docNorms[$field->name] = chr(
                        $similarity->encodeNorm(
                            $similarity->lengthNorm($field->name, 1)*
                                                                           $document->boost*
                            $field->boost 
                        )
                    );
                }
            }

            if ($field->isStored) {
                $storedFields[] = $field;
            }

            $this->addField($field);
        }

        foreach ($this->_fields as $fieldName => $field) {
            if (!$field->isIndexed) {
                continue;
            }

            if (!isset($this->_norms[$fieldName])) {
                $this->_norms[$fieldName] = str_repeat(
                    chr($similarity->encodeNorm($similarity->lengthNorm($fieldName, 0))),
                    $this->_docCount
                );
            }

            if (isset($docNorms[$fieldName])) {
                $this->_norms[$fieldName] .= $docNorms[$fieldName];
            } else {
                $this->_norms[$fieldName] .= chr($similarity->encodeNorm($similarity->lengthNorm($fieldName, 0)));
            }
        }

        $this->addStoredFields($storedFields);
    }
83461
public void setHighlighter(String highlighter) {

        try {
            m_highlighter = (I_CmsTermHighlighter)Class.forName(highlighter).newInstance();
        } catch (Exception e) {
            m_highlighter = null;
            LOG.error(e.getLocalizedMessage(), e);
        }
    }
1074191
public function addDocument(Zend_Search_Lucene_Document $document)
    {
        /**
 * Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter 
*/
        include_once 'Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php';

        if ($this->_currentSegment === null) {
            $this->_currentSegment =
                new Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter($this->_directory, $this->_newSegmentName());
        }
        $this->_currentSegment->addDocument($document);

        if ($this->_currentSegment->count() >= $this->maxBufferedDocs) {
            $this->commit();
        }

        $this->_maybeMergeSegments();

        $this->_versionUpdate++;
    }