Add '"' chars if necessary to a token value, and escape Lucene query
syntax reserved chars

@param string $token
String to escape
@param boolean $force = FALSE
Escape whatever happens

@return string
512648
public static function escapeToken($token, $force = false)
    {
        $escaped = preg_replace(self::RE_SPECIALS, '\\\\\\0', $token);

        if ($force || preg_match('/ /', $escaped) || strlen($token) != strlen($escaped)) {
            return '"' . $escaped . '"';
        } else {
            return $escaped;
        }
    }
249346
public function parse(Token $token)
    {
        parent::parse($token);

        $split = preg_split(self::REGEX, $token->getValue(), 2, PREG_SPLIT_DELIM_CAPTURE + PREG_SPLIT_NO_EMPTY);
        $splitSize = count($split);
        if (1 === $splitSize) {
            return $this->functionTokenParser->parse($token);
        }

        if (3 !== count($split) && 4 !== count($split)) {
            throw ExpressionLanguageExceptionFactory::createForCouldNotLexValue($token->getValue());
        }

        $values = [
            $this->parser->parse($split[0].$split[1]),
        ];
        if (3 === $splitSize) {
            $values[] = $this->parser->parse('<'.$split[2]);
        }

        if (4 === $splitSize) {
            $values[] = $this->parser->parse($split[2]);
            $values[] = $this->parser->parse('<'.$split[3]);
        }

        return $this->mergeValues($values);
    }
1007516
private function ConsumeChar( $token )
	{
		$curr = new CurrentToken();
		$curr->token = $token;
		$curr->value = null;
		$curr->anchor = $this->m_anchor;
		$curr->length = 1;
		array_push( $this->m_token, $curr );
	}
517117
public function match($source, TokenFactoryInterface $factory)
    {
        $pos = 0;
        $len = strlen($this->_substr);

        while (($pos = strpos($source, $this->_substr, $pos)) !== false) {
            yield $factory->create(Token::NAME, ['pos' => $pos, 'length' => $len]);

            $pos += $len;
        }
    }
486244
public function enterToken(array $token, $content)
    {
        switch ($token[Lexer::TYPE]) {
            case Lexer::T_ARRAY_CLOSE:
                return $this->parent;

            case Lexer::T_ARRAY_ARROW:
                $this->currentKey = array_pop($this->values);

                if (!is_scalar($this->currentKey)) {
                    throw new TokenException('array key must be scalar', $token);
                }
                break;

            case Lexer::T_QUOTED_STRING:
            case Lexer::T_NUMBER:
            case Lexer::T_TRUE:
            case Lexer::T_FALSE:
            case Lexer::T_NULL:
                $value = $this->getValueOfToken($token);
                $this->addValue($value);
                break;

            case Lexer::T_ARRAY_OPEN:
                $array = new ArrayContext($token, $this);
                $this->addValue($array);

                return $array;
                break;

            case Lexer::T_FUNC_OPEN:
                $func = new FuncContext($token, $this);
                $this->addValue($func);

                return $func;

            case Lexer::T_COMMA:
                if ($this->currentKey !== null) {
                    return null;
                }
                break;

            default:
                return null;
        }

        return $this;
    }
996434
private function indexCode($force = false) {
        if (!$this->index_code || $force) {
            foreach ($this->index as $word => $code) {
                $this->index_code[$code] = $word;
            }
        }
    }
408654
private function getSolrFieldPrefix(WordToken $token)
    {
        if ($token->domain === '') {
            return '';
        }

        if (isset($this->domainFieldMap[$token->domain])) {
            return $this->domainFieldMap[$token->domain] . ':';
        }

        return $this->defaultFieldName . ':';
    }
348029
private static function getSegmentFromToken($token, $key, array $tokens)
    {
        $relation_tokens = array('>', '~', '+');

        if (\in_array($token, $relation_tokens, true)) { // not a segment
            return false;
        }

        $segment = (object) array(
            'selector' => '',
            'relation_token' => false,
            'attribute_filters' => array(),
            'pseudo_filters' => array()
        );

        if (isset($tokens[$key-1]) && \in_array($tokens[$key-1], $relation_tokens, true)) { // get relationship token
            $segment->relation_token = $tokens[$key-1];
        }

        if (ctype_alpha($token)) { // simple element selector
            $segment->selector = $token;
            return $segment;
        }

        $char_tmp_replaced = false;
        if (strpos($token, '\\') !== false) {
            $token = preg_replace_callback( // temporary replace escaped characters
                '#(\\\\)(.{1})#',
                function ($matches) {
                    return 'ESCAPED'. \ord($matches[2]);
                },
                $token
            );
            $char_tmp_replaced = true;
        }

        $segment->selector = $token;

        while (preg_match('/(.*)\:(not|contains|has)\((.+)\)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2].'('.$matches[3].')';
        }

        while (preg_match('/(.*)\:([a-z][a-z\-]+)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\[([^]]+)\]$/', $segment->selector, $matches)) { // attribute selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\.([^\.\#]+)$/i', $segment->selector, $matches)) { // class selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'class~="'.$matches[2].'"';
        }

        while (preg_match('/(.*)\#([^\.\#]+)$/i', $segment->selector, $matches)) { // id selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'id="'.$matches[2].'"';
        }

        if ($char_tmp_replaced) { // restore temporary replaced characters
            $set_escape_back = function (string $str) {
                return preg_replace_callback(
                    '#(ESCAPED)([0-9]{1,3})#',
                    function ($matches) {
                        return \chr($matches[2]);
                    },
                    $str
                );
            };

            $segment->selector = $set_escape_back($segment->selector);

            foreach ($segment->attribute_filters as &$attr_filter) {
                $attr_filter = $set_escape_back($attr_filter);
            }
        }

        return $segment;
    }
862702
public static function getSpaceFromToken($token) {
        $pieces = explode('::', $token);
        if ( count($pieces) != 4 ) return false;
        $space = implode('::', array_slice($pieces,0,2));
        return $space;
    }
875866
private function match($token)
    {
        if ( ! $this->lexer->isNextToken($token) ) {
            $this->syntaxError($this->lexer->getLiteral($token));
        }

        return $this->lexer->moveNext();
    }
859191
public static function getTokenName($token)
    {
        if (is_int($token[self::T_KEY])) {
            return token_name($token[self::T_KEY]);
        } else {
            return $token[self::T_KEY];
        }
    }
496887
public function decodeToken($token)
    {
        try {
            $parser = new Parser();

            if (!$token = $parser->parse((string) $token)) {
                throw new Exception('Token inválido');
            }

            return $token;
        } catch (Exception $e) {
            abort(401, 'Favor efetuar o login novamente');
        }
    }
84038
protected function formatTokenTypeReservedTopLevel($addedNewline, $tab, $token, $queryValue)
    {
        $this->indentation
            ->setIncreaseSpecialIndent(true)
            ->decreaseSpecialIndentIfCurrentIndentTypeIsSpecial();

        $this->newLine->writeNewLineBecauseOfTopLevelReservedWord($addedNewline, $tab);

        if (WhiteSpace::tokenHasExtraWhiteSpaces($token)) {
            $queryValue = \preg_replace('/\s+/', ' ', $queryValue);
        }
        Token::tokenHasLimitClause($token, $this->parentheses, $this);

        return $queryValue;
    }
835597
public static function tokenHasExtraWhiteSpaceLeft($token)
    {
        return
            $token[Tokenizer::TOKEN_VALUE] === '.'
            || $token[Tokenizer::TOKEN_VALUE] === ','
            || $token[Tokenizer::TOKEN_VALUE] === ';';
    }
1236456
protected function getTextAction(Token $token)
    {
        // action to use for each line of the content of a <pre> Tag
        $tagPreBr = new Node('br', array('style' => array(), 'num' => 0), false);

        $actions = array();

        // if we are not in a <pre> tag
        if (!$this->tagPreIn) {
            // save the action
            $actions[] = new Node('write', array('txt' => $this->textParser->prepareTxt($token->getData())), false);
        } else { // else (if we are in a <pre> tag)
            // prepare the text
            $data = str_replace("\r", '', $token->getData());
            $lines = explode("\n", $data);

            // foreach line of the text
            foreach ($lines as $k => $txt) {
                // transform the line
                $txt = str_replace("\t", self::HTML_TAB, $txt);
                $txt = str_replace(' ', '&nbsp;', $txt);

                // add a break line
                if ($k > 0) {
                    $actions[] = clone $tagPreBr;
                }

                // save the action
                $actions[] = new Node('write', array('txt' => $this->textParser->prepareTxt($txt, false)), false);
            }
        }
        return $actions;
    }
568952
public function renderAtxContent(Token $token) {
        $level = $token->get('level', 1);
        $content = $token->get('content', '');

        return "\n" . str_pad('', $level, '#') . ' ' . $content;
    }
1023721
protected function _scopeWords(Query $query, TokenInterface $token)
    {
        if ($this->_isFullTextEnabled()) {
            return $this->_scopeWordsInFulltext($query, $token);
        }

        $like = 'LIKE';
        if ($token->negated()) {
            $like = 'NOT LIKE';
        }

        // * Matches any one or more characters.
        // ! Matches any single character.
        $value = str_replace(['*', '!'], ['%', '_'], $token->value());

        if ($token->where() === 'or') {
            $query->orWhere(["SearchDatasets.words {$like}" => "%{$value}%"]);
        } elseif ($token->where() === 'and') {
            $query->andWhere(["SearchDatasets.words {$like}" => "%{$value}%"]);
        } else {
            $query->where(["SearchDatasets.words {$like}" => "%{$value}%"]);
        }

        return $query;
    }