Handle lexers unmatched state

@param string                  $content
@param int                     $position
@param \ViKon\Parser\TokenList $tokenList
19904
protected function handleUnmatchedState($content, $position, TokenList $tokenList) {
        if (!empty($content)) {
            $this->parseContent($content, $tokenList);
        }
    }
530390
protected void parse( String content ) {
        Tokenizer tokenizer = new CndTokenizer(false, true);
        TokenStream tokens = new TokenStream(content, tokenizer, false);
        tokens.start();
        while (tokens.hasNext()) {
            // Keep reading while we can recognize one of the two types of statements ...
            if (tokens.matches("<", ANY_VALUE, "=", ANY_VALUE, ">")) {
                parseNamespaceMapping(tokens);
            } else if (tokens.matches("[", ANY_VALUE, "]")) {
                parseNodeTypeDefinition(tokens);
            } else {
                Position position = tokens.previousPosition();
                throw new ParsingException(position, CndI18n.expectedNamespaceOrNodeDefinition.text(tokens.consume(),
                                                                                                    position.getLine(),
                                                                                                    position.getColumn()));
            }
        }
    }
451504
public function isTokenEqualToOrFollowedBy($tokenPosition, $stringValue)
    {
        return $this->isTokenEqualTo($tokenPosition, $stringValue)
            || $this->isTokenFollowedBy($tokenPosition, $stringValue);
    }
588889
protected function handleEndState($content, $position, TokenList $tokenList) {
        if (!empty($content)) {
            $tokenList->addToken($this->name, $position)
                ->set('content', $content);
        }
    }
266234
final public void addPosition(int position) {
    if (tokenPosition == null) {
      tokenPosition = new MtasPosition(position);
    } else {
      tokenPosition.add(position);
    }
  }
746909
protected function removeTagContent(Node $body, $position, $pattern): void
    {
        if ($body->getNode($position) instanceof TextNode) {
            $positionBody = $body->getNode($position)->getAttribute('data');
            $positionBody = preg_replace($pattern, '', $positionBody);

            $body->getNode($position)->setAttribute('data', $positionBody);
        }
    }
259716
public function getTokenStringValueAt($tokenPosition)
    {
        if (!isset($this->tokens[$tokenPosition])) {
            throw new \OutOfBoundsException(sprintf('Token at offset %d does not exist', $tokenPosition));
        }

        $originalToken = $this->tokens[$tokenPosition];

        return is_string($originalToken) ? $originalToken : $originalToken[static::TOKEN_ORIGINAL_VALUE_OFFSET];
    }
604202
public static function parse(Parser $parser, TokensList $list, array $options = [])
    {
        $ret = new self();

        /**
         * The state of the parser.
         *
         * Below are the states of the parser.
         *
         *      0 -------------------[ data type ]--------------------> 1
         *
         *      1 ----------------[ size and options ]----------------> 2
         *
         * @var int
         */
        $state = 0;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // Skipping whitespaces and comments.
            if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
                continue;
            }

            if ($state === 0) {
                $ret->name = strtoupper((string) $token->value);
                if (($token->type !== Token::TYPE_KEYWORD) || (! ($token->flags & Token::FLAG_KEYWORD_DATA_TYPE))) {
                    $parser->error('Unrecognized data type.', $token);
                }
                $state = 1;
            } elseif ($state === 1) {
                if (($token->type === Token::TYPE_OPERATOR) && ($token->value === '(')) {
                    $parameters = ArrayObj::parse($parser, $list);
                    ++$list->idx;
                    $ret->parameters = (($ret->name === 'ENUM') || ($ret->name === 'SET')) ?
                        $parameters->raw : $parameters->values;
                }
                $ret->options = OptionsArray::parse($parser, $list, static::$DATA_TYPE_OPTIONS);
                ++$list->idx;
                break;
            }
        }

        if (empty($ret->name)) {
            return null;
        }

        --$list->idx;

        return $ret;
    }
36152
public function parseToken($content, $position, $state, TokenList $tokenList) {
        if (!empty($content)) {
            $tokenList->addToken($this->name, $position)
                ->set('content', $content);
        }
    }
384820
public static function parse(Parser $parser, TokensList $list, array $options = [])
    {
        $ret = new self();

        /**
         * Counts brackets.
         *
         * @var int
         */
        $brackets = 0;

        /**
         * The state of the parser.
         *
         * Below are the states of the parser.
         *
         *      0 ---------------------[ options ]---------------------> 1
         *
         *      1 ----------------------[ field ]----------------------> 2
         *
         *      2 -------------------------[ , ]-----------------------> 0
         *
         * @var int
         */
        $state = 0;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // End of statement.
            if ($token->type === Token::TYPE_DELIMITER) {
                break;
            }

            // Skipping comments.
            if ($token->type === Token::TYPE_COMMENT) {
                continue;
            }

            // Skipping whitespaces.
            if ($token->type === Token::TYPE_WHITESPACE) {
                if ($state === 2) {
                    // When parsing the unknown part, the whitespaces are
                    // included to not break anything.
                    $ret->unknown[] = $token;
                }
                continue;
            }

            if ($state === 0) {
                $ret->options = OptionsArray::parse($parser, $list, $options);

                if ($ret->options->has('AS')) {
                    for (; $list->idx < $list->count; ++$list->idx) {
                        if ($list->tokens[$list->idx]->type === Token::TYPE_DELIMITER) {
                            break;
                        }
                        $ret->unknown[] = $list->tokens[$list->idx];
                    }
                    break;
                }

                $state = 1;
            } elseif ($state === 1) {
                $ret->field = Expression::parse(
                    $parser,
                    $list,
                    [
                        'breakOnAlias' => true,
                        'parseField' => 'column',
                    ]
                );
                if ($ret->field === null) {
                    // No field was read. We go back one token so the next
                    // iteration will parse the same token, but in state 2.
                    --$list->idx;
                }
                $state = 2;
            } elseif ($state === 2) {
                if ($token->type === Token::TYPE_OPERATOR) {
                    if ($token->value === '(') {
                        ++$brackets;
                    } elseif ($token->value === ')') {
                        --$brackets;
                    } elseif (($token->value === ',') && ($brackets === 0)) {
                        break;
                    }
                } elseif (! empty(Parser::$STATEMENT_PARSERS[$token->value])) {
                    // We have reached the end of ALTER operation and suddenly found
                    // a start to new statement, but have not find a delimiter between them

                    if (! ($token->value === 'SET' && $list->tokens[$list->idx - 1]->value === 'CHARACTER')) {
                        $parser->error(
                            'A new statement was found, but no delimiter between it and the previous one.',
                            $token
                        );
                        break;
                    }
                } elseif ((array_key_exists($token->value, self::$DB_OPTIONS)
                    || array_key_exists($token->value, self::$TABLE_OPTIONS))
                    && ! self::checkIfColumnDefinitionKeyword($token->value)
                ) {
                    // This alter operation has finished, which means a comma was missing before start of new alter operation
                    $parser->error(
                        'Missing comma before start of a new alter operation.',
                        $token
                    );
                    break;
                }
                $ret->unknown[] = $token;
            }
        }

        if ($ret->options->isEmpty()) {
            $parser->error(
                'Unrecognized alter operation.',
                $list->tokens[$list->idx]
            );
        }

        --$list->idx;

        return $ret;
    }
851156
def append_position_to_token_list(token_list):
    """"""
    return [PositionToken(value.content, value.gd, index, index+1) for (index, value) in enumerate(token_list)]
499255
private function stripWhitespace(&$tokens, $position)
    {
        switch ($tokens[$position][0]) {
            case self::TOKEN_PLACEHOLDER:
            case self::TOKEN_SECTION:
            case self::TOKEN_SECTION_INVERT:
                // Analyze first token of section, and strip leading newlines
                $sectionTokens = $tokens[$position][1]['content'];
                if (0 === count($sectionTokens)) {
                    break;
                }
                $token = $sectionTokens[0];
                if ($token[0] === self::TOKEN_CONTENT) {
                    $content = preg_replace('/^\s*?(\r\n?|\n)/s', '', $token[1]);
                    $token = [
                        self::TOKEN_CONTENT,
                        $content,
                        'original_content' => $token[1],
                    ];
                    $sectionTokens[0] = $token;
                    $tokens[$position][1]['content'] = $sectionTokens;
                    break;
                }
                break;
            default:
                break;
        }

        // Analyze preceding token; if content token, and ending with a newline
        // and optionally whitespace, trim the whitespace.
        if (($position - 1) > -1) {
            $previous = $tokens[$position - 1];
            $type = $previous[0];
            if ($type === self::TOKEN_CONTENT) {
                $content = preg_replace('/(\r\n?|\n)\s+$/s', '$1', $previous[1]);
                $previous = [
                    self::TOKEN_CONTENT,
                    $content,
                    'original_content' => $previous[1],
                ];
                $tokens[$position - 1] = $previous;
            }
        }

        // Analyze next token. If it is a content token, and begins with optional
        // whitespace, followed by a newline, trim this whitespace.
        if (isset($tokens[$position + 1])) {
            $next = $tokens[$position + 1];
            $type = $next[0];
            if ($type === self::TOKEN_CONTENT) {
                $content = preg_replace('/^\s*?(\r\n?|\n)/s', '', $next[1]);
                $next = [
                    self::TOKEN_CONTENT,
                    $content,
                    'original_content' => $next[1],
                ];
                $tokens[$position + 1] = $next;
            }
        }
    }
1077362
public function orderAt($uuid, $position, $userId, $webspaceKey, $locale)
    {
        $document = $this->documentManager->find($uuid, $locale);

        $parentDocument = $this->inspector->getParent($document);
        $siblingDocuments = $this->inspector->getChildren($parentDocument);

        $siblings = array_values($siblingDocuments->toArray()); // get indexed array
        $countSiblings = count($siblings);
        $currentPosition = array_search($document, $siblings) + 1;

        if ($countSiblings < $position || $position <= 0) {
            throw new InvalidOrderPositionException(
                sprintf(
                    'Cannot order node "%s" at out-of-range position "%s", must be >= 0 && < %d"',
                    $this->inspector->getPath($document),
                    $position,
                    $countSiblings
                )
            );
        }

        if ($position === $countSiblings) {
            // move to the end
            $this->documentManager->reorder($document, null);
        } else {
            if ($currentPosition < $position) {
                $targetSibling = $siblings[$position];
            } elseif ($currentPosition > $position) {
                $targetSibling = $siblings[$position - 1];
            }

            $this->documentManager->reorder($document, $targetSibling->getUuid());
        }

        $this->documentManager->flush();

        return $this->documentToStructure($document);
    }
117867
public void addContent(DroolsToken token) {
        if (startOffset == -1) {
            startOffset = token.getStartIndex();
        }
        endOffset = token.getStopIndex();
        this.content.add(token);
    }
789924
protected function parse_tokens_and_matches($tokens, $values, $content) {

        $pairs = array_combine($tokens, $values);
        ksort($pairs); // First literals, then mappings
        foreach ($pairs as $token => $value) {
            // If one token has already been processed, continue
            if (array_key_exists($token, $this->allpairs)) {
                continue;
            }

            // If the pair is one literal token, just keep it unmodified
            if (substr($token, 0, 1) == '[') {
                $this->allpairs[$token] = $value;

            // If the pair is one mapping token, let's process it
            } else if (substr($token, 0, 1) == '{') {
                $ctoken = $token;

                // First, resolve mappings to literals if necessary
                if (substr($token, 1, 1) == '[') {
                    $literaltoken = trim($token, '{}');
                    if (array_key_exists($literaltoken, $this->allpairs)) {
                        $ctoken = '{' . $this->allpairs[$literaltoken] . '}';
                    }
                }

                // Look for mapping in fixedvalues before going to DB
                $plaintoken = trim($ctoken, '{}');
                if (array_key_exists($plaintoken, $this->fixedvalues)) {
                    $this->allpairs[$token] = $this->fixedvalues[$plaintoken];

                 // Last chance, fetch value from backup_ids_temp, via mapping
                } else {
                    if ($mapping = restore_dbops::get_backup_ids_record($this->restoreid, $plaintoken, $value)) {
                        $this->allpairs[$token] = $mapping->newitemid;
                    }
                }
            }
        }

        // Apply all the conversions array (allpairs) to content
        krsort($this->allpairs); // First mappings, then literals
        $content = str_replace(array_keys($this->allpairs), $this->allpairs, $content);

        return $content;
    }
1115670
@Override
    public void handleText(
            final char[] buffer, final int offset, final int len,
            final int line, final int col)
            throws ParseException {

        this.someSelectorsMatch = false;
        for (int i = 0; i < this.selectorsLen; i++) {

            this.selectorMatches[i] =
                    this.selectorFilters[i].matchText(false, this.markupLevel, this.markupBlocks[this.markupLevel]);
            if (this.selectorMatches[i]) {
                this.someSelectorsMatch = true;
            }

        }

        if (this.someSelectorsMatch) {
            markCurrentSelection();
            this.selectedHandler.handleText(buffer, offset, len, line, col);
            unmarkCurrentSelection();
            return;
        }

        unmarkCurrentSelection();
        this.nonSelectedHandler.handleText(buffer, offset, len, line, col);

    }
619114
static function ltrim_position($content, $position, $br = true)
    {
        $blanks = array("\t", "\0", "\x0B", " ");

        if ($br) {
            $blanks[] = "\r";
            $blanks[] = "\n";
        }

        while (isset($content[$position]) && isset($content[$position + 1])
            && in_array($content[$position], $blanks, true)
        ) {
            $position++;
        }

        return $position;
    }