called by the inline parser, when it found a separator.
@param string $token
169849
public function addSeparator($token)
    {
        $cellContent = $this->wikiContentArr[$this->separatorCount];
        if ($cellContent === '') {
            if ($this->previousGenerator) {
                $this->previousGenerator->setColSpan($this->previousGenerator->getColSpan() + 1);
            }
        } else {
            if (preg_match('/^\s\s/', $cellContent) &&
                preg_match('/\s\s$/', $cellContent)) {
                if (trim($cellContent) != '') {
                    $this->generator->setAlign('center');
                }
            } elseif (preg_match('/^\s\s/', $cellContent)) {
                $this->generator->setAlign('right');
            } elseif (preg_match('/\s\s$/', $cellContent) && preg_match('/^\S/', $cellContent)) {
                $this->generator->setAlign('left');
            }
            $this->row->addGenerator($this->generator);
            $this->wikiContent .= $cellContent;
            $this->previousGenerator = $this->generator;
            $this->generator = $this->documentGenerator->getInlineGenerator($this->generatorName);
            if ($token == '^') {
                $this->generator->setIsHeader(true);
            }
            ++$this->separatorCount;
            $this->contents[$this->separatorCount] = '';
            $this->wikiContentArr[$this->separatorCount] = '';
        }
        $this->currentSeparator = $token;
        $this->wikiContent .= $token;
    }
1152430
private function _processParenthesisClose($token) {
		$this->statementStack->getCurrentStackItem()->openParentheses -= 1;

		// Decrease the number of opened brackets
		if ($this->_inFuncCall) {
			$this->_fcLeftParenthesis -= 1;
		} elseif ($this->_inControlStatement || $this->_inFunctionStatement) {
			$this->_csLeftParenthesis -= 1;
		}
		// If 0 we are not in the call anymore
		if ($this->_fcLeftParenthesis === 0) {
			$this->_inFuncCall = false;
			array_pop($this->_currentFuncCall);
		}
		// If 0 we are not in the statement anymore
		if ($this->_csLeftParenthesis === 0) {
			if ($this->_inControlStatement) {
				$this->_inControlStatement = false;
				$this->_justAfterControlStmt = true;
				$this->_checkNeedBraces();
			} elseif ($this->_inFunctionStatement && !$this->_inInterface) {
				$this->_inFunctionStatement = false;
				$this->_justAfterFuncStmt = true;
			}
		}

		// We the count arrive to 0 we probably have something to do
		if ($this->statementStack->getCurrentStackItem()->openParentheses === 0) {

			// We are in a array declaration, we unstack
			if ($this->statementStack->getCurrentStackItem()->type === StatementItem::TYPE_ARRAY && $this->statementStack->getCurrentStackItem()->name !== 'square_bracket_open') {
				$this->statementStack->pop();
			}
		}
	}
1148201
protected function parseToken($token)
    {
        list($id, $content) = $token;
        if ($id == T_INLINE_HTML) {
            $content = $this->renderShortcodes($content);
        }

        return $content;
    }
368606
private function _processSemiColon($token) {
		// ";" should never be preceded by a whitespace
		$this->_checkNoWhiteSpaceBefore($token->text);

		// ";" should never be preceded by ;
		$this->_checkEmptyStatement();

		// If we are in a statement not surrounded by curly braces, we unstack the last line.
		if ($this->statementStack->getCurrentStackItem()->noCurly === true) {
			$this->statementStack->pop();
		}
	}
1320319
private function resolveToken( $token )
    {
        $replaceString = "";
        $tokenParts = $this->tokenParts( $token );

        foreach ( $tokenParts as $tokenPart )
        {
            if ( $this->isTokenGroup( $tokenPart ) )
            {
                $groupTokenArray = $this->extractTokens( $this->groupLookupTable[$tokenPart] );
                $replaceString = $this->groupLookupTable[$tokenPart];

                foreach ( $groupTokenArray as $groupToken )
                {
                    $replaceString = str_replace( $groupToken, $this->resolveToken( $groupToken ), $replaceString );
                }
                // We want to stop after the first matching token part / identifier is found
                // <id1|id2> if id1 has a value, id2 will not be used.
                // In this case id1 or id1 is a token group.
                break;
            }
            else
            {
                if ( isset( $this->attributeArray[$tokenPart] ) && $this->attributeArray[$tokenPart] !== '' && $this->attributeArray[$tokenPart] !== false )
                {
                    $replaceString = $this->attributeArray[$tokenPart];
                    // We want to stop after the first matching token part / identifier is found
                    // <id1|id2> if id1 has a value, id2 will not be used.
                    break;
                }
            }
        }
        return $replaceString;
    }
360424
public function setSeparator($separator)
    {
        if (!in_array(
            $separator,
            [
                ' ',
                '-'
            ],
            true
        )
        ) {
            $separator = ' ';
        }

        $this->separator = $separator;

        return $this;
    }
233151
public function setAccessToken($token)
    {
        if (is_string($token)) {
            $token = new Token($token);
        }
        Requestor::setToken($token);
    }
1341516
public static function isClause($token)
    {
        if (($token->type === Token::TYPE_KEYWORD && isset(Parser::$STATEMENT_PARSERS[$token->keyword]))
            || ($token->type === Token::TYPE_NONE && strtoupper($token->token) === 'DELIMITER')
        ) {
            return 2;
        } elseif ($token->type === Token::TYPE_KEYWORD && isset(Parser::$KEYWORD_PARSERS[$token->keyword])
        ) {
            return 1;
        }

        return false;
    }
552204
public function setToken(string $token)
    {
        if (empty($selector)) {
            Throw new TokenException('Empty tokens are not allowed.');
        }

        $this->token = $token;
    }
776750
protected function _setTokenEnd($delimiter)
    {
        if (!is_null($delimiter)) {
            $delimiter = $this->_normalizeTokenDelimiter($delimiter);
        }

        $this->tokenEnd = $delimiter;
    }
1004684
public function setToken($token)
    {
        $this->_tokenString = (string) $token;
        $this->_token       = $token;
        return $this;
    }
467291
protected static function highlightToken($token)
    {
        $type = $token[self::TOKEN_TYPE];

        if (self::is_cli()) {
            $token = $token[self::TOKEN_VALUE];
        } else {
            if (defined('ENT_IGNORE')) {
              $token = htmlentities($token[self::TOKEN_VALUE],ENT_COMPAT | ENT_IGNORE ,'UTF-8');
            } else {
              $token = htmlentities($token[self::TOKEN_VALUE],ENT_COMPAT,'UTF-8');
            }
        }

        if ($type===self::TOKEN_TYPE_BOUNDARY) {
            return self::highlightBoundary($token);
        } elseif ($type===self::TOKEN_TYPE_WORD) {
            return self::highlightWord($token);
        } elseif ($type===self::TOKEN_TYPE_BACKTICK_QUOTE) {
            return self::highlightBacktickQuote($token);
        } elseif ($type===self::TOKEN_TYPE_QUOTE) {
            return self::highlightQuote($token);
        } elseif ($type===self::TOKEN_TYPE_RESERVED) {
            return self::highlightReservedWord($token);
        } elseif ($type===self::TOKEN_TYPE_RESERVED_TOPLEVEL) {
            return self::highlightReservedWord($token);
        } elseif ($type===self::TOKEN_TYPE_RESERVED_NEWLINE) {
            return self::highlightReservedWord($token);
        } elseif ($type===self::TOKEN_TYPE_NUMBER) {
            return self::highlightNumber($token);
        } elseif ($type===self::TOKEN_TYPE_VARIABLE) {
            return self::highlightVariable($token);
        } elseif ($type===self::TOKEN_TYPE_COMMENT || $type===self::TOKEN_TYPE_BLOCK_COMMENT) {
            return self::highlightComment($token);
        }

        return $token;
    }
357837
protected function removeParenthesisFromStart($token) {
        $parenthesisRemoved = 0;

        $trim = trim($token);
        if ($trim !== '' && $trim[0] === '(') { // remove only one parenthesis pair now!
            $parenthesisRemoved++;
            $trim[0] = ' ';
            $trim = trim($trim);
        }

        $parenthesis = $parenthesisRemoved;
        $i = 0;
        $string = 0;
        // Whether a string was opened or not, and with which character it was open (' or ")
        $stringOpened = '';
        while ($i < strlen($trim)) {

            if ($trim[$i] === "\\") {
                $i += 2; // an escape character, the next character is irrelevant
                continue;
            }

            if ($trim[$i] === "'") {
                if ($stringOpened === '') {
                    $stringOpened = "'";
                } elseif ($stringOpened === "'") {
                    $stringOpened = '';
                }
            }

            if ($trim[$i] === '"') {
                if ($stringOpened === '') {
                    $stringOpened = '"';
                } elseif ($stringOpened === '"') {
                    $stringOpened = '';
                }
            }

            if (($stringOpened === '') && ($trim[$i] === '(')) {
                $parenthesis++;
            }

            if (($stringOpened === '') && ($trim[$i] === ')')) {
                if ($parenthesis == $parenthesisRemoved) {
                    $trim[$i] = ' ';
                    $parenthesisRemoved--;
                }
                $parenthesis--;
            }
            $i++;
        }
        return trim($trim);
    }
1202631
private function match($token)
    {
        if (!$this->lexer->isNextToken($token)) {
            throw SyntaxException::expectedToken(
                $this->lexer->getLiteral($token),
                null,
                $this->lexer
            );
        }

        return $this->lexer->moveNext();
    }
1039683
private function skipToNextStringOrNamespaceSeparator(\ArrayIterator $tokens)
    {
        while ($tokens->valid() && (T_STRING !== $tokens->current()[0]) && (T_NS_SEPARATOR !== $tokens->current()[0])) {
            $tokens->next();
        }
    }
187212
public function invalidateToken(Token $token)
    {
        $this->tokenCache->remove($token->getHash());

        $this->logger->log(sprintf('Deleted token %s.', $token->getIdentifier()), LOG_INFO);
    }
280786
public function match($token)
    {
        $lookaheadType = $this->lexer->lookahead['type'];

        // Short-circuit on first condition, usually types match
        if ($lookaheadType === $token) {
            $this->lexer->moveNext();

            return;
        }

        // If parameter is not identifier (1-99) must be exact match
        if ($token < Lexer::T_IDENTIFIER) {
            $this->syntaxError($this->lexer->getLiteral($token));
        }

        // If parameter is keyword (200+) must be exact match
        if ($token > Lexer::T_IDENTIFIER) {
            $this->syntaxError($this->lexer->getLiteral($token));
        }

        // If parameter is T_IDENTIFIER, then matches T_IDENTIFIER (100) and keywords (200+)
        if ($token === Lexer::T_IDENTIFIER && $lookaheadType < Lexer::T_IDENTIFIER) {
            $this->syntaxError($this->lexer->getLiteral($token));
        }

        $this->lexer->moveNext();
    }