@param Tokenizer $tokenizer
@param string    $string

@return array
771885
public static function isUserDefinedVariable(Tokenizer $tokenizer, $string)
    {
        if (!$tokenizer->getNextToken() && self::isUserDefinedVariableString($string)) {
            $tokenizer->setNextToken(self::getUserDefinedVariableString($string));
        }
    }
1025
protected function getTokenStrings(int $row, array $tokens): array
    {
        $pieces = [];
        foreach ($tokens as $key => $token) {
            if ($token['line'] > $row) {
                break;
            }
            if ($token['line'] < $row) {
                continue;
            }
            if ($this->verbose) {
                $type = $token['type'];
                $content = $token['content'];
                $content = '`' . str_replace(["\r\n", "\n", "\r", "\t"], ['\r\n', '\n', '\r', '\t'], $content) . '`';

                unset($token['type']);
                unset($token['content']);
                $token['content'] = $content;

                $tokenList = [];
                foreach ($token as $k => $v) {
                    if (is_array($v)) {
                        if (empty($v)) {
                            continue;
                        }
                        $v = json_encode($v);
                    }
                    $tokenList[] = $k . '=' . $v;
                }
                $pieces[] = $type . ' (' . $key . ') ' . implode(', ', $tokenList);
            } else {
                $pieces[] = $token['type'];
            }
        }
        if ($this->verbose) {
            return $pieces;
        }
        return [implode(' ', $pieces)];
    }
440084
private function ParseFilters($tokenString, $nextStop)
    {
        $filters = array();
        if ($nextStop === false)
        {
            return $filters;
        }
        $trimString = trim(substr($tokenString, $nextStop));
        if ($trimString == '')
        {
            return $filters;
        }
        if (!Str::StartsWith(self::FilterSeparator, $trimString))
        {
            return false;
        }
        $filtersStart = strpos($tokenString, self::FilterSeparator, $nextStop) + strlen(self::FilterSeparator);
        $filterNames = explode(self::FilterSeparator, substr($tokenString, $filtersStart));
        foreach ($filterNames as $filterName)
        {
            $filter = trim($filterName);
            if (!ctype_alnum($filter))
            {
                return false;
            }
            $filters[] = $filter;
        }
        return $filters;
    }
820036
def custom_tokenizer(self) -> spacyTokenizer:
        
        prefix_re = re.compile(r'''^[\[()\-.,@#$%^&*?|<~+_:;>!"']''')
        infix_re = re.compile(r'''[\[()\-,@#$%^&*?|<~+_:;>!"']|(?<![0-9])\.|\.(?![0-9])|\n+ ''')
        return spacyTokenizer(self.nlp.vocab, rules=None, prefix_search=prefix_re.search, suffix_search=None,
                              infix_finditer=infix_re.finditer, token_match=None)
680317
protected static function tokenize($string)
    {
        self::init();

        $tokens = array();

        // Used for debugging if there is an error while tokenizing the string
        $original_length = strlen($string);

        // Used to make sure the string keeps shrinking on each iteration
        $old_string_len = strlen($string) + 1;

        $token = null;

        $current_length = strlen($string);

        // Keep processing the string until it is empty
        while ($current_length) {
            // If the string stopped shrinking, there was a problem
            if ($old_string_len <= $current_length) {
                $tokens[] = array(
                    self::TOKEN_VALUE=>$string,
                    self::TOKEN_TYPE=>self::TOKEN_TYPE_ERROR
                );

                return $tokens;
            }
            $old_string_len =  $current_length;

            // Determine if we can use caching
            if ($current_length >= self::$max_cachekey_size) {
                $cacheKey = substr($string,0,self::$max_cachekey_size);
            } else {
                $cacheKey = false;
            }

            // See if the token is already cached
            if ($cacheKey && isset(self::$token_cache[$cacheKey])) {
                // Retrieve from cache
                $token = self::$token_cache[$cacheKey];
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_hits++;
            } else {
                // Get the next token and the token type
                $token = self::getNextToken($string, $token);
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_misses++;

                // If the token is shorter than the max length, store it in cache
                if ($cacheKey && $token_length < self::$max_cachekey_size) {
                    self::$token_cache[$cacheKey] = $token;
                }
            }

            $tokens[] = $token;

            // Advance the string
            $string = substr($string, $token_length);

            $current_length -= $token_length;
        }

        return $tokens;
    }
255124
private function getTokens()
    {
        $tokenVector = array(
            'query' => array(),
        );
        $tokenVector['query'] = $this->tokenizer->getTokenVector();
        $tokens = $this->applyFilters($tokenVector);
        if (!empty($tokens)) {
            $tokens = array_keys($tokens['query']);
        }

        return $tokens;
    }
245145
protected static function getFunctionString($string, array &$matches)
    {
        return [
            Tokenizer::TOKEN_TYPE => Tokenizer::TOKEN_TYPE_RESERVED,
            Tokenizer::TOKEN_VALUE => \substr($string, 0, \strlen($matches[1]) - 1),
        ];
    }
1355173
public static function TokenizeWithFlags( $arg1, $arg2, $flagString )
	{
		if ( $arg1 instanceof Undefined || empty( $arg1 ) )
		{
			return EmptyIterator::$Shared;
		}

		$input = $arg1 . "";

		if ( $arg2 instanceof Undefined )
		{
			throw XPath2Exception::withErrorCodeAndParams( "XPTY0004", Resources::XPTY0004, array( "empty-sequence()", "xs:string in fn:tokenize" ) );
		}

	    $flagString = ExtFuncs::PrepareRegexFlags( $flagString );
	    $pattern = ExtFuncs::PrepareRegexPattern( $arg2 . "", $flagString );

		if (  preg_match( "/$pattern/", "" ) )
		{
			throw XPath2Exception::withErrorCodeAndParam( "FORX0003", Resources::FORX0003, $pattern );
		}

		return new NodeIterator( function() use( $input, $pattern, $flagString )
		{
			return ExtFuncs::StringEnumerator( $input, $pattern, $flagString );
		} );
	}
1176130
public function parseString($string)
    {
        $tokens = new Stack($this->tokenizer->tokenize($string));

        if ($tokens[0]->type === Tokenizer::T_EOF) {
            return new CTX\AccountInfoList();
        }

        $ctx = new CTX();
        $this->reduceStruct($tokens, $ctx);

        return $ctx->accountInfoList[0];
    }
8305
public function getTokens($string)
    {
        $tokens      = [];
        $scanResults = $this->engine->getTokenizer()->scan($string);

        foreach ($scanResults as $scanResult) {
            if (
                isset($scanResult['type'])
                && isset($scanResult['name'])
                && \Mustache_Tokenizer::T_ESCAPED === $scanResult['type']
            ) {
                $tokens[$scanResult['name']] = true;
            }
        }

        return array_keys($tokens);
    }
798276
private function doFilter(TokenFilterInterface $filter, array $tokens): array
    {
        $result = [];
        foreach ($tokens as $token) {
            $result = array_merge($result, $filter->filter($token));
        }

        return $result;
    }
247551
def listified_tokenizer(source):
    """"""
    io_obj = io.StringIO(source)
    return [list(a) for a in tokenize.generate_tokens(io_obj.readline)]
1291231
public function tokenize($string)
    {
        $matches = array();
        $count = preg_match_all($this->pattern, $string, $matches, $this->flags, $this->offset);
        if($count === false) { 
            return array();
        }
        return $matches[0];
        
    }
898852
private function extractUseStatement(\ArrayIterator $tokens)
    {
        $result = [''];
        while ($tokens->valid()
            && (self::T_LITERAL_USE_SEPARATOR !== $tokens->current()[0])
            && (self::T_LITERAL_END_OF_USE !== $tokens->current()[0])
        ) {
            if (T_AS === $tokens->current()[0]) {
                $result[] = '';
            }
            if (T_STRING === $tokens->current()[0] || T_NS_SEPARATOR === $tokens->current()[0]) {
                $result[\count($result) - 1] .= $tokens->current()[1];
            }
            $tokens->next();
        }

        if (1 == \count($result)) {
            $backslashPos = strrpos($result[0], '\\');

            if (false !== $backslashPos) {
                $result[] = substr($result[0], $backslashPos + 1);
            } else {
                $result[] = $result[0];
            }
        }

        return array_reverse($result);
    }
596172
public static function tokenize(string $plainData) : array
    {
        $tokens = [];

        if(! empty($plainData)) {
            $tokens = (new self($plainData))->tokenizeInternal()->tokens;
        }

        return $tokens;
    }
838342
protected function parseListContent(TokenList $tokenList) {
        $lastToken = $tokenList->last();
        if ($lastToken->getName() === $this->name) {
            $tokenList->removeTokenAt(-1); // Remove last LIST_BLOCK (content)

            $content = $lastToken->get('content', '');

            $containsParagraphs = $tokenList->last()->get('forceParagraph', false);

            if (preg_match('/(?:\n(?:[ \t]*(?=\n))?){2,}/i', $content) === 1) {
                $containsParagraphs = true;

                // Remove starting spaces or tab
                $content = explode("\n", $content);
                $pad = null;
                foreach ($content as $index => &$item) {
                    // Find out lin starting space length up to 4 characters
                    if ($pad === null && $index > 0 && preg_match('/^[ \t]*$/i', $item) !== 1) {
                        if (preg_match('/^[ ]{0,4}/', $item, $matches) === 1) {
                            $pad = strlen($matches[0]);
                        } else {
                            $pad = 4;
                        }
                    }

                    // Remove line starting space or tab characters
                    if ($pad !== null) {
                        $item = preg_replace('/^(?: {' . $pad . '}|\t)/', '', $item);
                    }
                }

                $content = implode("\n", $content);
            }

            $content = "\n" . $content . "\n";

            $contentTokenList = $this->parseContent($content, null, true);

            // Remove beginning and ending new line feed
            if ($contentTokenList->first()->getName() === EolRule::NAME
                && $contentTokenList->last()->getName() === EolRule::NAME
            ) {
                $contentTokenList->removeTokenAt(0);  // Remove first EOL
                $contentTokenList->removeTokenAt(-1); // Remove last EOL
            }

            // Remove paragraphs
            if (!$containsParagraphs
                && $contentTokenList->first()->getName() === PRule::NAME . AbstractBlockRule::OPEN
                && $contentTokenList->last()->getName() === PRule::NAME . AbstractBlockRule::CLOSE
            ) {
                $contentTokenList->removeTokenAt(0);  // Remove first P
                $contentTokenList->removeTokenAt(-1); // Remove last P
            }

            // Add new line indicator for paragraphs
            for ($i = 0; $i < count($contentTokenList); $i++) {
                $token = $contentTokenList->getTokenAt($i);
                if ($token->getName() === EolRule::NAME) {
                    $contentTokenList->removeTokenAt($i);
                    $contentTokenList->insertTokenAt($this->name . '_ITEM_EOL', $token->getPosition(), $i);
                }
            }

            // Change LIST_BLOCK to BASE token
            $startRule = $this->set->getStartRule();

            for ($i = 0; $i < count($contentTokenList); $i++) {
                if ($contentTokenList->getTokenAt($i)->getName() === $this->name) {
                    $token = $contentTokenList->getTokenAt($i);
                    $contentTokenList->removeTokenAt($i);
                    $contentTokenList->insertTokenAt($startRule->getName(), $token->getPosition(), $i)
                        ->set('content', $token->get('content', ''));
                }
            }

            $tokenList->merge($contentTokenList);
        }
    }
544098
private function filterTokensByClass(array $tokens, $tokenClass)
    {
        return array_filter(
            $tokens,
            function (Token $token) use ($tokenClass) {
                return get_class($token) === $tokenClass;
            }
        );
    }