Splits the token list into variant & parameter arrays.

@throws InvalidVariantException

@param array<string> $tokenBundle
@param string $fromField

@return string[][]
357780
private function splitVariantAndParamTokens(array $tokenBundle, $fromField)
    {
        if (PreferenceInterface::MIME === $fromField) {
            $this->validateBundleMimeVariant($tokenBundle);
            $variantTokenList = array_slice($tokenBundle, 0, 3);
            $paramTokenList = array_slice($tokenBundle, 3);
        } else {
            $variantTokenList = array_slice($tokenBundle, 0, 1);
            $paramTokenList = array_slice($tokenBundle, 1);
        }

        return array($variantTokenList, $paramTokenList);
    }
995022
public function parse(Token $token)
    {
        $lineno = $token->getLine();
        $stream = $this->parser->getStream();

        $count = null;
        $vars = new ArrayExpression([], $lineno);
        $domain = null;
        $locale = null;
        if (!$stream->test(Token::BLOCK_END_TYPE)) {
            if ($stream->test('count')) {
                // {% trans count 5 %}
                $stream->next();
                $count = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('with')) {
                // {% trans with vars %}
                $stream->next();
                $vars = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('from')) {
                // {% trans from "messages" %}
                $stream->next();
                $domain = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('into')) {
                // {% trans into "fr" %}
                $stream->next();
                $locale = $this->parser->getExpressionParser()->parseExpression();
            } elseif (!$stream->test(Token::BLOCK_END_TYPE)) {
                throw new SyntaxError('Unexpected token. Twig was looking for the "with", "from", or "into" keyword.', $stream->getCurrent()->getLine(), $stream->getSourceContext());
            }
        }

        // {% trans %}message{% endtrans %}
        $stream->expect(Token::BLOCK_END_TYPE);
        $body = $this->parser->subparse([$this, 'decideTransFork'], true);

        if (!$body instanceof TextNode && !$body instanceof AbstractExpression) {
            throw new SyntaxError('A message inside a trans tag must be a simple text.', $body->getTemplateLine(), $stream->getSourceContext());
        }

        $stream->expect(Token::BLOCK_END_TYPE);

        return new TransNode($body, $domain, $count, $vars, $locale, $lineno, $this->getTag());
    }
435170
private static Map<String, String> splitCookieToken(String tokenStr) {
    Map<String, String> map = new HashMap<String, String>();
    StringTokenizer st = new StringTokenizer(tokenStr, COOKIE_ATTR_SEPARATOR);

    while (st.hasMoreTokens()) {
      String part = st.nextToken();
      int separator = part.indexOf(COOKIE_KEY_VALUE_SEPARATOR);
      if (separator == -1) {
        LOG.error("Invalid token string " + tokenStr);
        return null;
      }
      String key = part.substring(0, separator);
      String value = part.substring(separator + 1);
      map.put(key, value);
    }
    return map;
  }
1169048
public function tokenize($input) : iterable
    {
        if (!is_string($input)) {
            throw new core\exceptions\InvalidArgumentType($input, ['string']);
        }

        // Change line breaks, tabs etc. into whitespaces.
        $length = strlen($input);
        $cursor = 0;
        $i      = 0;
        $tokens = new formats\tokens\Argv;

        while ($cursor < $length) {
            if (preg_match('/\s+/A', $input, $match, null, $cursor)) {
            } elseif (preg_match('/([^="\'\s]+?)(=?)('.static::PATTERN_QUOTED.'+)/A', $input, $match, null, $cursor)) {
                $tokens->set($i++, $match[1].$match[2].stripcslashes(str_replace(['"\'', '\'"', '\'\'', '""'], '', substr($match[3], 1, -1))));
            } elseif (preg_match('/'.static::PATTERN_QUOTED.'/A', $input, $match, null, $cursor)) {
                $tokens->set($i++, stripcslashes(substr($match[0], 1, -1)));
            } elseif (preg_match('/'.static::PATTERN_BARE.'/A', $input, $match, null, $cursor)) {
                $tokens->set($i++, stripcslashes($match[1]));
            } else {
                throw new \RuntimeException('Failed to parse string near "... '.substr($input, $cursor, 10).' ..."');
            }

            $cursor += strlen($match[0]);
        }

        return $tokens;
    }
410414
private function getMatchingIndexes($fromField, array $matchingList, PreferenceInterface $clientPref)
    {
        $matchingIndexList = array();

        foreach ($matchingList as $key => $matching) {
            if ($this->partialLangMatches($fromField, $matching, $clientPref)) {
                $matchingIndexList[] = $key;
            }
        }

        return $matchingIndexList;
    }
182348
private function genericAll($clientField, $serverField, $fromField)
    {
        $clientPrefList = $this->parsePreferences(false, $clientField, $fromField);
        $serverPrefList = $this->parsePreferences(true, $serverField, $fromField);

        return $this->negotiator->negotiateAll($clientPrefList, $serverPrefList, $fromField);
    }
1273537
protected function splitRelationField($field)
    {
        $parts = explode(':', $field);
        $partsCount = count($parts);

        return [implode('.', array_slice($parts, 0, $partsCount - 1)), $parts[$partsCount - 1]];
    }
1095010
public function getTranslatedFields(?string $from, ?string $to, string $remoteServiceKey, Content $content): array
    {
        $posixFrom = null;
        if (null !== $from) {
            $this->guard->enforceSourceLanguageVersionExist($content, $from);
            $posixFrom = $this->localeConverter->convertToPOSIX($from);
        }
        $this->guard->enforceTargetLanguageExist($to);

        $sourceContent     = $this->guard->fetchContent($content, $from);
        $payload           = $this->encoder->encode($sourceContent);
        $posixTo           = $this->localeConverter->convertToPOSIX($to);
        $remoteService     = $this->clientProvider->get($remoteServiceKey);
        $translatedPayload = $remoteService->translate($payload, $posixFrom, $posixTo);

        return $this->encoder->decode($translatedPayload);
    }
716620
static function parseIdentifierList(array $tokens, &$i) {
    $list = array();
    while (TRUE) {
      $id = ParserUtil::nextSubstantialIncl($tokens, $i);
      if (T_STRING !== $id) {
        return FALSE;
      }
      $list[] = $tokens[$i][1];
      $id = ParserUtil::nextSubstantialExcl($tokens, $i);
      if (',' !== $id) {
        break;
      }
      ++$i;
    }
    return $list;
  }
328037
public function tokenize($httpField, $fromField)
    {
        $quoteSeparators = array('"', "'");
        $tokenList = array();
        $stringAccumulator = '';
        $lastQuote = '';

        // Iterate through field, character-by-character
        for ($i = 0; $i < strlen($httpField); $i++) {
            $chr = substr($httpField, $i, 1);

            switch (true) {

                // We are at the end of a quoted string
                case $chr === $lastQuote:
                    $tokenList[] = $stringAccumulator;
                    $stringAccumulator = '';
                    $lastQuote = '';
                    break;

                // We have found the beginning of a quoted string
                case in_array($chr, $quoteSeparators):
                    $lastQuote = $chr;
                    break;

                // We are already within a quoted string, but not yet at the end
                case strlen($lastQuote):
                    $stringAccumulator .= $chr;
                    break;

                // Separators found, add previously accumulated string & separator to token list
                case Tokens::isSeparator($chr, Preference::MIME === $fromField):
                    if (strlen($stringAccumulator)) {
                        $tokenList[] = $stringAccumulator;
                        $stringAccumulator = '';
                    }

                    $tokenList[] = $chr;
                    break;

                // Simply accumulate characters
                default:
                    $stringAccumulator .= $chr;
                    break;
            }
        }

        // Handle final component
        if (strlen($stringAccumulator)) {
            $tokenList[] = $stringAccumulator;
        }

        // Remove any padding whitespace from token list
        $tokenList = array_map('trim', $tokenList);

        return $tokenList;
    }
786652
private function processTokens()
    {
        foreach ($this->getTokensStream() as list($string, $name, $pattern, $default)) {
            if (!\preg_match('/^[a-zA-Z0-9_]+$/', $name)) {
                throw new InvalidArgumentException("Invalid param name “{$name}” (source: {$this->pattern})");
            }

            Patterns::validatePatternName($name);

            $patternName = null;
            if ($patternObj = $this->patterns[$pattern] ?? null) {
                $patternName = $pattern;
                /** @var PatternInterface $patternObj */
                $pattern = $patternObj->getRegex();
            }

            if (!(new RegexTester())->isSubregex($pattern)) {
                throw new InvalidArgumentException("Invalid regex: {$pattern} (source: {$this->pattern})");
            }

            yield [
                $string,
                $name,
                $pattern,
                $default,
                $patternName,
            ];
        }
    }
1164278
private function genericBest($clientField, $serverField, $fromField)
    {
        $clientPrefList = $this->parsePreferences(false, $clientField, $fromField);
        $serverPrefList = $this->parsePreferences(true, $serverField, $fromField);

        return $this->negotiator->negotiateBest($clientPrefList, $serverPrefList, $fromField)->getVariant();
    }
838342
protected function parseListContent(TokenList $tokenList) {
        $lastToken = $tokenList->last();
        if ($lastToken->getName() === $this->name) {
            $tokenList->removeTokenAt(-1); // Remove last LIST_BLOCK (content)

            $content = $lastToken->get('content', '');

            $containsParagraphs = $tokenList->last()->get('forceParagraph', false);

            if (preg_match('/(?:\n(?:[ \t]*(?=\n))?){2,}/i', $content) === 1) {
                $containsParagraphs = true;

                // Remove starting spaces or tab
                $content = explode("\n", $content);
                $pad = null;
                foreach ($content as $index => &$item) {
                    // Find out lin starting space length up to 4 characters
                    if ($pad === null && $index > 0 && preg_match('/^[ \t]*$/i', $item) !== 1) {
                        if (preg_match('/^[ ]{0,4}/', $item, $matches) === 1) {
                            $pad = strlen($matches[0]);
                        } else {
                            $pad = 4;
                        }
                    }

                    // Remove line starting space or tab characters
                    if ($pad !== null) {
                        $item = preg_replace('/^(?: {' . $pad . '}|\t)/', '', $item);
                    }
                }

                $content = implode("\n", $content);
            }

            $content = "\n" . $content . "\n";

            $contentTokenList = $this->parseContent($content, null, true);

            // Remove beginning and ending new line feed
            if ($contentTokenList->first()->getName() === EolRule::NAME
                && $contentTokenList->last()->getName() === EolRule::NAME
            ) {
                $contentTokenList->removeTokenAt(0);  // Remove first EOL
                $contentTokenList->removeTokenAt(-1); // Remove last EOL
            }

            // Remove paragraphs
            if (!$containsParagraphs
                && $contentTokenList->first()->getName() === PRule::NAME . AbstractBlockRule::OPEN
                && $contentTokenList->last()->getName() === PRule::NAME . AbstractBlockRule::CLOSE
            ) {
                $contentTokenList->removeTokenAt(0);  // Remove first P
                $contentTokenList->removeTokenAt(-1); // Remove last P
            }

            // Add new line indicator for paragraphs
            for ($i = 0; $i < count($contentTokenList); $i++) {
                $token = $contentTokenList->getTokenAt($i);
                if ($token->getName() === EolRule::NAME) {
                    $contentTokenList->removeTokenAt($i);
                    $contentTokenList->insertTokenAt($this->name . '_ITEM_EOL', $token->getPosition(), $i);
                }
            }

            // Change LIST_BLOCK to BASE token
            $startRule = $this->set->getStartRule();

            for ($i = 0; $i < count($contentTokenList); $i++) {
                if ($contentTokenList->getTokenAt($i)->getName() === $this->name) {
                    $token = $contentTokenList->getTokenAt($i);
                    $contentTokenList->removeTokenAt($i);
                    $contentTokenList->insertTokenAt($startRule->getName(), $token->getPosition(), $i)
                        ->set('content', $token->get('content', ''));
                }
            }

            $tokenList->merge($contentTokenList);
        }
    }
640686
public static function fromToken(TokenInterface $token)
    {
        $user = $token->getUser();

        if ($user instanceof RoleableInterface) {
            $sids = [];
            $roles = $user->getRoles();

            foreach ($roles as $role) {
                $sids[] = self::fromAccount($role);
            }

            return $sids;
        }

        throw new InvalidArgumentException('The user class must implement "Fxp\Component\Security\Model\Traits\RoleableInterface"');
    }
582978
private function matchSingleClientPreference($fromField, PreferenceInterface $clientPreference, array $matchingList)
    {
        foreach ($this->matcherList as $matcher) {
            if ($matcher->hasMatch($fromField, $matchingList, $clientPreference)) {
                $matchingList = $matcher->match($fromField, $matchingList, $clientPreference);

                break;
            }
        }

        return $matchingList;
    }
232937
private function getVariables(GuardEvent $event): array
    {
        $token = $this->tokenStorage->getToken();

        if (null === $token) {
            throw new InvalidTokenConfigurationException(sprintf('There are no tokens available for workflow %s.', $event->getWorkflowName()));
        }

        if (method_exists($token, 'getRoleNames')) {
            $roleNames = $token->getRoleNames();
            $roles = array_map(function (string $role) { return new Role($role, false); }, $roleNames);
        } else {
            @trigger_error(sprintf('Not implementing the getRoleNames() method in %s which implements %s is deprecated since Symfony 4.3.', \get_class($token), TokenInterface::class), E_USER_DEPRECATED);

            $roles = $token->getRoles(false);
            $roleNames = array_map(function (Role $role) { return $role->getRole(); }, $roles);
        }

        if (null !== $this->roleHierarchy && method_exists($this->roleHierarchy, 'getReachableRoleNames')) {
            $roleNames = $this->roleHierarchy->getReachableRoleNames($roleNames);
            $roles = array_map(function (string $role) { return new Role($role, false); }, $roleNames);
        } elseif (null !== $this->roleHierarchy) {
            $roles = $this->roleHierarchy->getReachableRoles($roles);
            $roleNames = array_map(function (Role $role) { return $role->getRole(); }, $roles);
        }

        $variables = [
            'token' => $token,
            'user' => $token->getUser(),
            'subject' => $event->getSubject(),
            'roles' => $roles,
            'role_names' => $roleNames,
            // needed for the is_granted expression function
            'auth_checker' => $this->authorizationChecker,
            // needed for the is_* expression function
            'trust_resolver' => $this->trustResolver,
            // needed for the is_valid expression function
            'validator' => $this->validator,
        ];

        return $variables;
    }
433118
public function process($tokenList) {
        $unparsed = $tokenList['INTO'];
        foreach ($unparsed as $k => $token) {
            if ($this->isWhitespaceToken($token) || $this->isCommaToken($token)) {
                unset($unparsed[$k]);
            }
        }
        $tokenList['INTO'] = array_values($unparsed);
        return $tokenList;
    }