Parses an array of tokens. Returns a single node that is a container node.

@param Token[] $tokens

@return ContainerNode
503731
public function parse(array $tokens)
    {
        $parser = new Parser($this->symbolContainer);

        if ($this->customGrammarChecker !== null) {
            $parser->setCustomGrammarChecker($this->customGrammarChecker);
        }

        $rootNode = $parser->parse($tokens);

        return $rootNode;
    }
620155
public function parse(Token $token): Node
    {
        $stream = $this->parser->getStream();

        $stream->expect(Token::BLOCK_END_TYPE);

        $body = $this->parser->subparse(function (Token $token): bool {
            return $token->test('endoptional');
        }, true);

        $stream->expect(Token::BLOCK_END_TYPE);

        return new OptionalNode($body, $token->getLine(), $this->getTag());
    }
41935
static function parseUseTrait(array $tokens, &$i) {
    ++$i;
    $names = ParserUtil::parseIdentifierList($tokens, $i);
    if (FALSE === $names) {
      return FALSE;
    }
    $id = ParserUtil::nextSubstantialIncl($tokens, $i);
    if (';' === $id) {
      ++$i;
      return new AstUseTrait($names);
    }
    elseif ('{' === $id) {
      ParserUtil::skipCurly($tokens, $i);
      return new AstUseTrait($names);
    }
    else {
      return FALSE;
    }
  }
1237904
protected function __add( array $tokens )
	{
		$updated = false;

		foreach ( $tokens as $token ) {
			if ( ! $this->__contains( $token ) ) {
				$this->tokens[] = $token;
				$updated = true;
			}
		}

		if ( $updated ) {
			$this->__update();
		}

		return $this;
	}
116001
public static function transpileMatchRegex(array $tokens)
    {
        $regex = [];

        foreach ($tokens as $token) {
            $var = $token instanceof Variable ? $token : ($token instanceof Delimiter ? $token->next : null);

            if (null !== $var && $var instanceof Variable && null !== ($optgrp = self::makeOptGrp($var))) {
                $regex[] = $optgrp;
                break;
            }

            $regex[] = $token;
        }

        return implode('', $regex);
    }
608547
protected function filterTokens(array $tokens)
    {
        $filteredTokens = array_filter($tokens, function (Token $t) {
            return $t->getType() !== TokenType::Whitespace;
        });

        // Return the array values only, because array_filter preserves the keys
        return array_values($filteredTokens);
    }
1338220
protected static function parseSequence(TokenIterator $tokens, \ArrayIterator $options) {
        $result = [];
        $not = [null, '', ']', ')', '|'];
        while (!in_array($tokens->current(), $not, true)) {
            $atom = static::parseAtom($tokens, $options);
            if ($tokens->current() == '...') {
                $atom = [new OneOrMore($atom)];
                $tokens->move();
            }
            if ($atom) {
                $result = array_merge($result, $atom);
            }
        }
        return $result;
    }
1202932
private function sortTokens(array $tokens): array
    {
        $tagCounts = $this->docTagHelper->getTagCounts($tokens);

        usort($tokens, function (array $leftToken, array $rightToken) use ($tagCounts): int {
            return $this->compareTokensForSorting($leftToken, $rightToken, $tagCounts);
        });

        return $tokens;
    }
805906
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        return array_combine(self::columns($tokens), self::values($tokens));
    }
746980
public function process($tokens, $keys = array()) {

        $base_expr = "";
        $expr = array();
        $currCategory = "";

        if ($this->isStatement($keys)) {
            foreach ($tokens as $token) {

                $trim = trim($token);
                $base_expr .= $token;

                if ($trim === '') {
                    continue;
                }

                $upper = strtoupper($trim);

                switch ($upper) {

                case 'EXTENDED':
                case 'PARTITIONS':
                    return array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $token);
                    break;

                case 'FORMAT':
                    if ($currCategory === '') {
                        $currCategory = $upper;
                        $expr[] = array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $trim);
                    }
                    // else?
                    break;

                case '=':
                    if ($currCategory === 'FORMAT') {
                        $expr[] = array('expr_type' => ExpressionType::OPERATOR, 'base_expr' => $trim);
                    }
                    // else?
                    break;

                case 'TRADITIONAL':
                case 'JSON':
                    if ($currCategory === 'FORMAT') {
                        $expr[] = array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $trim);
                        return array('expr_type' => ExpressionType::EXPRESSION, 'base_expr' => trim($base_expr),
                                     'sub_tree' => $expr);
                    }
                    // else?
                    break;

                default:
                // ignore the other stuff
                    break;
                }
            }
            return empty($expr) ? null : $expr;
        }

        foreach ($tokens as $token) {

            $trim = trim($token);

            if ($trim === '') {
                continue;
            }

            switch ($currCategory) {

            case 'TABLENAME':
                $currCategory = 'WILD';
                $expr[] = array('expr_type' => ExpressionType::COLREF, 'base_expr' => $trim,
                                'no_quotes' => $this->revokeQuotation($trim));
                break;

            case '':
                $currCategory = 'TABLENAME';
                $expr[] = array('expr_type' => ExpressionType::TABLE, 'table' => $trim,
                                'no_quotes' => $this->revokeQuotation($trim), 'alias' => false, 'base_expr' => $trim);
                break;

            default:
                break;
            }
        }
        return empty($expr) ? null : $expr;
    }
728955
private function clearStandaloneLines(array &$nodes, array &$tokens)
    {
        if ($this->lineTokens > 1) {
            // this is the third or later node on this line, so it can't be standalone
            return;
        }

        $prev = null;
        if ($this->lineTokens === 1) {
            // this is the second node on this line, so it can't be standalone
            // unless the previous node is whitespace.
            if ($prev = end($nodes)) {
                if (!$this->tokenIsWhitespace($prev)) {
                    return;
                }
            }
        }

        if ($next = reset($tokens)) {
            // If we're on a new line, bail.
            if ($next[Mustache_Tokenizer::LINE] !== $this->lineNum) {
                return;
            }

            // If the next token isn't whitespace, bail.
            if (!$this->tokenIsWhitespace($next)) {
                return;
            }

            if (count($tokens) !== 1) {
                // Unless it's the last token in the template, the next token
                // must end in newline for this to be standalone.
                if (substr($next[Mustache_Tokenizer::VALUE], -1) !== "\n") {
                    return;
                }
            }

            // Discard the whitespace suffix
            array_shift($tokens);
        }

        if ($prev) {
            // Return the whitespace prefix, if any
            return array_pop($nodes);
        }
    }
801227
protected static function _tokenize(array $tokens) {
		$inString = false;
		$results = array();
		$cpt = 0;
		$previousTokenId = null;
		foreach ($tokens as $tokenId => $token) {
			if ($token['content'] === '"') {
				if (!$inString) {
					$token['id'] = T_START_DOUBLE_QUOTE;
					$token['name'] = 'T_START_DOUBLE_QUOTE';
				} else {
					$token['id'] = T_END_DOUBLE_QUOTE;
					$token['name'] = 'T_END_DOUBLE_QUOTE';
				}
				$inString = !$inString;
			}

			$isCurlyBrace = (
				$token['content'] === '$' &&
				isset($tokens[$tokenId + 1]) &&
				$tokens[$tokenId + 1]['content'] === '{'
			);
			if ($isCurlyBrace) {
				$token['id'] = T_DOLLAR_CURLY_BRACES;
				$token['name'] = 'T_DOLLAR_CURLY_BRACES';
			}

			$isArray = (
				!$token['id'] &&
				$token['content'] === '(' &&
				$previousTokenId !== null &&
				$tokens[$previousTokenId]['id'] === T_ARRAY
			);
			if ($isArray) {
				$token['id'] = T_ARRAY_OPEN;
				$token['name'] = 'T_ARRAY_OPEN';
			}

			$isShortArray = (
				!$token['id'] &&
				$token['content'] === '[' &&
				$previousTokenId !== null && (
					$tokens[$previousTokenId]['id'] !== T_VARIABLE &&
					$tokens[$previousTokenId]['id'] !== T_ENCAPSED_AND_WHITESPACE
				)
			);
			if ($isShortArray) {
				$token['id'] = T_SHORT_ARRAY_OPEN;
				$token['name'] = 'T_SHORT_ARRAY_OPEN';
			}

			if ($token['content'] === '(' && !$token['id']) {
				$token['id'] = T_START_BRACKET;
				$token['name'] = 'T_START_BRACKET';
			}

			if ($token['id'] !== T_WHITESPACE) {
				$previousTokenId = $tokenId;
			}

			if ($token['id'] === T_IF) {
				$i = $cpt;
				$spaces = '';
				while ($i-- > 0 && $results[$i]['id'] === T_WHITESPACE) {
					$spaces = $results[$i]['content'] . $spaces;
				}
				if ($i >= 0 && $results[$i]['id'] === T_ELSE) {
					$token['id'] = T_ELSEIF;
					$token['name'] = 'T_ELSEIF';
					$token['content'] = $results[$i]['content'] . $spaces . $token['content'];
					$cpt = $i;
				}
			}

			$results[$cpt++] = $token;
		}

		return $results;
	}
1344198
public function parse(Token $token)
    {
        $expr = $this->parser->getExpressionParser()->parseExpression();

        $this->parser->getStream()->expect(Token::BLOCK_END_TYPE);

        return new TransDefaultDomainNode($expr, $token->getLine(), $this->getTag());
    }
46298
public static Partitions parse(String nodes) {
        Partitions result = new Partitions();

        try {
            List<RedisClusterNode> mappedNodes = TOKEN_PATTERN.splitAsStream(nodes).filter(s -> !s.isEmpty())
                    .map(ClusterPartitionParser::parseNode).collect(Collectors.toList());
            result.addAll(mappedNodes);
        } catch (Exception e) {
            throw new RedisException("Cannot parse " + nodes, e);
        }

        return result;
    }
448539
public function parse(array $tokens)
    {
        // Filter away any whitespace
        $tokens = $this->filterTokens($tokens);

        // Insert missing implicit multiplication tokens
        if (static::allowImplicitMultiplication()) {
            $tokens = $this->parseImplicitMultiplication($tokens);
        }

        $this->tokens = $tokens;

        // Perform the actual parsing
        return $this->shuntingYard($tokens);
    }
1067152
public function parse(Token $token)
    {
        $lineno = $token->getLine();
        $stream = $this->parser->getStream();

        list ($file, $group, $priority, $attributes) = $this->parseArguments($token);

        $content = null;
        if (!$file) {
            $content = $this->parser->subparse([$this, 'decideBlockEnd'], true);
            $stream->expect(Token::BLOCK_END_TYPE);
        }

        return new TwigNodeStyle($content, $file, $group, $priority, $attributes, $lineno, $this->getTag());
    }
654417
public function replaceTokens($tokens){
        $keys = array_keys($tokens);
        $values = array_values($tokens);

        return str_replace($keys, $values, $this->getExpression());
    }