Expand all unqualified names using the given namespace context in the given source code tokens.

@param array $tokens
@param NamespaceContext $context
@return string
342066
protected function expandNames(array & $tokens, NamespaceContext $context)
	{
		// Expand name if one of these tokens is found before the name.
		static $beforeTokens = [
			T_NEW, // Object instantiation
			T_INSTANCEOF // Instanceof check
		];
		
		// Expand name if one of these tokens is found after the name
		static $afterTokens = [
			T_DOUBLE_COLON, // Static access or class constant lookup
			T_VARIABLE // Type-hinted function / method argument
		];
		
		$count = count($tokens) - 1;
		$buffer = '';
		
		for($size = count($tokens), $i = 0; $i < $size; $i++)
		{
			if(is_array($tokens[$i]) && ($tokens[$i][0] == T_STRING || $tokens[$i][0] == T_NS_SEPARATOR))
			{
				$pos = $i;
				$before = $pos - 1;
				$after = $pos + 1;
				$name = $tokens[$i][1];
				
				$i++;
				
				// Read name with namespace parts if needed.
				for(; $i < $size; $i++)
				{
					if(is_array($tokens[$i]))
					{
						switch($tokens[$i][0])
						{
							case T_WHITESPACE:
							case T_COMMENT:
							case T_DOC_COMMENT:
								continue 2;
							case T_NS_SEPARATOR:
							case T_STRING:
								$name .= $tokens[$i][1];
								continue 2;
						}
					}
					
					break;
				}
				
				// Read first significant token before the name.
				$prev = NULL;
				while($before >= 0)
				{
					$prev = $tokens[$before];
					
					if(is_array($prev))
					{
						switch($prev[0])
						{
							case T_WHITESPACE:
							case T_COMMENT:
							case T_DOC_COMMENT:
								$before--;
								continue 2;
						}
					}
					
					break;
				}
				
				// Read first significant token after the name.
				$next = NULL;
				while($after < $count)
				{
					$next = $tokens[$after];
						
					if(is_array($next))
					{
						switch($next[0])
						{
							case T_WHITESPACE:
							case T_COMMENT:
							case T_DOC_COMMENT:
								$after++;
								continue 2;
						}
					}
						
					break;
				}
				
				// Expand names as needed.
				$expand = false;
				
				if(is_array($prev) && in_array($prev[0], $beforeTokens))
				{
					$expand = true;
				}
				
				if(is_array($next) && in_array($next[0], $afterTokens))
				{
					$expand = true;
				}
				
				if($expand)
				{
					$buffer .= '\\' . ltrim($context->lookup($name), '\\');
				}
				else
				{
					$buffer .= $name;
				}
			}
			
			if(is_array($tokens[$i]))
			{
				$buffer .= $tokens[$i][1];
			}
			else
			{
				$buffer .= $tokens[$i];
			}
		}
		
		return $buffer;
	}
823601
private function getNamespaceDeclarations(Tokens $tokens)
    {
        $namespaces = array();

        foreach ($tokens as $index => $token) {
            if (!$token->isGivenKind(T_NAMESPACE)) {
                continue;
            }

            $declarationEndIndex = $tokens->getNextTokenOfKind($index, array(';', '{'));

            $namespaces[] = array(
                'end' => $declarationEndIndex,
                'name' => trim($tokens->generatePartialCode($index + 1, $declarationEndIndex - 1)),
                'start' => $index,
            );
        }

        return $namespaces;
    }
801227
protected static function _tokenize(array $tokens) {
		$inString = false;
		$results = array();
		$cpt = 0;
		$previousTokenId = null;
		foreach ($tokens as $tokenId => $token) {
			if ($token['content'] === '"') {
				if (!$inString) {
					$token['id'] = T_START_DOUBLE_QUOTE;
					$token['name'] = 'T_START_DOUBLE_QUOTE';
				} else {
					$token['id'] = T_END_DOUBLE_QUOTE;
					$token['name'] = 'T_END_DOUBLE_QUOTE';
				}
				$inString = !$inString;
			}

			$isCurlyBrace = (
				$token['content'] === '$' &&
				isset($tokens[$tokenId + 1]) &&
				$tokens[$tokenId + 1]['content'] === '{'
			);
			if ($isCurlyBrace) {
				$token['id'] = T_DOLLAR_CURLY_BRACES;
				$token['name'] = 'T_DOLLAR_CURLY_BRACES';
			}

			$isArray = (
				!$token['id'] &&
				$token['content'] === '(' &&
				$previousTokenId !== null &&
				$tokens[$previousTokenId]['id'] === T_ARRAY
			);
			if ($isArray) {
				$token['id'] = T_ARRAY_OPEN;
				$token['name'] = 'T_ARRAY_OPEN';
			}

			$isShortArray = (
				!$token['id'] &&
				$token['content'] === '[' &&
				$previousTokenId !== null && (
					$tokens[$previousTokenId]['id'] !== T_VARIABLE &&
					$tokens[$previousTokenId]['id'] !== T_ENCAPSED_AND_WHITESPACE
				)
			);
			if ($isShortArray) {
				$token['id'] = T_SHORT_ARRAY_OPEN;
				$token['name'] = 'T_SHORT_ARRAY_OPEN';
			}

			if ($token['content'] === '(' && !$token['id']) {
				$token['id'] = T_START_BRACKET;
				$token['name'] = 'T_START_BRACKET';
			}

			if ($token['id'] !== T_WHITESPACE) {
				$previousTokenId = $tokenId;
			}

			if ($token['id'] === T_IF) {
				$i = $cpt;
				$spaces = '';
				while ($i-- > 0 && $results[$i]['id'] === T_WHITESPACE) {
					$spaces = $results[$i]['content'] . $spaces;
				}
				if ($i >= 0 && $results[$i]['id'] === T_ELSE) {
					$token['id'] = T_ELSEIF;
					$token['name'] = 'T_ELSEIF';
					$token['content'] = $results[$i]['content'] . $spaces . $token['content'];
					$cpt = $i;
				}
			}

			$results[$cpt++] = $token;
		}

		return $results;
	}
116001
public static function transpileMatchRegex(array $tokens)
    {
        $regex = [];

        foreach ($tokens as $token) {
            $var = $token instanceof Variable ? $token : ($token instanceof Delimiter ? $token->next : null);

            if (null !== $var && $var instanceof Variable && null !== ($optgrp = self::makeOptGrp($var))) {
                $regex[] = $optgrp;
                break;
            }

            $regex[] = $token;
        }

        return implode('', $regex);
    }
996120
protected function fromTokens($tokens, $parseContext)
    {
        $analyser = new Analyser();
        $analysis = new Analysis();
        reset($tokens);
        $token = '';
        $imports = Analyser::$defaultImports; // Use @OA\* for swagger-php annotations (unless overwritten by a use statement)

        $parseContext->uses = [];
        $schemaContext = $parseContext; // Use the parseContext until a definitionContext  (class or trait) is created.
        $classDefinition = false;
        $traitDefinition = false;
        $comment = false;
        $line = 0;
        $lineOffset = $parseContext->line ?: 0;
        while ($token !== false) {
            $previousToken = $token;
            $token = $this->nextToken($tokens, $parseContext);
            if (is_array($token) === false) { // Ignore tokens like "{", "}", etc
                continue;
            }
            if ($token[0] === T_DOC_COMMENT) {
                if ($comment) { // 2 Doc-comments in succession?
                    $this->analyseComment($analysis, $analyser, $comment, new Context(['line' => $line], $schemaContext));
                }
                $comment = $token[1];
                $line = $token[2] + $lineOffset;
                continue;
            }
            if (in_array($token[0], [T_ABSTRACT, T_FINAL])) {
                $token = $this->nextToken($tokens, $parseContext); // Skip "abstract" and "final" keywords
            }
            if ($token[0] === T_CLASS) { // Doc-comment before a class?
                if (is_array($previousToken) && $previousToken[0] === T_DOUBLE_COLON) {
                    //php 5.5 class name resolution (i.e. ClassName::class)
                    continue;
                }
                $token = $this->nextToken($tokens, $parseContext);

                if (is_string($token) && ($token === '(' || $token === '{')) {
                    // php7 anonymous classes (i.e. new class() { public function foo() {} };)
                    continue;
                }

                $schemaContext = new Context(['class' => $token[1], 'line' => $token[2]], $parseContext);
                if ($classDefinition) {
                    $analysis->addClassDefinition($classDefinition);
                }
                $classDefinition = [
                    'class' => $token[1],
                    'extends' => null,
                    'properties' => [],
                    'methods' => [],
                    'context' => $schemaContext,
                ];
                // @todo detect end-of-class and reset $schemaContext
                $token = $this->nextToken($tokens, $parseContext);
                if ($token[0] === T_EXTENDS) {
                    $schemaContext->extends = $this->parseNamespace($tokens, $token, $parseContext);
                    $classDefinition['extends'] = $schemaContext->fullyQualifiedName($schemaContext->extends);
                }
                if ($comment) {
                    $schemaContext->line = $line;
                    $this->analyseComment($analysis, $analyser, $comment, $schemaContext);
                    $comment = false;
                    continue;
                }
            }
            if ($token[0] === T_TRAIT) {
                $classDefinition = false;
                $token = $this->nextToken($tokens, $parseContext);
                $schemaContext = new Context(['trait' => $token[1], 'line' => $token[2]], $parseContext);
                if ($traitDefinition) {
                    $analysis->addTraitDefinition($traitDefinition);
                }
                $traitDefinition = [
                    'trait' => $token[1],
                    'properties' => [],
                    'methods' => [],
                    'context' => $schemaContext,
                ];
                if ($comment) {
                    $schemaContext->line = $line;
                    $this->analyseComment($analysis, $analyser, $comment, $schemaContext);
                    $comment = false;
                    continue;
                }
            }
            if ($token[0] === T_STATIC) {
                $token = $this->nextToken($tokens, $parseContext);
                if ($token[0] === T_VARIABLE) { // static property
                    $propertyContext = new Context(
                        [
                            'property' => substr($token[1], 1),
                            'static' => true,
                            'line' => $line,
                        ],
                        $schemaContext
                    );
                    if ($classDefinition) {
                        $classDefinition['properties'][$propertyContext->property] = $propertyContext;
                    }
                    if ($traitDefinition) {
                        $traitDefinition['properties'][$propertyContext->property] = $propertyContext;
                    }
                    if ($comment) {
                        $this->analyseComment($analysis, $analyser, $comment, $propertyContext);
                        $comment = false;
                    }
                    continue;
                }
            }

            if (in_array($token[0], [T_PRIVATE, T_PROTECTED, T_PUBLIC, T_VAR])) { // Scope
                $token = $this->nextToken($tokens, $parseContext);
                if ($token[0] == T_STATIC) {
                    $token = $this->nextToken($tokens, $parseContext);
                }
                if ($token[0] === T_VARIABLE) { // instance property
                    $propertyContext = new Context(
                        [
                            'property' => substr($token[1], 1),
                            'line' => $line,
                        ],
                        $schemaContext
                    );
                    if ($classDefinition) {
                        $classDefinition['properties'][$propertyContext->property] = $propertyContext;
                    }
                    if ($traitDefinition) {
                        $traitDefinition['properties'][$propertyContext->property] = $propertyContext;
                    }
                    if ($comment) {
                        $this->analyseComment($analysis, $analyser, $comment, $propertyContext);
                        $comment = false;
                    }
                } elseif ($token[0] === T_FUNCTION) {
                    $token = $this->nextToken($tokens, $parseContext);
                    if ($token[0] === T_STRING) {
                        $methodContext = new Context(
                            [
                                'method' => $token[1],
                                'line' => $line,
                            ],
                            $schemaContext
                        );
                        if ($classDefinition) {
                            $classDefinition['methods'][$token[1]] = $methodContext;
                        }
                        if ($traitDefinition) {
                            $traitDefinition['methods'][$token[1]] = $methodContext;
                        }
                        if ($comment) {
                            $this->analyseComment($analysis, $analyser, $comment, $methodContext);
                            $comment = false;
                        }
                    }
                }
                continue;
            } elseif ($token[0] === T_FUNCTION) {
                $token = $this->nextToken($tokens, $parseContext);
                if ($token[0] === T_STRING) {
                    $methodContext = new Context(
                        [
                            'method' => $token[1],
                            'line' => $line,
                        ],
                        $schemaContext
                    );
                    if ($classDefinition) {
                        $classDefinition['methods'][$token[1]] = $methodContext;
                    }
                    if ($traitDefinition) {
                        $traitDefinition['methods'][$token[1]] = $methodContext;
                    }
                    if ($comment) {
                        $this->analyseComment($analysis, $analyser, $comment, $methodContext);
                        $comment = false;
                    }
                }
            }
            if (in_array($token[0], [T_NAMESPACE, T_USE]) === false) { // Skip "use" & "namespace" to prevent "never imported" warnings)
                // Not a doc-comment for a class, property or method?
                if ($comment) {
                    $this->analyseComment($analysis, $analyser, $comment, new Context(['line' => $line], $schemaContext));
                    $comment = false;
                }
            }
            if ($token[0] === T_NAMESPACE) {
                $parseContext->namespace = $this->parseNamespace($tokens, $token, $parseContext);
                continue;
            }
            if ($token[0] === T_USE) {
                $statements = $this->parseUseStatement($tokens, $token, $parseContext);
                foreach ($statements as $alias => $target) {
                    if ($target[0] === '\\') {
                        $target = substr($target, 1);
                    }

                    $parseContext->uses[$alias] = $target;

                    // i'm in the case use trait
                    if ($alias == $target && $classDefinition) {
                        $classDefinition['traits'][] = $alias;
                    }

                    if (Analyser::$whitelist === false) {
                        $imports[strtolower($alias)] = $target;
                    } else {
                        foreach (Analyser::$whitelist as $namespace) {
                            if (strcasecmp(substr($target, 0, strlen($namespace)), $namespace) === 0) {
                                $imports[strtolower($alias)] = $target;
                                break;
                            }
                        }
                    }
                }
                $analyser->docParser->setImports($imports);
                continue;
            }
        }
        if ($comment) { // File ends with a T_DOC_COMMENT
            $this->analyseComment($analysis, $analyser, $comment, new Context(['line' => $line], $schemaContext));
        }
        if ($classDefinition) {
            $analysis->addClassDefinition($classDefinition);
        }
        if ($traitDefinition) {
            $analysis->addTraitDefinition($traitDefinition);
        }

        return $analysis;
    }
77178
private function activeNamespace(int $tokenID): string
    {
        foreach ($this->namespaces as $namespace => $position) {
            if ($tokenID >= $position[self::O_TOKEN] && $tokenID <= $position[self::C_TOKEN]) {
                return $namespace;
            }
        }

        //Seems like no namespace declaration
        $this->namespaces[''] = [
            self::O_TOKEN => 0,
            self::C_TOKEN => count($this->tokens),
            self::N_USES  => [],
        ];

        return '';
    }
1039683
private function skipToNextStringOrNamespaceSeparator(\ArrayIterator $tokens)
    {
        while ($tokens->valid() && (T_STRING !== $tokens->current()[0]) && (T_NS_SEPARATOR !== $tokens->current()[0])) {
            $tokens->next();
        }
    }
525298
private static function getNthTokenWithoutWhitespace($tokens, $i) {
		$j = 0;
		foreach ($tokens as $token) {
			if ($token['token'] == 'T_WHITESPACE') {
				continue;
			}
			if ($i == $j) {
				return $token;
			}
			$j++;
		}
		return null;
	}
1235605
private function parseTypes(\ArrayIterator $tokens, Context $context, $parserContext)
    {
        $types = [];
        $token = '';
        while ($tokens->valid()) {
            $token = $tokens->current();

            if ($token === '|') {
                if (count($types) === 0) {
                    throw new \RuntimeException(
                        'A type is missing before a type separator'
                    );
                }

                if ($parserContext !== self::PARSER_IN_COMPOUND
                    && $parserContext !== self::PARSER_IN_ARRAY_EXPRESSION
                    && $parserContext !== self::PARSER_IN_COLLECTION_EXPRESSION
                ) {
                    throw new \RuntimeException(
                        'Unexpected type separator'
                    );
                }

                $tokens->next();
            } elseif ($token === '?') {
                if ($parserContext !== self::PARSER_IN_COMPOUND
                    && $parserContext !== self::PARSER_IN_ARRAY_EXPRESSION
                    && $parserContext !== self::PARSER_IN_COLLECTION_EXPRESSION
                ) {
                    throw new \RuntimeException(
                        'Unexpected nullable character'
                    );
                }

                $tokens->next();
                $type = $this->parseTypes($tokens, $context, self::PARSER_IN_NULLABLE);
                $types[] = new Nullable($type);
            } elseif ($token === '(') {
                $tokens->next();
                $type = $this->parseTypes($tokens, $context, self::PARSER_IN_ARRAY_EXPRESSION);

                $resolvedType = new Array_($type);

                $token = $tokens->current();
                // Someone did not properly close their array expression ..
                if ($token === null) {
                    break;
                }

                // we generate arrays corresponding to the number of '[]' after the ')'
                $numberOfArrays = (strlen($token) - 1) / 2;
                for ($i = 0; $i < $numberOfArrays - 1; ++$i) {
                    $resolvedType = new Array_($resolvedType);
                }

                $types[] = $resolvedType;
                $tokens->next();
            } elseif ($parserContext === self::PARSER_IN_ARRAY_EXPRESSION && $token[0] === ')') {
                break;
            } elseif ($token === '<') {
                if (count($types) === 0) {
                    throw new \RuntimeException(
                        'Unexpected collection operator "<", class name is missing'
                    );
                }

                $classType = array_pop($types);
                if ($classType !== null) {
                    $types[] = $this->resolveCollection($tokens, $classType, $context);
                }

                $tokens->next();
            } elseif ($parserContext === self::PARSER_IN_COLLECTION_EXPRESSION
                && ($token === '>' || trim($token) === ',')
                ) {
                break;
            } else {
                $type = $this->resolveSingleType($token, $context);
                $tokens->next();
                if ($parserContext === self::PARSER_IN_NULLABLE) {
                    return $type;
                }

                $types[] = $type;
            }
        }

        if ($token === '|') {
            throw new \RuntimeException(
                'A type is missing after a type separator'
            );
        }

        if (count($types) === 0) {
            if ($parserContext === self::PARSER_IN_NULLABLE) {
                throw new \RuntimeException(
                    'A type is missing after a nullable character'
                );
            }

            if ($parserContext === self::PARSER_IN_ARRAY_EXPRESSION) {
                throw new \RuntimeException(
                    'A type is missing in an array expression'
                );
            }

            if ($parserContext === self::PARSER_IN_COLLECTION_EXPRESSION) {
                throw new \RuntimeException(
                    'A type is missing in a collection expression'
                );
            }
        } elseif (count($types) === 1) {
            return $types[0];
        }

        return new Compound($types);
    }
583194
public static function normalize(array $context)
    {
        $diff = array_diff_key($context, self::$defaults);
        if (!empty($diff)) {
            $errorMessage = 'unknown index '.implode(', ', array_keys($diff));
            throw new InvalidContext($errorMessage);
        }
        if (!empty($context['namespace'])) {
            if (preg_match('/^\\\\?(.*?)\\\\?$/s', $context['namespace'], $ns)) {
                $context['namespace'] = $ns[1] ? $ns[1].'\\' : null;
            }
        } else {
            $context['namespace'] = null;
        }
        return array_replace(self::$defaults, $context);
    }
908437
private function getLicenseTokenIndex(array $tokens)
    {
        $licenseTokenIndex = null;

        foreach ($tokens as $index => $token) {
            if ($token[0] === T_COMMENT) {
                $licenseTokenIndex = $index;
            }

            // if we reach the class declaration then it does not have a license
            if ($token[0] === T_CLASS) {
                break;
            }
        }

        return $licenseTokenIndex;
    }
1010459
public Parser<Token> token(String... tokenNames) {
    if (tokenNames.length == 0) return Parsers.never();
    @SuppressWarnings("unchecked")
    Parser<Token>[] ps = new Parser[tokenNames.length];
    for(int i = 0; i < tokenNames.length; i++) {
      ps[i] = Parsers.token(InternalFunctors.tokenWithSameValue(word(tokenNames[i])));
    }
    return Parsers.or(ps);
  }
21798
def add_token_to_namespace(self, token: str, namespace: str = 'tokens') -> int:
        
        if not isinstance(token, str):
            raise ValueError("Vocabulary tokens must be strings, or saving and loading will break."
                             "  Got %s (with type %s)" % (repr(token), type(token)))
        if token not in self._token_to_index[namespace]:
            index = len(self._token_to_index[namespace])
            self._token_to_index[namespace][token] = index
            self._index_to_token[namespace][index] = token
            return index
        else:
            return self._token_to_index[namespace][token]
996650
public static function alias(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        $operation = SqlOperation::create($tokens);
        if ($operation === SqlOperations::INSERT) return null;
        if ($operation === SqlOperations::UPDATE) return $tokens['UPDATE'][0]['alias']['name'];
        return $tokens['FROM'][0]['alias']['name'];
    }
966725
protected function extractNamespace(array $tokens) : string {
    $namespace = '';
    $found = FALSE;
    foreach ($tokens as $token) {
      if (is_array($token) && $token[0] == T_NAMESPACE) {
        $found = TRUE;
      }
      if (is_array($token) && ($token[0] == T_NS_SEPARATOR || $token[0] == T_STRING)) {
        $namespace .= $token[1];
      }
      elseif ($found && $token == ';') {
        return $namespace;
      }
    }
    return $namespace;
  }
835396
public function parse(array $tokens)
    {
        $symbolNodes = $this->detectSymbols($tokens);

        $nodes = $this->createTreeByBrackets($symbolNodes);

        $nodes = $this->transformTreeByFunctions($nodes);

        $this->checkGrammar($nodes);

        // Wrap the nodes in an array node.
        $rootNode = new ContainerNode($nodes);

        return $rootNode;
    }
530777
private static function getName($key, array $tokens): ?string
    {
        $class      = null;
        $tokenCount = \count($tokens);

        for ($i = $key; $i < $tokenCount; $i++) {
            if (\is_array($tokens[$i])) {
                if ($tokens[$i][0] === \T_STRING) {
                    $class .= $tokens[$i][1];
                } elseif ($tokens[$i][0] === \T_WHITESPACE) {
                    return $class;
                }
            }
        }

        return null;
    }