Takes a list of tokens from the tokenizer, and returns a parse tree.
173905
def parse_regex(regex_tokens):
    
    # We add a closing brace because that represents the final pop of the stack.
    tokens = [')'] + regex_tokens[::-1]

    def wrap(lst):
        """ Turn list into sequence when it contains several items. """
        if len(lst) == 1:
            return lst[0]
        else:
            return Sequence(lst)

    def _parse():
        or_list = []
        result = []

        def wrapped_result():
            if or_list == []:
                return wrap(result)
            else:
                or_list.append(result)
                return Any([wrap(i) for i in or_list])

        while tokens:
            t = tokens.pop()

            if t.startswith('(?P<'):
                variable = Variable(_parse(), varname=t[4:-1])
                result.append(variable)

            elif t in ('*', '*?'):
                greedy = (t == '*')
                result[-1] = Repeat(result[-1], greedy=greedy)

            elif t in ('+', '+?'):
                greedy = (t == '+')
                result[-1] = Repeat(result[-1], min_repeat=1, greedy=greedy)

            elif t in ('?', '??'):
                if result == []:
                    raise Exception('Nothing to repeat.' + repr(tokens))
                else:
                    greedy = (t == '?')
                    result[-1] = Repeat(result[-1], min_repeat=0, max_repeat=1, greedy=greedy)

            elif t == '|':
                or_list.append(result)
                result = []

            elif t in ('(', '(?:'):
                result.append(_parse())

            elif t == '(?!':
                result.append(Lookahead(_parse(), negative=True))

            elif t == '(?=':
                result.append(Lookahead(_parse(), negative=False))

            elif t == ')':
                return wrapped_result()

            elif t.startswith('#'):
                pass

            elif t.startswith('{'):
                # TODO: implement!
                raise Exception('{}-style repitition not yet supported' % t)

            elif t.startswith('(?'):
                raise Exception('%r not supported' % t)

            elif t.isspace():
                pass
            else:
                result.append(Regex(t))

        raise Exception("Expecting ')' token")

    result = _parse()

    if len(tokens) != 0:
        raise Exception("Unmatched parantheses.")
    else:
        return result
269682
def get_tokens(self, format_string):
        
        return [t for t in self.tokenize_pattern.split(format_string) if t]
1010459
public Parser<Token> token(String... tokenNames) {
    if (tokenNames.length == 0) return Parsers.never();
    @SuppressWarnings("unchecked")
    Parser<Token>[] ps = new Parser[tokenNames.length];
    for(int i = 0; i < tokenNames.length; i++) {
      ps[i] = Parsers.token(InternalFunctors.tokenWithSameValue(word(tokenNames[i])));
    }
    return Parsers.or(ps);
  }
321770
def _parse_literal_list(self, indent):
        """"""
        if self._cur_token['type'] not in self._literals:
            raise Exception(
                "Parser failed, _parse_literal_list was called on non-literal"
                " {} on line {}.".format(
                    repr(self._cur_token['value']), self._cur_token['line']
                )
            )

        # find next token after whitespace without incrementing
        temp_position = self._cur_position
        while (
            temp_position < self.num_tokens-1 and (
                self.tokens[temp_position]['type'] is TT.ws or
                self.tokens[temp_position]['type'] in self._literals
            )
        ):
            temp_position += 1
        next_token = self.tokens[temp_position]

        # end of stream
        if next_token['type'] is TT.ws:
            return self._cur_token['value']
        elif next_token['type'] is TT.comma:
            return self._parse_comma_list()
        elif next_token['type'] is TT.lbreak:
            while (
                temp_position < self.num_tokens-1 and
                self.tokens[temp_position]['type'] in (TT.lbreak, TT.ws)
            ):
                temp_position += 1
            if self.tokens[temp_position]['type'] in self._literals:
                return self._parse_newline_list(indent)
            else:
                rval = self._cur_token['value']
                self._increment()
                return rval
        else:
            rval = self._cur_token['value']
            self._increment()
            return rval
607764
public CommonTree parse(final CommonTokenStream tokens)
            throws RecognizerException {
        if (_log.isDebugEnabled()) {
            _log.debug("3. Parsing tokens: {}", tokens.toString());
        }
        try {
            CobolStructureParser parser = new CobolStructureParserImpl(tokens,
                    getErrorHandler());
            cobdata_return parserResult = parser.cobdata();
            if (parser.getNumberOfSyntaxErrors() != 0 || parserResult == null) {
                throw (new RecognizerException("Parsing failed. "
                        + parser.getNumberOfSyntaxErrors() + " syntax errors."
                        + " Last error was "
                        + getErrorHistory().get(getErrorHistory().size() - 1)));
            }
            return (CommonTree) parserResult.getTree();
        } catch (RecognitionException e) {
            throw (new RecognizerException(e));
        }
    }
544560
private void convertTokenDefinitions() throws GrammarException,
	    TreeException {
	tokenVisibility.clear();
	Map<String, ParseTreeNode> helpers = getHelperTokens();
	Map<String, ParseTreeNode> tokens = getTokens();
	convertTokenDefinitions(helpers, tokens);
    }
9310
def _token_list_matches_tree(self, token_list):
        
        try:
            parsed_selector = cssselect.parse(
                ''.join(token.as_css() for token in token_list))[0]

            return bool(
                self.tree.xpath(
                    self.xpath_translator.selector_to_xpath(parsed_selector)))
        except:
            # On error, assume the selector matches the tree
            return True
638256
public static Token[] parse(String text, boolean optimize) {
        if (text == null) {
            return null;
        }
        if (text.length() == 0) {
            Token t = new Token(text);
            return new Token[] { t };
        }
        Token[] tokens = null;
        List<Token> tokenList = Tokenizer.tokenize(text, optimize);
        if (!tokenList.isEmpty()) {
            tokens = tokenList.toArray(new Token[0]);
            if (optimize) {
                tokens = Tokenizer.optimize(tokens);
            }
        }
        return tokens;
    }
1356764
def iter_tokens(cls, blob):
    
    readline_func = io.StringIO(blob.decode('utf-8')).readline
    return tokenize.generate_tokens(readline_func)
27793
def parse_stream(stream):
    """"""
    code = []

    for (line, col, (token, value)) in Tokenizer(stream).tokenize():
        if token == Tokenizer.STRING:
            value = '"' + value + '"'
        code.append(value)

    return code
641002
def parse_declarations(input = @tokens, options = {})
      input = TokenScanner.new(input) unless input.is_a?(TokenScanner)
      consume_declarations(input, options)
    end
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
1305674
function matcherFromTokens( tokens, context, xml ) {
	var token, matcher,
		i = 0;

	for ( ; (token = tokens[i]); i++ ) {
		if ( Expr.relative[ token.part ] ) {
			matcher = addCombinator( matcher, Expr.relative[ token.part ], context );
		} else {
			token.captures.push( context, xml );
			matcher = addMatcher( matcher, Expr.filter[ token.part ].apply( null, token.captures ) );
		}
	}

	return matcher;
}
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
335542
private Map<String, ParseTreeNode> getHelperTokens() throws TreeException {
	Map<String, ParseTreeNode> helpers = new HashMap<String, ParseTreeNode>();
	ParseTreeNode helperTree = parserTree.getChild("Helper");
	ParseTreeNode helperDefinitions = helperTree.getChild("HelperDefinitions");
	for (ParseTreeNode helperDefinition : helperDefinitions
		.getChildren("HelperDefinition")) {
	    String identifier = helperDefinition.getChild("IDENTIFIER")
		    .getText();
	    helpers.put(identifier, helperDefinition);
	}
	return helpers;
    }
764704
def as_tokens(ctx: List[ParserRuleContext]) -> List[str]:
    
    return [as_token(e) for e in ctx]