Parsing method that reads a list of one or more space-separated
tokens (nmtoken or name, depending on 'isNotation' argument)
510327
private WordResolver parseEnumerated(DTDElement elem, PrefixedName attrName,
                                         boolean isNotation)
        throws XMLStreamException
    {
        /* Need to use tree set to be able to construct the data
         * structs we need later on...
         */
        TreeSet<String> set = new TreeSet<String>();

        char c = skipDtdWs(true);
        if (c == ')') { // just to give more meaningful error msgs
            throwDTDUnexpectedChar(c, " (empty list; missing identifier(s))?");
        }

        HashMap<String,String> sharedEnums;

        if (isNotation) {
            sharedEnums = null;
        } else {
            sharedEnums = mSharedEnumValues;
            if (sharedEnums == null && !isNotation) {
                mSharedEnumValues = sharedEnums = new HashMap<String,String>();
            }
        }

        String id = isNotation ? readNotationEntry(c, attrName, elem.getLocation())
            : readEnumEntry(c, sharedEnums);
        set.add(id);
        
        while (true) {
            c = skipDtdWs(true);
            if (c == ')') {
                break;
            }
            if (c != '|') {
                throwDTDUnexpectedChar(c, "; missing '|' separator?");
            }
            c = skipDtdWs(true);
            id = isNotation ? readNotationEntry(c, attrName, elem.getLocation())
                : readEnumEntry(c, sharedEnums);
            if (!set.add(id)) {
                /* 03-Feb-2006, TSa: Hmmh. Apparently all other XML parsers
                 *    consider it's ok in non-validating mode. All right.
                 */
                if (mCfgFullyValidating) {
                    throwDTDAttrError("Duplicate enumeration value '"+id+"'",
                                      elem, attrName);
                }
            }
        }

        // Ok, let's construct the minimal data struct, then:
        return WordResolver.constructInstance(set);
    }
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
388221
protected List<String> parseNameList( DdlTokenStream tokens ) throws ParsingException {
        List<String> names = new LinkedList<String>();

        while (true) {
            names.add(parseName(tokens));

            if (!tokens.canConsume(COMMA)) {
                break;
            }
        }

        return names;
    }
1039436
def parse(contents, tokens=None):
    """"""
    # Shortcut for users who are interested in tokens
    if tokens is None:
        tokens = [t for t in tokenize(contents)]

    token_index, body = _ast_worker(tokens, len(tokens), 0, None)

    assert token_index == len(tokens)
    assert body.arguments == []

    return ToplevelBody(statements=body.statements)
707940
private static String[] splitOnTokens(String text, String oneAnySymbolStr, String noneOrMoreSymbolsStr) {
		if (!hasWildcards(text, ONE_ANY_SYMBOL, NONE_OR_MORE_SYMBOLS)) {
			return new String[] { text };
		}
		List<String> list = new ArrayList<>();
		StringBuilder buffer = new StringBuilder();

		boolean escapeMode = false;
		final int length = text.length();
		for (int i = 0; i < length; i++) {
			final char current = text.charAt(i);
			switch (current) {
				case ESCAPE_CHAR:
					if (escapeMode) {
						buffer.append(current);
					} else {
						escapeMode = true;
					}
					break;
				case ONE_ANY_SYMBOL:
					if (escapeMode) {
						buffer.append(current);
						escapeMode = false;
					} else {
						flushBuffer(buffer, list);
						if (!list.isEmpty() && noneOrMoreSymbolsStr.equals(list.get(list.size() - 1))) {
							list.set(list.size() - 1, oneAnySymbolStr);
							list.add(noneOrMoreSymbolsStr);
						} else {
							list.add(oneAnySymbolStr);
						}
					}
					break;
				case NONE_OR_MORE_SYMBOLS:
					if (escapeMode) {
						buffer.append(current);
						escapeMode = false;
					} else {
						flushBuffer(buffer, list);
						if (list.isEmpty() || !noneOrMoreSymbolsStr.equals(list.get(list.size() - 1))) {
							list.add(noneOrMoreSymbolsStr);
						}
					}
					break;
				default:
					if (escapeMode) {
						buffer.append(ESCAPE_CHAR);
						escapeMode = false;
					}
					buffer.append(current);
			}
		}
		if (escapeMode) {
			buffer.append(ESCAPE_CHAR);
		}
		if (buffer.length() != 0) {
			list.add(buffer.toString());
		}

		return list.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
	}
150778
def parse_tokens(self, tokens, debug=False):
        """"""
        # XXX Move the prefix computation into a wrapper around tokenize.
        p = parse.Parser(self.grammar, self.convert)
        p.setup()
        lineno = 1
        column = 0
        type = value = start = end = line_text = None
        prefix = u""
        for quintuple in tokens:
            type, value, start, end, line_text = quintuple
            if start != (lineno, column):
                assert (lineno, column) <= start, ((lineno, column), start)
                s_lineno, s_column = start
                if lineno < s_lineno:
                    prefix += "\n" * (s_lineno - lineno)
                    lineno = s_lineno
                    column = 0
                if column < s_column:
                    prefix += line_text[column:s_column]
                    column = s_column
            if type in (tokenize.COMMENT, tokenize.NL):
                prefix += value
                lineno, column = end
                if value.endswith("\n"):
                    lineno += 1
                    column = 0
                continue
            if type == token.OP:
                type = grammar.opmap[value]
            if debug:
                self.logger.debug("%s %r (prefix=%r)",
                                  token.tok_name[type], value, prefix)
            if p.addtoken(type, value, (prefix, start)):
                if debug:
                    self.logger.debug("Stop.")
                break
            prefix = ""
            lineno, column = end
            if value.endswith("\n"):
                lineno += 1
                column = 0
        else:
            # We never broke out -- EOF is too soon (how can this happen???)
            raise parse.ParseError("incomplete input",
                                   type, value, (prefix, start))
        return p.rootnode
1053497
def _parse_next_token(self):
        """"""
        while self._position < self.limit:
            token = self._next_pattern()
            if token:
                return token

        return None
1228414
def namelist_handle(tokens):
    """"""
    if len(tokens) == 1:
        return tokens[0]
    elif len(tokens) == 2:
        return tokens[0] + "\n" + tokens[0] + " = " + tokens[1]
    else:
        raise CoconutInternalException("invalid in-line nonlocal / global tokens", tokens)
186112
def parse_list(lexer: Lexer, is_const: bool) -> ListValueNode:
    """"""
    start = lexer.token
    item = parse_const_value if is_const else parse_value_value
    return ListValueNode(
        values=any_nodes(lexer, TokenKind.BRACKET_L, item, TokenKind.BRACKET_R),
        loc=loc(lexer, start),
    )
253295
private String readNmtoken(boolean isName) throws SAXException, IOException {
        char c;

        nameBufferPos = 0;

        // Read the first character.
        while (true) {
            c = readCh();
            switch (c) {
                case '%':
                case '<':
                case '>':
                case '&':
                case ',':
                case '|':
                case '*':
                case '+':
                case '?':
                case ')':
                case '=':
                case '\'':
                case '"':
                case '[':
                case ' ':
                case '\t':
                case '\n':
                case '\r':
                case ';':
                case '/':
                    unread(c);
                    if (nameBufferPos == 0) {
                        fatal("name expected");
                    }
                    String s = intern(nameBuffer, 0, nameBufferPos);
                    nameBufferPos = 0;
                    return s;
                default:
                    if (isName && (nameBufferPos == 0)
                            && (!(NCName.isNCNameStart(c)))) {
                        fatal("Not a name start character, U+"
                                + Integer.toHexString(c));
                    } else if (!(NCName.isNCNameTrail(c) || c == ':')) {
                        fatal("Not a name character, U+"
                                + Integer.toHexString(c));
                    }
                    if (nameBufferPos >= nameBuffer.length) {
                        nameBuffer = (char[]) extendArray(nameBuffer,
                                nameBuffer.length, nameBufferPos);
                    }
                    nameBuffer[nameBufferPos++] = c;
            }
        }
    }
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
787622
def parse_nbt(literal):
    """"""
    parser = Parser(tokenize(literal))
    tag = parser.parse()

    cursor = parser.token_span[1]
    leftover = literal[cursor:]
    if leftover.strip():
        parser.token_span = cursor, cursor + len(leftover)
        raise parser.error(f'Expected end of string but got {leftover!r}')

    return tag
1316301
def _parse_newline_list(self, indent):
        """"""
        if self._cur_token['type'] not in self._literals:
            raise Exception(
                "Parser failed, _parse_newline_list was called on non-literal"
                " {} on line {}.".format(
                    repr(self._cur_token['value']), self._cur_token['line']
                )
            )

        array = []
        new_indent = indent
        while not self._finished:
            if new_indent < indent:
                break
            elif new_indent == indent:
                while self._cur_token['type'] is TT.lbreak:
                    self._skip_newlines()
                    self._skip_whitespace()
                # look ahead to see if it's a comma seperated list
                temp_position = self._cur_position
                while (
                    temp_position < self.num_tokens-1 and
                    (
                        self.tokens[temp_position]['type'] is TT.ws or
                        self.tokens[temp_position]['type'] in self._literals
                    )
                ):
                    temp_position += 1

                if self.tokens[temp_position]['type'] is TT.comma:
                    array.append(self._parse_comma_list())
                else:
                    if self._cur_token['type'] is not TT.hyphen:
                        array.append(self._cur_token['value'])
                    elif self._nth_token()['type'] is TT.hyphen:
                        # two consecutive '-'s
                        array.append([])
                        self._increment()
                    self._increment()
            else:  # new_indent > indent
                while self._cur_token['type'] is TT.lbreak:
                    self._skip_newlines()
                    self._skip_whitespace()
                array.append(self._parse_newline_list(new_indent))

            self._skip_whitespace()
            if (
                not self._finished and
                self._cur_token['type'] not in (TT.lbreak, TT.hyphen)
            ):
                raise ParseError('newline', self._cur_token)

            temp_position = self._cur_position
            new_indent = 0
            while (
                temp_position < self.num_tokens-1 and
                self.tokens[temp_position]['type'] in (TT.lbreak, TT.ws)
            ):
                if self.tokens[temp_position]['type'] is TT.lbreak:
                    new_indent = 0
                else:
                    new_indent += 1
                temp_position += 1

        return array
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
790127
private List<String[]> readSection(BufferedReader bufferedReader) {
		List<String[]> section = new ArrayList<String[]>();
		String line = "";

		String currKey = null;
		StringBuffer currVal = new StringBuffer();
		boolean done = false;
		int linecount = 0;

		try {
			while (!done) {
				bufferedReader.mark(320);
				line = bufferedReader.readLine();
				String firstSecKey = section.isEmpty() ? ""
						: section.get(0)[0];
				if (line != null && line.matches("\\p{Space}*")) {
					// regular expression \p{Space}* will match line
					// having only white space characters
					continue;
				}
				if (line == null
						|| (!line.startsWith(" ") && linecount++ > 0 && (!firstSecKey
						.equals(START_SEQUENCE_TAG) || line
						.startsWith(END_SEQUENCE_TAG)))) {
					// dump out last part of section
					section.add(new String[]{currKey, currVal.toString()});
					bufferedReader.reset();
					done = true;
				} else {
					Matcher m = sectp.matcher(line);
					if (m.matches()) {
						// new key
						if (currKey != null) {
							section.add(new String[]{currKey,
								currVal.toString()});
						}
						// key = group(2) or group(4) or group(6) - whichever is
						// not null
						currKey = m.group(2) == null ? (m.group(4) == null ? m
								.group(6) : m.group(4)) : m.group(2);
						currVal = new StringBuffer();
			// val = group(3) if group(2) not null, group(5) if
						// group(4) not null, "" otherwise, trimmed
						currVal.append((m.group(2) == null ? (m.group(4) == null ? ""
								: m.group(5))
								: m.group(3)).trim());
					} else {
						// concatted line or SEQ START/END line?
						if (line.startsWith(START_SEQUENCE_TAG)
								|| line.startsWith(END_SEQUENCE_TAG)) {
							currKey = line;
						} else {
							currVal.append("\n"); // newline in between lines -
							// can be removed later
							currVal.append(currKey.charAt(0) == '/' ? line
									.substring(21) : line.substring(12));
						}
					}
				}
			}
		} catch (IOException e) {
			throw new ParserException(e.getMessage());
		} catch (RuntimeException e) {
			throw new ParserException(e.getMessage());
		}
		return section;
	}
1365358
def _parse_argument(cls, token):
        
        description = ""
        validator = None

        if " : " in token:
            token, description = tuple(token.split(" : ", 2))

            token = token.strip()

            description = description.strip()

        # Checking validator:
        matches = re.match(r"(.*)\((.*?)\)", token)
        if matches:
            token = matches.group(1).strip()
            validator = matches.group(2).strip()

        if token.endswith("?*"):
            return _argument(
                token.rstrip("?*"),
                Argument.MULTI_VALUED | Argument.OPTIONAL,
                description,
                None,
            )
        elif token.endswith("*"):
            return _argument(
                token.rstrip("*"),
                Argument.MULTI_VALUED | Argument.REQUIRED,
                description,
                None,
            )
        elif token.endswith("?"):
            return _argument(token.rstrip("?"), Argument.OPTIONAL, description, None)

        matches = re.match(r"(.+)=(.+)", token)
        if matches:
            return _argument(
                matches.group(1), Argument.OPTIONAL, description, matches.group(2)
            )

        return _argument(token, Argument.REQUIRED, description, None)
509172
def __parse(self, tokenlist):
        """"""
        #empty stack
        #iterate over symbollist
        tokenlist = [x for x in tokenlist]
        if not isinstance(tokenlist, list):
            raise TypeError("Expected list, got %s" % tokenlist.__class__.__name__)
        LOG.debug("get_trees: checking list: " + str(tokenlist))
        stack = [(0, Extended_S)]
        while True:
            state = stack[-1][0]
            if len(tokenlist):#FIXME: tokenlist with one element is reported as false
                token = tokenlist[0]
            else:
                token = EndSymbol()
            newdic = self.__parsertable.insert(state, token)
            action = newdic["action"]
            if action == "Fail":
                return False
            elif action == "Accept":
                return True
            if action == "Reduce":
                reductionrule = newdic["rule"]
                #TODO extract len(right side) of the rule and insert left side
                for rsymbol in reversed(reductionrule.rightside):
                    state, symbol = stack.pop() # TODO: check
                state = stack[-1][0]
                state = self.__parsertable.goto(state,reductionrule.leftside[0])
                stack.append((state, reductionrule.leftside[0]))
            elif action == "Shift":
                stack.append((newdic['dest'], tokenlist.pop(0)))
            else:
                raise ValueError("Unknown action")
        return False