Convenience shortcut method to parse a specific token type
357669
public BytesWritableToken visitBytesWritable(EditsElement e) throws IOException {
    return (BytesWritableToken)visit(tokenizer.read(new BytesWritableToken(e)));
  }
1364543
def datatype_from_token(self, token):
        

        if token.type == SystemRDLParser.ID:
            # Is an identifier for either an enum or struct type

            typ = self.compiler.namespace.lookup_type(get_ID_text(token))
            if typ is None:
                self.msg.fatal(
                    "Type '%s' is not defined" % get_ID_text(token),
                    SourceRef.from_antlr(token)
                )

            if rdltypes.is_user_enum(typ) or rdltypes.is_user_struct(typ):
                return typ
            else:
                self.msg.fatal(
                    "Type '%s' is not a struct or enum" % get_ID_text(token),
                    SourceRef.from_antlr(token)
                )

        else:
            return self._DataType_Map[token.type]
46383
Token parseAtom() throws ParseException {
		int ch = this.read();
		Token tok = null;
		switch (ch) {
		case T_LPAREN:
			return this.processParen();
		case T_LPAREN2:
			return this.processParen2(); // '(?:'
		case T_CONDITION:
			return this.processCondition(); // '(?('
		case T_MODIFIERS:
			return this.processModifiers(); // (?modifiers ... )
		case T_INDEPENDENT:
			return this.processIndependent();
		case T_DOT:
			this.next(); // Skips '.'
			tok = Token.token_dot;
			break;

		/**
		 * char-class ::= '[' ( '^'? range ','?)+ ']' range ::= '\d' | '\w' |
		 * '\s' | category-block | range-char | range-char '-' range-char
		 * range-char ::= '\[' | '\]' | '\\' | '\' [,-efnrtv] | bmp-code |
		 * character-2 bmp-char ::= '\' 'u' [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]
		 * [0-9a-fA-F]
		 */
		case T_LBRACKET:
			return this.parseCharacterClass(true);
		case T_SET_OPERATIONS:
			return this.parseSetOperations();

		case T_BACKSOLIDUS:
			switch (this.chardata) {
			case 'd':
			case 'D':
			case 'w':
			case 'W':
			case 's':
			case 'S':
				tok = this.getTokenForShorthand(this.chardata);
				this.next();
				return tok;

			case 'e':
			case 'f':
			case 'n':
			case 'r':
			case 't':
			case 'u':
			case 'v':
			case 'x': {
				int ch2 = this.decodeEscaped();
				if (ch2 < 0x10000) {
					tok = Token.createChar(ch2);
				} else {
					tok = Token.createString(REUtil.decomposeToSurrogates(ch2));
				}
			}
				break;

			case 'c':
				return this.processBacksolidus_c();
			case 'C':
				return this.processBacksolidus_C();
			case 'i':
				return this.processBacksolidus_i();
			case 'I':
				return this.processBacksolidus_I();
			case 'g':
				return this.processBacksolidus_g();
			case 'X':
				return this.processBacksolidus_X();
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
				return this.processBackreference();

			case 'P':
			case 'p':
				int pstart = this.offset;
				tok = processBacksolidus_pP(this.chardata);
				if (tok == null)
					throw this.ex("parser.atom.5", pstart);
				break;

			default:
				tok = Token.createChar(this.chardata);
			}
			this.next();
			break;

		case T_CHAR:
			if (this.chardata == ']' || this.chardata == '{'
					|| this.chardata == '}')
				throw this.ex("parser.atom.4", this.offset - 1);
			tok = Token.createChar(this.chardata);
			int high = this.chardata;
			this.next();
			if (REUtil.isHighSurrogate(high) && this.read() == T_CHAR
					&& REUtil.isLowSurrogate(this.chardata)) {
				char[] sur = new char[2];
				sur[0] = (char) high;
				sur[1] = (char) this.chardata;
				tok = Token.createParen(Token.createString(new String(sur)), 0);
				this.next();
			}
			break;

		default:
			throw this.ex("parser.atom.4", this.offset - 1);
		}
		return tok;
	}
751092
private Node parseTypeExpression(JsDocToken token) {
    // Save the source position before we consume additional tokens.
    int lineno = stream.getLineno();
    int charno = stream.getCharno();
    if (token == JsDocToken.QMARK) {
      // A QMARK could mean that a type is nullable, or that it's unknown.
      // We use look-ahead 1 to determine whether it's unknown. Otherwise,
      // we assume it means nullable. There are 8 cases:
      // {?} - right curly
      // ? - EOF (possible when the parseTypeString method is given a bare type expression)
      // {?=} - equals
      // {function(?, number)} - comma
      // {function(number, ?)} - right paren
      // {function(): ?|number} - pipe
      // {Array.<?>} - greater than
      // /** ? */ - EOC (inline types)
      // I'm not a big fan of using look-ahead for this, but it makes
      // the type language a lot nicer.
      token = next();
      if (token == JsDocToken.COMMA
          || token == JsDocToken.EQUALS
          || token == JsDocToken.RIGHT_SQUARE
          || token == JsDocToken.RIGHT_CURLY
          || token == JsDocToken.RIGHT_PAREN
          || token == JsDocToken.PIPE
          || token == JsDocToken.RIGHT_ANGLE
          || token == JsDocToken.EOC
          || token == JsDocToken.EOL
          || token == JsDocToken.EOF) {
        restoreLookAhead(token);
        return newNode(Token.QMARK);
      }

      return wrapNode(Token.QMARK, parseBasicTypeExpression(token), lineno, charno);
    } else if (token == JsDocToken.BANG) {
      return wrapNode(Token.BANG, parseBasicTypeExpression(next()), lineno, charno);
    } else {
      Node basicTypeExpr = parseBasicTypeExpression(token);
      lineno = stream.getLineno();
      charno = stream.getCharno();
      if (basicTypeExpr != null) {
        if (match(JsDocToken.QMARK)) {
          next();
          return wrapNode(Token.QMARK, basicTypeExpr, lineno, charno);
        } else if (match(JsDocToken.BANG)) {
          next();
          return wrapNode(Token.BANG, basicTypeExpr, lineno, charno);
        }
      }

      return basicTypeExpr;
    }
  }
1054715
private Node parseTypeofType(JsDocToken token) {
    Node typeofType = newNode(Token.TYPEOF);
    Node name = parseNameExpression(token);
    skipEOLs();
    typeofType.addChildToFront(name);
    return typeofType;
  }
932876
function(context: ParserContext): ?Token {
    if (
      !util.IsFollowsRuleSatisfied(
        context.getTokenWalker(),
        [
          { token: "keyword", values: ["goto"] },
          { token: "operator", values: [":"], optional: true }
        ],
        true
      )
    )
      return null;

    let value = context.reader.read();
    while (!context.reader.isEOF() && !/[\W]/.test(context.reader.peek()))
      value += context.reader.read();

    return context.createToken("label", value);
  }
1086593
function getParseRegexForToken(token) {
        switch (token) {
        case 'DDDD':
            return parseTokenThreeDigits;
        case 'YYYY':
            return parseTokenFourDigits;
        case 'YYYYY':
            return parseTokenSixDigits;
        case 'S':
        case 'SS':
        case 'SSS':
        case 'DDD':
            return parseTokenOneToThreeDigits;
        case 'MMM':
        case 'MMMM':
        case 'dd':
        case 'ddd':
        case 'dddd':
        case 'a':
        case 'A':
            return parseTokenWord;
        case 'X':
            return parseTokenTimestampMs;
        case 'Z':
        case 'ZZ':
            return parseTokenTimezone;
        case 'T':
            return parseTokenT;
        case 'MM':
        case 'DD':
        case 'YY':
        case 'HH':
        case 'hh':
        case 'mm':
        case 'ss':
        case 'M':
        case 'D':
        case 'd':
        case 'H':
        case 'h':
        case 'm':
        case 's':
            return parseTokenOneOrTwoDigits;
        default :
            return new RegExp(token.replace('\\', ''));
        }
    }
919321
def parse_header(cls, value='*/*'):
        """"""
        yield from sorted((
            cls.from_string(token.strip())
            for token in value.split(',')
            if token.strip()
        ), reverse=True)
28749
def _readtoken(self, name, pos, length):
        """"""
        if length is not None and int(length) > self.length - pos:
            raise ReadError("Reading off the end of the data. "
                            "Tried to read {0} bits when only {1} available.".format(int(length), self.length - pos))
        try:
            val = name_to_read[name](self, length, pos)
            return val, pos + length
        except KeyError:
            if name == 'pad':
                return None, pos + length
            raise ValueError("Can't parse token {0}:{1}".format(name, length))
        except TypeError:
            # This is for the 'ue', 'se' and 'bool' tokens. They will also return the new pos.
            return name_to_read[name](self, pos)
751180
private Node parseBasicTypeExpression(JsDocToken token) {
    if (token == JsDocToken.STAR) {
      return newNode(Token.STAR);
    } else if (token == JsDocToken.LEFT_CURLY) {
      skipEOLs();
      return parseRecordType(next());
    } else if (token == JsDocToken.LEFT_PAREN) {
      skipEOLs();
      return parseUnionType(next());
    } else if (token == JsDocToken.STRING) {
      String string = stream.getString();
      switch (string) {
        case "function":
          skipEOLs();
          return parseFunctionType(next());
        case "null":
        case "undefined":
          return newStringNode(string);
        case "typeof":
          skipEOLs();
          return parseTypeofType(next());
        default:
          return parseTypeName(token);
      }
    }

    restoreLookAhead(token);
    return reportGenericTypeSyntaxWarning();
  }
60318
private void parseLiteral(String token) {
        Matcher m = Pattern.compile(String.format(ESCAPED_LITERAL_PATTERN_TEMPLATE, escapeCharacter)).matcher(token);
        while (m.find()) {
            cleanedToken = trimSpecialCharacters(m.group(1));
            if (tokenHasContent()) {
                mergedTokens.add(new LiteralToken(cleanedToken));
            }
        }
    }
737903
public static <T> Parser<T> tokenType(final Class<? extends T> type, final String name) {
    return token(new TokenMap<T>() {
      @Override public T map(Token token) {
        if (type.isInstance(token.value())) {
          return type.cast(token.value());
        }
        return null;
      }
      @Override public String toString() {
        return name;
      }
    });
  }
428050
def read_token(fd):
    
    out = ''
    # Skip initial whitespace.

    while 1:
        ch = fd.read(1)
        if ch.isspace(): continue
        elif len(ch) == 0: return out
        else: break

    while not ch.isspace():
        out = out + ch
        ch = fd.read(1)
        if len(ch) == 0: break

    return out
1019751
private void parseToken(String token) {
        int len = token.length();

        for (int i = 0; i < len; i++) {
            int chr = sr.read();
            if (chr == -1) {
                throw new IllegalArgumentException("EOF reached while reading token: " + token);
            }
            chr = Character.toLowerCase((char)chr);
            int loTokenChar = token.charAt(i);

            if (loTokenChar != chr) {
                throw new IllegalArgumentException("Expected token: " + token + " at position " + sr.getPosition());
            }
        }
    }
907090
def tranz_context(parser, token):
    
    tokens = token.split_contents()

    parameters = {}
    for idx, token in enumerate(tokens[1:], start=1):
        if "=" in token:
            if token[0:token.index('=')] not in ("domain", "prefix", "locale"):
                raise TemplateSyntaxError(
                    "Unexpected token {0} in tag {{tag_name}}".format(token)
                )

            k, v = token[0:token.index('=')], token[token.index('=') + 1:]
            parameters[k] = v
        else:
            raise TemplateSyntaxError(
                "Unexpected token {0} in tag {{tag_name}}".format(token))

    return TranzContextNode(
        parameters.get('prefix', None),
        parameters.get('domain', None),
        parameters.get('locale', None)
    )
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
293432
function readSimpleToken(type, endRegex, trimRight) {
		if (trimRight) {
			let matchTrim = exec(/\S/g)
			if (matchTrim) {
				advanceTo(matchTrim.index)
			}
		}
		let match = exec(endRegex)
		if (!match) {
			throwSyntaxError(`Unterminated ${type}`)
		}
		let start = getSourcePoint()
		advanceTo(match.index)
		let token = createContentToken(type, start)
		advanceTo(endRegex.lastIndex)
		return token
	}