Resumes scanning until the next regular expression is matched, the end of
input is encountered or an I/O-Error occurs.

@return the next token
@throws java.io.IOException if any I/O-Error occurs
5985
public JSONToken yylex() throws java.io.IOException, ParseException {
        int zzInput;
        int zzAction;

        // cached fields:
        int zzCurrentPosL;
        int zzMarkedPosL;
        int zzEndReadL = zzEndRead;
        char[] zzBufferL = zzBuffer;
        char[] zzCMapL = ZZ_CMAP;

        int[] zzTransL = ZZ_TRANS;
        int[] zzRowMapL = ZZ_ROWMAP;
        int[] zzAttrL = ZZ_ATTRIBUTE;

        while (true) {
            zzMarkedPosL = zzMarkedPos;

            zzAction = -1;

            zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

            zzState = ZZ_LEXSTATE[zzLexicalState];

            zzForAction:
            {
                while (true) {

                    if (zzCurrentPosL < zzEndReadL)
                        zzInput = zzBufferL[zzCurrentPosL++];
                    else if (zzAtEOF) {
                        zzInput = YYEOF;
                        break zzForAction;
                    } else {
                        // store back cached positions
                        zzCurrentPos = zzCurrentPosL;
                        zzMarkedPos = zzMarkedPosL;
                        boolean eof = zzRefill();
                        // get translated positions and possibly new buffer
                        zzCurrentPosL = zzCurrentPos;
                        zzMarkedPosL = zzMarkedPos;
                        zzBufferL = zzBuffer;
                        zzEndReadL = zzEndRead;
                        if (eof) {
                            zzInput = YYEOF;
                            break zzForAction;
                        } else {
                            zzInput = zzBufferL[zzCurrentPosL++];
                        }
                    }
                    int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
                    if (zzNext == -1)
                        break zzForAction;
                    zzState = zzNext;

                    int zzAttributes = zzAttrL[zzState];
                    if ((zzAttributes & 1) == 1) {
                        zzAction = zzState;
                        zzMarkedPosL = zzCurrentPosL;
                        if ((zzAttributes & 8) == 8)
                            break zzForAction;
                    }

                }
            }

            // store back cached position
            zzMarkedPos = zzMarkedPosL;

            switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
                case 25: {
                    return new JSONToken(JSONToken.NULL, null);
                }
                case 29:
                    break;
                case 13: {
                    sb.append(yytext());
                }
                case 30:
                    break;
                case 18: {
                    sb.append('\b');
                }
                case 31:
                    break;
                case 9: {
                    return new JSONToken(JSONToken.LSQUARE);
                }
                case 32:
                    break;
                case 2: {
                    Long val = Long.valueOf(yytext());
                    return new JSONToken(JSONToken.INT, val);
                }
                case 33:
                    break;
                case 16: {
                    sb.append('\\');
                }
                case 34:
                    break;
                case 8: {
                    return new JSONToken(JSONToken.RBRACE);
                }
                case 35:
                    break;
                case 26: {
                    return new JSONToken(JSONToken.BOOL, Boolean.TRUE);
                }
                case 36:
                    break;
                case 23: {
                    sb.append('\'');
                }
                case 37:
                    break;
                case 5: {
                    sb = new StringBuffer();
                    yybegin(STR2);
                }
                case 38:
                    break;
                case 27: {
                    return new JSONToken(JSONToken.BOOL, Boolean.FALSE);
                }
                case 39:
                    break;
                case 12: {
                    return new JSONToken(JSONToken.COLON);
                }
                case 40:
                    break;
                case 21: {
                    sb.append('\r');
                }
                case 41:
                    break;
                case 3: {
                    return new JSONToken(JSONToken.IDENT, yytext());
                }
                case 42:
                    break;
                case 28: {
                    try {
                        sb.append((char) Integer
                                .parseInt(yytext().substring(2), 16));
                    } catch (Exception e) {
                        throw new ParseException(e.getMessage(), 0);
                    }
                }
                case 43:
                    break;
                case 10: {
                    return new JSONToken(JSONToken.RSQUARE);
                }
                case 44:
                    break;
                case 17: {
                    sb.append('/');
                }
                case 45:
                    break;
                case 11: {
                    return new JSONToken(JSONToken.COMMA);
                }
                case 46:
                    break;
                case 15: {
                    sb.append('"');
                }
                case 47:
                    break;
                case 24: {
                    Double val = Double.valueOf(yytext());
                    return new JSONToken(JSONToken.FLOAT, val);
                }
                case 48:
                    break;
                case 1: {
                    throw new ParseException("Unexpected char [" + yytext() + "]",
                            0);
                }
                case 49:
                    break;
                case 19: {
                    sb.append('\f');
                }
                case 50:
                    break;
                case 7: {
                    return new JSONToken(JSONToken.LBRACE);
                }
                case 51:
                    break;
                case 14: {
                    yybegin(YYINITIAL);
                    return new JSONToken(JSONToken.STRING, sb.toString());
                }
                case 52:
                    break;
                case 22: {
                    sb.append('\t');
                }
                case 53:
                    break;
                case 4: {
                    sb = new StringBuffer();
                    yybegin(STR1);
                }
                case 54:
                    break;
                case 20: {
                    sb.append('\n');
                }
                case 55:
                    break;
                case 6: {
                }
                case 56:
                    break;
                default:
                    if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
                        zzAtEOF = true;
                        return null;
                    } else {
                        zzScanError(ZZ_NO_MATCH);
                    }
            }
        }
    }
1198592
public Token lookahead() {
        Token current = token;
        if (current.next == null) {
            current.next = token_source.getNextToken();
        }
        return current.next;
    }
124407
private final int nextFromTree()
        throws XMLStreamException
    {
        int i;

        // First, do we need to finish currently open token?
        if (mTokenState < mStTextThreshold) {
            // No need to update state... will get taken care of
            /* 03-Mar-2006, TSa: Let's add a sanity check here, temporarily,
             *   to ensure we never skip any textual content when it is
             *   to be validated
             */
            if (mVldContent == XMLValidator.CONTENT_ALLOW_VALIDATABLE_TEXT) {
                if (mCurrToken == CHARACTERS || mCurrToken == CDATA) { // should never happen
                    throwParseError("Internal error: skipping validatable text");
                }
            }
            i = skipToken();
            // note: skipToken() updates the start location
        } else {
            // Start/end elements are never unfinished (ie. are always
            // completely read in)
            if (mCurrToken == START_ELEMENT) {
                // Start tag may be an empty tag:
                if (mStEmptyElem) {
                    // and if so, we'll then get 'virtual' close tag:
                    mStEmptyElem = false;
                    // ... and location info is correct already
                    // 27-Feb-2009, TSa: but we do have to handle validation of the end tag now
                    int vld = mElementStack.validateEndElement();
                    mVldContent = vld;
                    mValidateText = (vld == XMLValidator.CONTENT_ALLOW_VALIDATABLE_TEXT);
                    return END_ELEMENT;
                }
            } else if (mCurrToken == END_ELEMENT) {
                // Close tag removes current element from stack
                if (!mElementStack.pop()) { // false if root closed
                    // if so, we'll get to epilog, unless in fragment mode
                    if (!mConfig.inputParsingModeFragment()) {
                        return closeContentTree();
                    }
                    // in fragment mode, fine, we'll just continue
                }
            } else if (mCurrToken == CDATA && mTokenState <= TOKEN_PARTIAL_SINGLE) {
                /* Just returned a partial CDATA... that's ok, just need to
                 * know we won't get opening marker etc.
                 * The tricky part here is just to ensure there's at least
                 * one character; if not, need to just discard the empty
                 * 'event' (note that it is possible to have an initial
                 * empty CDATA event for truly empty CDATA block; but not
                 * partial ones!). Let's just read it like a new
                 * CData section first:
                 */
                // First, need to update the start location...
                mTokenInputTotal = mCurrInputProcessed + mInputPtr;
                mTokenInputRow = mCurrInputRow;
                mTokenInputCol = mInputPtr - mCurrInputRowStart;
                char c = (mInputPtr < mInputEnd) ? mInputBuffer[mInputPtr++]
                    : getNextChar(SUFFIX_IN_CDATA);
                if (readCDataPrimary(c)) { // got it all!
                    // note: can not be in coalescing mode at this point;
                    // as we can never have partial cdata without unfinished token
                    // ... still need to have gotten at least 1 char though:
                    if (mTextBuffer.size() > 0) {
                        return CDATA;
                    }
                    // otherwise need to continue and parse the next event
                } else {
                    // Hmmh. Have to verify we get at least one char from
                    // CData section; if so, we are good to go for now;
                    // if not, need to get that damn char first:
                    if (mTextBuffer.size() == 0
                            && readCDataSecondary(mCfgLazyParsing
                                    ? 1 : mShortestTextSegment)) {
                        // Ok, all of it read
                        if (mTextBuffer.size() > 0) {
                            // And had some contents
                            mTokenState = TOKEN_FULL_SINGLE;
                            return CDATA;
                        }
                        // if nothing read, we'll just fall back (see below)
                    } else { // good enough!
                        mTokenState = TOKEN_PARTIAL_SINGLE;
                        return CDATA;
                    }
                }
                
                /* If we get here, it was the end of the section, without
                 * any more text inside CDATA, so let's just continue
                 */
            }
            // Once again, need to update the start location info:
            mTokenInputTotal = mCurrInputProcessed + mInputPtr;
            mTokenInputRow = mCurrInputRow;
            mTokenInputCol = mInputPtr - mCurrInputRowStart;
            i = getNext();
        }

        if (i < 0) {
            // 07-Oct-2005, TSa: May be ok in fragment mode (not otherwise),
            //  but we can just check if element stack has anything, as that handles all cases
            if (!mElementStack.isEmpty()) {
                throwUnexpectedEOF();
            }
            return handleEOF(false);
        }

        /* 26-Aug-2004, TSa: We have to deal with entities, usually, if
         *   they are the next thing; even in non-expanding mode there
         *   are entities and then there are entities... :-)
         *   Let's start with char entities; they can be expanded right away.
         */
        while (i == '&') {
            mWsStatus = ALL_WS_UNKNOWN;

            /* 30-Aug-2004, TSa: In some contexts entities are not
             *    allowed in any way, shape or form:
             */
            if (mVldContent == XMLValidator.CONTENT_ALLOW_NONE) {
                /* May be char entity, general entity; whatever it is it's
                 * invalid!
                 */
                reportInvalidContent(ENTITY_REFERENCE);
            }

            /* Need to call different methods based on whether we can do
             * automatic entity expansion or not:
             */
            int ch = mCfgReplaceEntities ?
                fullyResolveEntity(true) : resolveCharOnlyEntity(true);

            if (ch != 0) {
                /* Char-entity... need to initialize text output buffer, then;
                 * independent of whether it'll be needed or not.
                 */
                /* 30-Aug-2004, TSa: In some contexts only white space is
                 *   accepted...
                 */
                if (mVldContent <= XMLValidator.CONTENT_ALLOW_WS) {
                    // As per xml specs, only straight white space is legal
                    if (ch > CHAR_SPACE) {
                        /* 21-Sep-2008, TSa: Used to also require a call to
                         *   'mElementStack.reallyValidating', if only ws
                         *   allowed, to cover the case where non-typing-dtd
                         *   was only used to discover SPACE type. But
                         *   now that we have CONTENT_ALLOW_WS_NONSTRICT,
                         *   shouldn't be needed.
                         */
                        //if (mVldContent < XMLValidator.CONTENT_ALLOW_WS || mElementStack.reallyValidating()) {
                        reportInvalidContent(CHARACTERS);
                    }
                }
                TextBuffer tb = mTextBuffer;
                tb.resetInitialized();
                if (ch <= 0xFFFF) {
                    tb.append((char) ch);
                } else {
                    ch -= 0x10000;
                    tb.append((char) ((ch >> 10)  + 0xD800));
                    tb.append((char) ((ch & 0x3FF)  + 0xDC00));
                }
                mTokenState = TOKEN_STARTED;
                return CHARACTERS;
            }

            /* Nope; was a general entity... in auto-mode, it's now been
             * expanded; in non-auto, need to figure out entity itself.
             */
            if (!mCfgReplaceEntities|| mCfgTreatCharRefsAsEntities) {
                if (!mCfgTreatCharRefsAsEntities) {
                    final EntityDecl ed = resolveNonCharEntity();
                    // Note: ed may still be null at this point
                    mCurrEntity = ed;
                }
                // Note: ed may still be null at this point
                mTokenState = TOKEN_FULL_COALESCED;
                /*
                // let's not worry about non-parsed entities, since this is unexpanded mode
                // ... although it'd be an error either way? Should we report it?
                if (ed != null && !ed.isParsed()) {
                    throwParseError("Reference to unparsed entity '"+ed.getName()+"' from content not allowed.");
                }
                */
                return ENTITY_REFERENCE;
            }

            // Otherwise automatic expansion fine; just need the next char:
            i = getNextChar(SUFFIX_IN_DOC);
        }

        if (i == '<') { // Markup
            // And then it should be easy to figure out type:
            char c = getNextChar(SUFFIX_IN_ELEMENT);
            if (c == '?') { // proc. inst
                // 30-Aug-2004, TSa: Not legal for EMPTY elements
                if (mVldContent == XMLValidator.CONTENT_ALLOW_NONE) {
                    reportInvalidContent(PROCESSING_INSTRUCTION);
                }
                return readPIPrimary();
            }
            
            if (c == '!') { // CDATA or comment
                // Need to figure out bit more first...
                int type = nextFromTreeCommentOrCData();
                // 30-Aug-2004, TSa: Not legal for EMPTY elements
                if (mVldContent == XMLValidator.CONTENT_ALLOW_NONE) {
                    reportInvalidContent(type);
                }
                return type;
            }
            if (c == '/') { // always legal (if name matches etc)
                readEndElem();
                return END_ELEMENT;
            }

            if (c == ':' || isNameStartChar(c)) {
                /* Note: checking for EMPTY content type is done by the
                 * validator, no need to check here
                 */
                handleStartElem(c);
                return START_ELEMENT;
            }
            if (c == '[') {
                throwUnexpectedChar(c, " in content after '<' (malformed <![CDATA[]] directive?)");
            }
            throwUnexpectedChar(c, " in content after '<' (malformed start element?).");
        }

        /* Text... ok; better parse the 'easy' (consequtive) portions right
         * away, since that's practically free (still need to scan those
         * characters no matter what, even if skipping).
         */
        /* But first, do we expect to get ignorable white space (only happens
         * in validating mode)? If so, needs bit different handling:
         */
        if (mVldContent <= XMLValidator.CONTENT_ALLOW_WS_NONSTRICT) {
            if (mVldContent == XMLValidator.CONTENT_ALLOW_NONE) {
                if (mElementStack.reallyValidating()) {
                    reportInvalidContent(CHARACTERS);
                }
            }
            if (i <= CHAR_SPACE) {
                /* Note: need not worry about coalescing, since non-whitespace
                 * text is illegal (ie. can not have CDATA)
                 */
                mTokenState = (readSpacePrimary((char) i, false)) ?
                    TOKEN_FULL_COALESCED : TOKEN_STARTED;
                return SPACE;
            }
            // Problem if we are really validating; otherwise not
            if (mElementStack.reallyValidating()) {
                reportInvalidContent(CHARACTERS);
            }
            /* otherwise, we know it's supposed to contain just space (or
             * be empty), but as we are not validating it's not an error
             * for this not to be true. Type should be changed to
             * CHARACTERS tho.
             */
        }

        // Further, when coalescing, can not be sure if we REALLY got it all
        if (readTextPrimary((char) i)) { // reached following markup
            mTokenState = TOKEN_FULL_SINGLE;
        } else {
            // If not coalescing, this may be enough for current event
            if (!mCfgCoalesceText
                && mTextBuffer.size() >= mShortestTextSegment) {
                mTokenState = TOKEN_PARTIAL_SINGLE;
            } else {
                mTokenState = TOKEN_STARTED;
            }
        }
        return CHARACTERS;
    }
1071778
public Token expect(@Nonnull String expected, @Nonnull TokenValidator validator) throws IOException {
        if (!hasNext()) {
            throw eof("Expected %s, got end of file", expected);
        } else if (validator.validate(unreadToken)) {
            lastToken = unreadToken;
            unreadToken = null;
            return lastToken;
        }
        throw failure(unreadToken, "Expected %s, but got '%s'", expected, Strings.escape(unreadToken.asString()));
    }
238253
public String nextToken() 
  {
    int delPos;
    /* Scan for the next delimiter or end of data */
    for (delPos = position; delPos < dataLength; delPos++)
    {
      char c = data.charAt (delPos);
      if (delimiters.indexOf (c) != -1)
        break;
    }
    /* Find out what the token should be */
    String ret = data.substring (position, delPos);
    /* Set the position to the next position */
    position = delPos+1;
    return ret;
  }
995144
protected Token nextEval() throws ScanException {
		char c1 = input.charAt(position);
		char c2 = position < input.length()-1 ? input.charAt(position+1) : (char)0;

		switch (c1) {
			case '*': return fixed(Symbol.MUL);
			case '/': return fixed(Symbol.DIV);
			case '%': return fixed(Symbol.MOD);
			case '+': return fixed(Symbol.PLUS);
			case '-': return fixed(Symbol.MINUS);
			case '?': return fixed(Symbol.QUESTION);
			case ':': return fixed(Symbol.COLON);
			case '[': return fixed(Symbol.LBRACK);
			case ']': return fixed(Symbol.RBRACK);
			case '(': return fixed(Symbol.LPAREN);
			case ')': return fixed(Symbol.RPAREN);
			case ',': return fixed(Symbol.COMMA);
			case '.':
				if (!isDigit(c2)) {
					return fixed(Symbol.DOT);
				}
				break;
			case '=':
				if (c2 == '=') {
					return fixed(Symbol.EQ);
				}
				break;
			case '&':
				if (c2 == '&') {
					return fixed(Symbol.AND);
				}
				break;
			case '|':
				if (c2 == '|') {
					return fixed(Symbol.OR);
				}
				break;
			case '!':
				if (c2 == '=') {
					return fixed(Symbol.NE);
				}
				return fixed(Symbol.NOT);
			case '<':
				if (c2 == '=') {
					return fixed(Symbol.LE);
				}
				return fixed(Symbol.LT);
			case '>': 
				if (c2 == '=') {
					return fixed(Symbol.GE);
				}
				return fixed(Symbol.GT);
			case '"':
			case '\'': return nextString();
		}
		
		if (isDigit(c1) || c1 == '.') {
			return nextNumber();
		}
		
		if (Character.isJavaIdentifierStart(c1)) {
			int i = position+1;
			int l = input.length();
			while (i < l && Character.isJavaIdentifierPart(input.charAt(i))) {
				i++;
			}
			String name = input.substring(position, i);
			Token keyword = keyword(name);
			return keyword == null ? token(Symbol.IDENTIFIER, name, i - position) : keyword;
		}

		throw new ScanException(position, "invalid character '" + c1 + "'", "expression token");
	}
69555
public void nextToken() {
        byte ct[] = ctype;
        int c;
        int lctype;
        sval = null;
        isQuoted = false;

        do {
            c = read();
            if (c < 0) {
                return;
            }
            lctype = (c < 256) ? ct[c] : unicode2ctype(c);
        } while (lctype == CT_WHITESPACE);

        if (lctype == CT_ALPHA) {
            int i = 0;
            do {
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char) c;
                c = read();
                lctype = c < 0 ? CT_WHITESPACE : (c < 256)? ct[c] : unicode2ctype(c);
            } while (lctype == CT_ALPHA);
            if (c >= 0) --next; // push last back
            sval = String.copyValueOf(buf, 0, i);
            return;
        }

        if (lctype == CT_QUOTE) {
            int quote = c;
            int i = 0;
            /* Invariants (because \Octal needs a lookahead):
             *   (i)  c contains char value
             *   (ii) d contains the lookahead
             */
            int d = read();
            while (d >= 0 && d != quote) {
                if (d == '\\') {
                    c = read();
                    int first = c;   /* To allow \377, but not \477 */
                    if (c >= '0' && c <= '7') {
                        c = c - '0';
                        int c2 = read();
                        if ('0' <= c2 && c2 <= '7') {
                            c = (c << 3) + (c2 - '0');
                            c2 = read();
                            if ('0' <= c2 && c2 <= '7' && first <= '3') {
                                c = (c << 3) + (c2 - '0');
                                d = read();
                            } else
                                d = c2;
                        } else
                          d = c2;
                    } else {
                        switch (c) {
                        case 'a':
                            c = 0x7;
                            break;
                        case 'b':
                            c = '\b';
                            break;
                        case 'f':
                            c = 0xC;
                            break;
                        case 'n':
                            c = '\n';
                            break;
                        case 'r':
                            c = '\r';
                            break;
                        case 't':
                            c = '\t';
                            break;
                        case 'v':
                            c = 0xB;
                            break;
                        }
                        d = read();
                    }
                } else {
                    c = d;
                    d = read();
                }
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char)c;
            }

            if (d == quote) {
                isQuoted = true;
            }
            sval = String.copyValueOf(buf, 0, i);
        }
    }
90689
public int getNextToken() throws java.io.IOException {
  int zzInput;
  int zzAction;

  // cached fields:
  int zzCurrentPosL;
  int zzMarkedPosL;
  int zzEndReadL = zzEndRead;
  char [] zzBufferL = zzBuffer;
  char [] zzCMapL = ZZ_CMAP;

  int [] zzTransL = ZZ_TRANS;
  int [] zzRowMapL = ZZ_ROWMAP;
  int [] zzAttrL = ZZ_ATTRIBUTE;

  while (true) {
    zzMarkedPosL = zzMarkedPos;

    yychar+= zzMarkedPosL-zzStartRead;

    zzAction = -1;

    zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

    zzState = ZZ_LEXSTATE[zzLexicalState];


    zzForAction: {
      while (true) {
  
        if (zzCurrentPosL < zzEndReadL)
          zzInput = zzBufferL[zzCurrentPosL++];
        else if (zzAtEOF) {
          zzInput = YYEOF;
          break zzForAction;
        }
        else {
          // store back cached positions
          zzCurrentPos  = zzCurrentPosL;
          zzMarkedPos   = zzMarkedPosL;
          boolean eof = zzRefill();
          // get translated positions and possibly new buffer
          zzCurrentPosL  = zzCurrentPos;
          zzMarkedPosL   = zzMarkedPos;
          zzBufferL      = zzBuffer;
          zzEndReadL     = zzEndRead;
          if (eof) {
            zzInput = YYEOF;
            break zzForAction;
          }
          else {
            zzInput = zzBufferL[zzCurrentPosL++];
          }
        }
        int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
        if (zzNext == -1) break zzForAction;
        zzState = zzNext;

        int zzAttributes = zzAttrL[zzState];
        if ( (zzAttributes & 1) == 1 ) {
          zzAction = zzState;
          zzMarkedPosL = zzCurrentPosL;
          if ( (zzAttributes & 8) == 8 ) break zzForAction;
        }

      }
    }

    // store back cached position
    zzMarkedPos = zzMarkedPosL;

    switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
      case 7: 
        { return HOST;
        }
      case 15: break;
      case 6: 
        { return HASHTAG;
        }
      case 16: break;
      case 12: 
        { return ACRONYM;
        }
      case 17: break;
      case 11: 
        { return ACRONYM_DEP;
        }
      case 18: break;
      case 14: 
        { return URL;
        }
      case 19: break;
      case 1: 
        { /* ignore */
        }
      case 20: break;
      case 5: 
        { return MENTION;
        }
      case 21: break;
      case 4: 
        { return PERCENTAGE;
        }
      case 22: break;
      case 8: 
        { return NUM;
        }
      case 23: break;
      case 3: 
        { return CJ;
        }
      case 24: break;
      case 2: 
        { return ALPHANUM;
        }
      case 25: break;
      case 10: 
        { return COMPANY;
        }
      case 26: break;
      case 9: 
        { return APOSTROPHE;
        }
      case 27: break;
      case 13: 
        { return EMAIL;
        }
      case 28: break;
      default: 
        if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
          zzAtEOF = true;
          return YYEOF;
        } 
        else {
          zzScanError(ZZ_NO_MATCH);
        }
    }
  }
}
239630
protected final Token lookahead(int index) throws ScanException, ParseException {
		if (lookahead.isEmpty()) {
			lookahead = new LinkedList<LookaheadToken>();
		}
		while (index >= lookahead.size()) {
			lookahead.add(new LookaheadToken(scanner.next(), scanner.getPosition()));
		}
		return lookahead.get(index).token;
	}
1298913
@Override
  public int nextStartPosition() throws IOException {
    if (findMatches()) {
      currentMatch = queueMatches.get(0);
      queueMatches.remove(0);
      noMorePositions = false;
      return currentMatch.startPosition();
    } else {
      currentMatch = null;
      noMorePositions = true;
      return NO_MORE_POSITIONS;
    }
  }
405511
@SuppressWarnings("unchecked")
  @Override
  protected T getNext() {
    T token = null;
    if (lexer == null) {
      return token;
    }
    try {
      token = (T) lexer.next();
      while (token != null && token.word().equals(WhitespaceLexer.NEWLINE)) {
        if (eolIsSignificant) {
          return token;
        } else {
          token = (T) lexer.next();
        }
      }
    } catch (IOException e) {
      // do nothing, return null
    }
    return token;
  }
236555
@Nonnull
    protected Token nextSymbol() throws IOException {
        lastChar = 0;
        return new Token(buffer, bufferOffset, 1, lineNo, linePos);
    }
956991
private int next() throws IOException, UnterminatedCommentException {
		int c = get();
		if (c == '/') {
			switch (peek()) {
			case '/':
				for (;;) {
					c = get();
					if (c <= '\n') {
						break;
					}
				}
				break;
			case '*':
				get();
				while (c != ' ') {
					switch (get()) {
					case '*':
						if (peek() == '/') {
							get();
							c = ' ';
						}
						break;
					case EOF:
						throw new UnterminatedCommentException(currentByteIndex, line, column);
					}
				}
				break;
			}
		}
		theY = theX;
		theX = c;
		return c;
	}
531574
public String nextToken() 
    {
        if (m_tokenOffset_ < 0) {
            if (m_nextOffset_ < 0) {
                throw new NoSuchElementException("No more tokens in String");   
            }
            // pre-calculations of tokens not done
            if (m_returnDelimiters_) {
                int tokenlimit = 0;
                int c = UTF16.charAt(m_source_, m_nextOffset_);
                boolean contains = delims == null 
                    ? m_delimiters_.contains(c) 
                    : c < delims.length && delims[c];
                if (contains) {
                     if (m_coalesceDelimiters_) {
                        tokenlimit = getNextNonDelimiter(m_nextOffset_);
                     } else {
                        tokenlimit = m_nextOffset_ + UTF16.getCharCount(c);
                        if (tokenlimit == m_length_) {
                            tokenlimit = -1;
                        }
                     }
                }
                else {
                    tokenlimit = getNextDelimiter(m_nextOffset_);
                }
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                }
                m_nextOffset_ = tokenlimit;
                return result;
            }
            else {
                int tokenlimit = getNextDelimiter(m_nextOffset_);
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                    m_nextOffset_ = tokenlimit;
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                    m_nextOffset_ = getNextNonDelimiter(tokenlimit);
                }
                
                return result;
            }
        }
        // count was called before and we have all the tokens
        if (m_tokenOffset_ >= m_tokenSize_) {
            throw new NoSuchElementException("No more tokens in String");
        }
        String result;
        if (m_tokenLimit_[m_tokenOffset_] >= 0) {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_],
                                         m_tokenLimit_[m_tokenOffset_]);
        }
        else {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_]);
        }
        m_tokenOffset_ ++;
        m_nextOffset_ = -1;
        if (m_tokenOffset_ < m_tokenSize_) {
            m_nextOffset_ = m_tokenStart_[m_tokenOffset_];
        }
        return result;
    }
713001
public String nextVASPTokenFollowing(String string) throws IOException {
        int index;
        String line;
        while (inputBuffer.ready()) {
            line = inputBuffer.readLine();
            index = line.indexOf(string);
            if (index > 0) {
                index = index + string.length();
                line = line.substring(index);
                st = new StringTokenizer(line, " =\t");
                while (!st.hasMoreTokens() && inputBuffer.ready()) {
                    line = inputBuffer.readLine();
                    st = new StringTokenizer(line, " =\t");
                }
                if (st.hasMoreTokens()) {
                    fieldVal = st.nextToken();
                } else {
                    fieldVal = null;
                }
                break;
            }
        }
        return fieldVal;
    }
1329124
@Requires("nextToken == null")
  @Ensures("result == (nextToken != null)")
  protected boolean lex() throws IOException {
    int startOffset = currentOffset;
    StringBuilder buffer = new StringBuilder();
    int c = readChar(true);
    if (c == -1) {
      return false;
    }
    buffer.append((char) c);
    switch (c) {
      case '/':
        /* Comments. */
        c = readChar(true);
        switch (c) {
          case '/':
            buffer.append((char) c);
            do {
              c = readChar(false);
              buffer.append((char) c);
            } while (c != '\n');
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          case '*':
            buffer.append((char) c);
            for (;;) {
              c = readChar(false);
              buffer.append((char) c);
              if (c == '*') {
                c = readChar(false);
                buffer.append((char) c);
                if (c == '/') {
                  break;
                }
              }
            }
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          default:
            unreadChar(c);
            nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                  startOffset);
        }
        break;

      case '\'':
      case '"':
        /* Quoted strings. */
        int delim = c;
        for (;;) {
          c = readChar(false);
          buffer.append((char) c);
          if (c == delim) {
            break;
          }
          if (c == '\\') {
            buffer.append((char) readChar(false));
          }
        }
        nextToken = new Token(TokenKind.QUOTE, buffer.toString(), startOffset);
        break;

      default:
        if (Character.isJavaIdentifierStart(c)) {
          /* Identifiers. */
          while ((c = readChar(true)) != -1
                 && Character.isJavaIdentifierPart(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.WORD, buffer.toString(), startOffset);
        } else if (Character.isWhitespace(c)) {
          /* White space. */
          while ((c = readChar(true)) != -1 && Character.isWhitespace(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.SPACE, buffer.toString(),
                                startOffset);
        } else {
          /* Symbol. */
          nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                startOffset);
        }
    }
    return true;
  }
58487
public String nextValue() throws IOException {
        Token tkn = nextToken();
        String ret = null;
        switch (tkn.type) {
            case TT_TOKEN:
            case TT_EORECORD:
                ret = tkn.content.toString();
                break;
            case TT_EOF:
                ret = null;
                break;
            case TT_INVALID:
            default:
                // error no token available (or error)
                throw new IOException(
                        "(line " + getLineNumber() + ") invalid parse sequence");
            /// unreachable: break;
        }
        return ret;
    }