Gets next token, skipping empty lines.

@throws IOException if reading the next token fails
350091
private void getFirstToken(StreamTokenizer tokenizer) throws IOException {
        while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
        }

        if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
            tokenizer.ttype = StreamTokenizer.TT_WORD;
        } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
            tokenizer.ttype = '?';
        }
    }
768256
@Nullable
    public JsonToken next() throws IOException, JsonException {
        if (unreadToken != null) {
            JsonToken tmp = unreadToken;
            unreadToken = null;
            return tmp;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == JsonToken.kNewLine ||
                lastChar == JsonToken.kCarriageReturn ||
                lastChar == JsonToken.kSpace ||
                lastChar == JsonToken.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == JsonToken.kDoubleQuote) {
                return nextString();
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                return nextNumber();
            } else if (lastChar == '_' ||
                       (lastChar >= 'a' && lastChar <= 'z') ||
                       (lastChar >= 'A' && lastChar <= 'Z')) {
                return nextToken();
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // UTF-8 characters are only allowed inside JSON string literals.
                throw newParseException("Illegal character in JSON structure: '\\u%04x'", lastChar);
            } else {
                return nextSymbol();
            }
        }

        return null;
    }
1252402
public int read() throws IOException {
        synchronized (lock) {
            ensureOpen();
            for (;;) {
                if (nextChar >= nChars) {
                    fill();
                    if (nextChar >= nChars)
                        return -1;
                }
                if (skipLF) {
                    skipLF = false;
                    if (cb[nextChar] == '\n') {
                        nextChar++;
                        continue;
                    }
                }
                return cb[nextChar++];
            }
        }
    }
174990
private void skipToEndOfLine() throws IOException {
    while (pos < limit || fillBuffer(1)) {
      char c = buffer[pos++];
      if (c == '\n') {
        lineNumber++;
        lineStart = pos;
        break;
      } else if (c == '\r') {
        break;
      }
    }
  }
993511
@Override
    public int read()
        throws IOException
    {
        synchronized ( lock )
        {
            ensureOpen();
            if ( next >= length )
                return -1;
            return seq.charAt( next++ );
        }
    }
1200997
protected final boolean readLine() throws IOException {
		if (this.stream == null || this.overLimit) {
			return false;
		}

		int countInWrapBuffer = 0;

		// position of matching positions in the delimiter byte array
		int delimPos = 0;

		while (true) {
			if (this.readPos >= this.limit) {
				// readBuffer is completely consumed. Fill it again but keep partially read delimiter bytes.
				if (!fillBuffer(delimPos)) {
					int countInReadBuffer = delimPos;
					if (countInWrapBuffer + countInReadBuffer > 0) {
						// we have bytes left to emit
						if (countInReadBuffer > 0) {
							// we have bytes left in the readBuffer. Move them into the wrapBuffer
							if (this.wrapBuffer.length - countInWrapBuffer < countInReadBuffer) {
								// reallocate
								byte[] tmp = new byte[countInWrapBuffer + countInReadBuffer];
								System.arraycopy(this.wrapBuffer, 0, tmp, 0, countInWrapBuffer);
								this.wrapBuffer = tmp;
							}

							// copy readBuffer bytes to wrapBuffer
							System.arraycopy(this.readBuffer, 0, this.wrapBuffer, countInWrapBuffer, countInReadBuffer);
							countInWrapBuffer += countInReadBuffer;
						}

						this.offset += countInWrapBuffer;
						setResult(this.wrapBuffer, 0, countInWrapBuffer);
						return true;
					} else {
						return false;
					}
				}
			}

			int startPos = this.readPos - delimPos;
			int count;

			// Search for next occurrence of delimiter in read buffer.
			while (this.readPos < this.limit && delimPos < this.delimiter.length) {
				if ((this.readBuffer[this.readPos]) == this.delimiter[delimPos]) {
					// Found the expected delimiter character. Continue looking for the next character of delimiter.
					delimPos++;
				} else {
					// Delimiter does not match.
					// We have to reset the read position to the character after the first matching character
					//   and search for the whole delimiter again.
					readPos -= delimPos;
					delimPos = 0;
				}
				readPos++;
			}

			// check why we dropped out
			if (delimPos == this.delimiter.length) {
				// we found a delimiter
				int readBufferBytesRead = this.readPos - startPos;
				this.offset += countInWrapBuffer + readBufferBytesRead;
				count = readBufferBytesRead - this.delimiter.length;

				// copy to byte array
				if (countInWrapBuffer > 0) {
					// check wrap buffer size
					if (this.wrapBuffer.length < countInWrapBuffer + count) {
						final byte[] nb = new byte[countInWrapBuffer + count];
						System.arraycopy(this.wrapBuffer, 0, nb, 0, countInWrapBuffer);
						this.wrapBuffer = nb;
					}
					if (count >= 0) {
						System.arraycopy(this.readBuffer, 0, this.wrapBuffer, countInWrapBuffer, count);
					}
					setResult(this.wrapBuffer, 0, countInWrapBuffer + count);
					return true;
				} else {
					setResult(this.readBuffer, startPos, count);
					return true;
				}
			} else {
				// we reached the end of the readBuffer
				count = this.limit - startPos;
				
				// check against the maximum record length
				if (((long) countInWrapBuffer) + count > this.lineLengthLimit) {
					throw new IOException("The record length exceeded the maximum record length (" + 
							this.lineLengthLimit + ").");
				}

				// Compute number of bytes to move to wrapBuffer
				// Chars of partially read delimiter must remain in the readBuffer. We might need to go back.
				int bytesToMove = count - delimPos;
				// ensure wrapBuffer is large enough
				if (this.wrapBuffer.length - countInWrapBuffer < bytesToMove) {
					// reallocate
					byte[] tmp = new byte[Math.max(this.wrapBuffer.length * 2, countInWrapBuffer + bytesToMove)];
					System.arraycopy(this.wrapBuffer, 0, tmp, 0, countInWrapBuffer);
					this.wrapBuffer = tmp;
				}

				// copy readBuffer to wrapBuffer (except delimiter chars)
				System.arraycopy(this.readBuffer, startPos, this.wrapBuffer, countInWrapBuffer, bytesToMove);
				countInWrapBuffer += bytesToMove;
				// move delimiter chars to the beginning of the readBuffer
				System.arraycopy(this.readBuffer, this.readPos - delimPos, this.readBuffer, 0, delimPos);

			}
		}
	}
655334
public final String getNextLine() throws IOException {
    String str = null;
    if (buf_end - buf_pos <= 0) {
      if (fillBuffer() < 0) {
        return null;
      }
    }
    int lineend = -1;
    for (int i = buf_pos; i < buf_end; i++) {
      if (buffer[i] == '\n') {
        lineend = i;
        break;
      }
      // check for only '\r' as line end
      if ((i - buf_pos > 0) && buffer[i - 1] == '\r') {
        lineend = i - 1;
        break;
      }
    }
    if (lineend < 0) {
      StringBuilder input = new StringBuilder(256);
      int c;
      int lastC = 0;
      while (((c = read()) != -1) && (c != '\n') && (lastC != '\r')) {
        input.append((char) c);
        lastC = c;
      }
      if ((c == -1) && (input.length() == 0)) {
        return null;
      }
      return input.toString();
    }
    if (lineend > 0 && buffer[lineend] == '\n' && buffer[lineend - 1] == '\r') {
      str = new String(buffer, buf_pos, lineend - buf_pos - 1);
    } else {
      str = new String(buffer, buf_pos, lineend - buf_pos);
    }
    buf_pos = lineend + 1;
    return str;
  }
855882
@Nullable
  public ICommonsList <String> readNext () throws IOException
  {
    ICommonsList <String> ret = null;
    do
    {
      final String sNextLine = _getNextLine ();
      if (!m_bHasNext)
      {
        // should throw if still pending?
        return ret;
      }
      final ICommonsList <String> r = m_aParser.parseLineMulti (sNextLine);
      if (ret == null)
        ret = r;
      else
        ret.addAll (r);
    } while (m_aParser.isPending ());
    return ret;
  }
1366452
public Type readNextToken() throws IOException {
		int c;
		if (currentCharacter >= 0 && !Character.isWhitespace(currentCharacter)) {
			//there's still a character left from the last step
			c = currentCharacter;
			currentCharacter = -1;
		} else {
			//skip whitespace characters
			c = skipWhitespace();
		}
		if (c < 0) {
			return null;
		}
		
		//handle character
		if (c =='{') {
			currentTokenType = Type.START_OBJECT;
		} else if (c == '}') {
			currentTokenType = Type.END_OBJECT;
		} else if (c == '[') {
			currentTokenType = Type.START_ARRAY;
		} else if (c == ']') {
			currentTokenType = Type.END_ARRAY;
		} else if (c == ':') {
			currentTokenType = Type.COLON;
		} else if (c == ',') {
			currentTokenType = Type.COMMA;
		} else if (c == '"') {
			currentTokenType = Type.STRING;
		} else if (c == '-' || (c >= '0' && c<= '9')) {
			currentTokenType = Type.NUMBER;
			//the next token is a number. save the last character read because
			//readNumber() will need it.
			currentCharacter = c;
		} else if (c == 't') {
			int c2 = r.read();
			int c3 = r.read();
			int c4 = r.read();
			if (c2 == 'r' && c3 == 'u' & c4 == 'e') {
				currentTokenType = Type.TRUE;
			} else {
				currentTokenType = null;
			}
		} else if (c == 'f') {
			int c2 = r.read();
			int c3 = r.read();
			int c4 = r.read();
			int c5 = r.read();
			if (c2 == 'a' && c3 == 'l' & c4 == 's' && c5 == 'e') {
				currentTokenType = Type.FALSE;
			} else {
				currentTokenType = null;
			}
		} else if (c == 'n') {
			int c2 = r.read();
			int c3 = r.read();
			int c4 = r.read();
			if (c2 == 'u' && c3 == 'l' & c4 == 'l') {
				currentTokenType = Type.NULL;
			} else {
				currentTokenType = null;
			}
		} else {
			currentTokenType = null;
		}
		
		if (currentTokenType == null) {
			throw new IllegalStateException("Unrecognized token: " + (char)c);
		}
		
		return currentTokenType;
	}
10600
private void nextToken(StreamTokenizer tokenizer) throws IOException {
    tokenizer.nextToken();
    if((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
      tokenizer.ttype = StreamTokenizer.TT_WORD;
    }
    else if((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
      tokenizer.ttype = '?';
    }
    if(LOG.isDebugging()) {
      if(tokenizer.ttype == StreamTokenizer.TT_NUMBER) {
        LOG.debug("token: " + tokenizer.nval);
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_WORD) {
        LOG.debug("token: " + tokenizer.sval);
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_EOF) {
        LOG.debug("token: EOF");
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_EOL) {
        LOG.debug("token: EOL");
      }
      else {
        LOG.debug("token type: " + tokenizer.ttype);
      }
    }
  }
783108
@Override
  @SuppressWarnings("unchecked")
  protected T getNext() {
    // if (lexer == null) {
    //   return null;
    // }
    try {
      return (T) lexer.next();
    } catch (IOException e) {
      throw new RuntimeIOException(e);
    }
    // cdm 2007: this shouldn't be necessary: PTBLexer decides for itself whether to return CRs based on the same flag!
    // get rid of CRs if necessary
    // while (!tokenizeNLs && PTBLexer.cr.equals(((HasWord) token).word())) {
    //   token = (T)lexer.next();
    // }

    // horatio: we used to catch exceptions here, which led to broken
    // behavior and made it very difficult to debug whatever the
    // problem was.
  }
1015494
protected LexToken readToken() throws LexException
	{
		LexToken tok = reader.getLast();
		reader.nextToken();
		return tok;
	}
733943
public synchronized Token peekToken() throws IOException {
        if (mLookahead.empty()) {
            return mLookahead.push(scanToken());
        }
        else {
            return mLookahead.peek();
        }
    }
955808
private void skipEmptyLines() {
		int tmp = index;
		do {
			tmp = skipLineSpace(tmp);
			if (tmp < tokens.size() && tokens.get(tmp).kind != Token.Kind.NewLine) {
				return; // done
			} else if (tmp >= tokens.size()) {
				index = tmp;
				return; // end-of-file reached
			}
			// otherwise, skip newline and continue
			tmp = tmp + 1;
			index = tmp;
		} while (true);
		// deadcode
	}
285339
private Token scanText(int c) throws IOException {
        // Read first character in text so that source info does not include
        // tags.
        c = mSource.read();

        int startLine = mSource.getLineNumber();
        int startPos = mSource.getStartPosition();
        int endPos = mSource.getEndPosition();
        StringBuilder buf = new StringBuilder(256);

        while (c != -1) {
            if (c == SourceReader.ENTER_CODE) {
                if (mEmitSpecial) {
                    mLookahead.push(makeStringToken(Token.ENTER_CODE,
                                                    mSource.getBeginTag()));
                }
                break;
            }
            else if (c == SourceReader.ENTER_TEXT) {
                buf.append(mSource.getEndTag());
            }
            else {
                buf.append((char)c);
            }

            if (mSource.peek() < 0) {
                endPos = mSource.getEndPosition();
            }

            c = mSource.read();
        }

        if (c == -1) {
            // If the last token in the source file is text, trim all trailing
            // whitespace from it.

            int length = buf.length();
            
            int i;
            for (i = length - 1; i >= 0; i--) {
                if (buf.charAt(i) > ' ') {
                    break;
                }
            }

            buf.setLength(i + 1);
        }

        String str = buf.toString();
        return new StringToken(startLine, startPos, endPos,
                               Token.STRING, str);
    }
1167337
public T getNext() throws IOException {
    try {
      return mapper.readValue(jsonParser, clazz);
    } catch (EOFException e) {
      return null;
    }
  }
289351
public Token next() throws IOException {

        // skip whitespace
        while (Character.isWhitespace((char)lastChar))
        {
            readChar();
        }

        // handle punctuation
        switch (lastChar)
        {
            case -1:  readChar(); return Token.TokenEOF;
            case '{': readChar(); return Token.TokenBraceL;
            case '}': readChar(); return Token.TokenBraceR;
            case '[': readChar(); return Token.TokenBrackL;
            case ']': readChar(); return Token.TokenBrackR;
            case ':': readChar(); return Token.TokenColon;
            case ',': readChar(); return Token.TokenComma;

            case '"':
            case '\'':
                String stringValue = readString();
                return new Token(stringValue);

            case '-':
            case '.':
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                Number numberValue = readNumber();
                return new Token(numberValue);

            case 'n': 
            case 't':
            case 'f':
                String ident = readIdentifier();

                if (ident.equals("null"))  return Token.TokenNull;
                if (ident.equals("true"))  return Token.TokenTrue;
                if (ident.equals("false")) return Token.TokenFalse;

                throw new IOException("Unexpected identifier '" + ident + "' " + onLineCol());

            default:
                throw new IOException("Unexpected character '" + (char)lastChar + "' " + onLineCol());

        }

    }