Checks if the ingest is catching up.
If the ingest is consuming finalized segment, it's assumed to be behind.
Otherwise, catching up is based on the position of the input stream.
@throws IOException
347766
private void setCatchingUp() throws IOException {
    try {
      if (inputEditStream != null && inputEditStream.isInProgress()) {
        catchingUp = (inputEditStream.length() - inputEditStream.getPosition() > catchUpLag);
      } else {
        catchingUp = true;
      }
    } catch (Exception e) {
      catchingUp = true;
    }
  }
138145
private boolean readNextChunkFromFile() throws IOException {
        if (readOffset >= filledUpToWhenStarted || readOffset < 0) {
            return false;
        }

        bufferedReader.clear();
        dataFile.read(readOffset, bufferedReader);
        bufferedReader.position(0);
        if (bufferedReader.limit() > 0) {
            readOffset += bufferedReader.limit();
            while (bufferedReader.remaining() > 0) {
                byte[] dst = new byte[elementSize];
                bufferedReader.get(dst);
                pendingElements.add(dst);
            }
        }
        return true;
    }
124226
private boolean setUpNextChunk() throws IOException {
        byte[] chunkData = new byte[DEFAULT_CHUNK_SIZE];
        int chunkSizeInBytes = 0;
        while (chunkSizeInBytes < DEFAULT_CHUNK_SIZE) {
            /** Read from the buffer of the decoded stream */
            if (null != decodedStreamBuffer
                    && decodedStreamBuffer.hasNext()) {
                chunkData[chunkSizeInBytes++] = decodedStreamBuffer.next();
            }
            /** Read from the wrapped stream */
            else {
                int bytesToRead = DEFAULT_CHUNK_SIZE - chunkSizeInBytes;
                int count = is.read(chunkData, chunkSizeInBytes, bytesToRead);
                if (count != -1) {
                    if (null != decodedStreamBuffer)
                        decodedStreamBuffer.buffer(chunkData, chunkSizeInBytes, count);
                    chunkSizeInBytes += count;
                }
                else
                    break;
            }
        }
        if (chunkSizeInBytes == 0){
            byte[] signedFinalChunk = createSignedChunk(FINAL_CHUNK);
            currentChunkIterator = new ChunkContentIterator(signedFinalChunk);
            return true;
        }
        else {
            if (chunkSizeInBytes < chunkData.length) {
                chunkData = Arrays.copyOf(chunkData, chunkSizeInBytes);
            }
            byte[] signedChunkContent = createSignedChunk(chunkData);
            currentChunkIterator = new ChunkContentIterator(signedChunkContent);
            return false;
        }
    }
514198
@SuppressWarnings("unchecked")
  private void readNext() {
    // Compute the index of the next entry in the segment.
    final long index = getNextIndex();

    try {
      // Read more bytes from the segment if necessary.
      if (memory.remaining() < maxEntrySize) {
        long position = channel.position() + memory.position();
        channel.position(position);
        memory.clear();
        channel.read(memory);
        channel.position(position);
        memory.flip();
      }

      // Mark the buffer so it can be reset if necessary.
      memory.mark();

      try {
        // Read the length of the entry.
        final int length = memory.getInt();

        // If the buffer length is zero then return.
        if (length <= 0 || length > maxEntrySize) {
          memory.reset().limit(memory.position());
          nextEntry = null;
          return;
        }

        // Read the checksum of the entry.
        long checksum = memory.getInt() & 0xFFFFFFFFL;

        // Compute the checksum for the entry bytes.
        final Checksum crc32 = new CRC32();
        crc32.update(memory.array(), memory.position(), length);

        // If the stored checksum equals the computed checksum, return the entry.
        if (checksum == crc32.getValue()) {
          int limit = memory.limit();
          memory.limit(memory.position() + length);
          E entry = namespace.deserialize(memory);
          memory.limit(limit);
          nextEntry = new Indexed<>(index, entry, length);
        } else {
          memory.reset().limit(memory.position());
          nextEntry = null;
        }
      } catch (BufferUnderflowException e) {
        memory.reset().limit(memory.position());
        nextEntry = null;
      }
    } catch (IOException e) {
      throw new StorageException(e);
    }
  }
833478
synchronized void processFinished() {
        final InternalState required = this.requiredState;
        final InternalState state = this.internalState;
        // If the server was not stopped
        if(required == InternalState.STOPPED && state == InternalState.PROCESS_STOPPING) {
            finishTransition(InternalState.PROCESS_STOPPING, InternalState.PROCESS_STOPPED);
        } else {
            this.requiredState = InternalState.STOPPED;
            if ( !(internalSetState(getTransitionTask(InternalState.PROCESS_STOPPING), internalState, InternalState.PROCESS_STOPPING)
                    && internalSetState(getTransitionTask(InternalState.PROCESS_REMOVING), internalState, InternalState.PROCESS_REMOVING)
                    && internalSetState(getTransitionTask(InternalState.STOPPED), internalState, InternalState.STOPPED)) ){
                this.requiredState = InternalState.FAILED;
                internalSetState(null, internalState, InternalState.PROCESS_STOPPED);
            }
        }
    }
1316222
private boolean fillLast() throws IOException {
        int count;

        if (lastFilled == pageSize) {
            throw new IllegalStateException();
        }
        count = src.read(pages[lastNo], lastFilled, pageSize - lastFilled);
        if (count <= 0) {
            if (count == 0) {
                throw new IllegalStateException();
            }
            return false;
        }
        lastFilled += count;
        return true;
    }
769670
private boolean readIntoPushBack() throws IOException {
        // File finished?
        boolean eof = false;
        // Next char.
        final int ch = in.read();
        if (ch >= 0) {
            // Discard whitespace at start?
            if (!(pulled == 0 && isWhiteSpace(ch))) {
                // Good code.
                pulled += 1;
                // Parse out the &stuff;
                if (ch == '&') {
                    // Process the &
                    readAmpersand();
                } else {
                    // Not an '&', just append.
                    pushBack.append((char) ch);
                }
            }
        } else {
            // Hit end of file.
            eof = true;
        }
        return eof;
    }
232513
public boolean nextBoolean() throws IOException {
    int p = peeked;
    if (p == PEEKED_NONE) {
      p = doPeek();
    }
    if (p == PEEKED_TRUE) {
      peeked = PEEKED_NONE;
      pathIndices[stackSize - 1]++;
      return true;
    } else if (p == PEEKED_FALSE) {
      peeked = PEEKED_NONE;
      pathIndices[stackSize - 1]++;
      return false;
    }
    throw new IllegalStateException("Expected a boolean but was " + peek() + locationString());
  }
538473
private boolean checkMultiReadBuffer() throws IOException {
        //first check existing buffer
        if (null != this.buffer) {
            if (this.buffer.hasRemaining()) {
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "checkMultiReadBuffer, remaining ->" + this);
                }
                return true;
            }
            if (firstReadCompleteforMulti) { // multiRead enabled and subsequent read
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "checkMultiReadBuffer, buffer is completely read, ready this buffer for the subsequent read ->" + this);
                }
                postDataBuffer.get(postDataIndex).flip(); // make position 0 , to read it again from start
                postDataIndex++;
            } else {
                this.buffer.release();
            }
            this.buffer = null;
        }
        // no buffer read from store or first read of buffer
        if (firstReadCompleteforMulti) {
            // first read from what we have stored, if need more than go to channel
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                Tr.debug(tc, "checkMultiReadBuffer ,index ->" + postDataIndex + " ,storage.size ->" + postDataBuffer.size());
            }
            if (postDataBuffer.size() <= postDataIndex) {
                //get remaining from channel now as read needs more than the stored
                readRemainingFromChannel();
            }
            if (postDataBuffer.size() > postDataIndex) {
                this.buffer = postDataBuffer.get(postDataIndex);
            }

            if (null != this.buffer) {
                // record the new amount of data read from the store
                this.bytesReadFromStore += this.buffer.remaining();
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "checkMultiReadBuffer ->" + this);
                }
                return true;
            }
        } else { // multiRead enabled and first read
            if (getBufferFromChannel()) {
                // store the channel buffer
                postDataBuffer.add(postDataIndex, this.buffer.duplicate());
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "checkMultiReadBuffer, buffer ->" + postDataBuffer.get(postDataIndex)
                                 + " ,buffersize ->" + postDataBuffer.size() + " ,index ->" + postDataIndex);
                }
                postDataIndex++;

                // record the new amount of data read from the channel
                this.bytesRead += this.buffer.remaining();
                return true;
            }
        }

        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
            Tr.debug(tc, "checkMultiReadBuffer:  no more buffer ->" + this);
        }
        return false;
    }
398055
public void consumeUntil(Parser recognizer, IntervalSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) {
        CSSToken t;
        boolean finish;
        TokenStream input = recognizer.getInputStream();
        do {
            Token next = input.LT(1);
            if (next instanceof CSSToken) {
                t = (CSSToken) input.LT(1);
                if (t.getType() == Token.EOF) {
                    logger.trace("token eof ");
                    break;
                }
            } else
                break; /* not a CSSToken, probably EOF */
            // consume token if does not match
            finish = (t.getLexerState().isBalanced(mode, ls, t) && follow.contains(t.getType()));
            if (!finish) {
                logger.trace("Skipped: {}", t);
                input.consume();
            }
        } while (!finish);
    }
1095481
@Override
    public boolean isFinished() {
        
      //Return -1 here because if we are closed or finished, we're at the end of the stream
        if(isClosed){
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                Tr.debug(tc, "Stream is closed so it is finished");
            return true;
        }
        //Call into the HttpInboundServiceContext to find out if the headers and body
        //have been fully read by the HTTP Channel, then return the result
        boolean isFinished = this.httpin.isFinished();
        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
            Tr.debug(tc, "isFinished returning : " + isFinished);
        }
        return isFinished;
    }
1014644
private boolean next() throws IOException {
    if (valuesRead >= endOfPageValueCount) {
      if (valuesRead >= totalValueCount) {
        // How do we get here? Throw end of stream exception?
        return false;
      }
      readPage();
    }
    ++valuesRead;
    // TODO: Don't read for flat schemas
    //repetitionLevel = repetitionLevelColumn.nextInt();
    return definitionLevelColumn.nextInt() == maxDefLevel;
  }
547771
private void complete() throws IOException
    {
    if( source_ != null)
      {
      try
        {
        thread_.join();
        }
      catch( Exception e)
        {
        logger_.error( "Thread=" + thread_ + " not completed", e);
        }

      IOException failure =
        failure_ == null
        ? null
        : new IOException( "Can't write filter output", failure_);
      
      failure_      = null;
      filterInput_  = null;
      source_       = null;
      target_       = null;
      thread_       = null;

      if( failure != null)
        {
        throw failure;
        }
      }
    }
607334
@Override
    public void waitForFinish() throws IOException
    {
        if (future == null)
        {
            throw new IllegalStateException("not started");
        }
        try
        {
            future.get();
        }
        catch (InterruptedException | ExecutionException ex)
        {
            throw new IOException(ex);
        }
    }
932479
private int completeRead(int b) throws IOException, StreamIntegrityException {
    if (b == END_OF_STREAM) {
      handleEndOfStream();
    } else {
      // Have we reached the end of the stream?
      int c = pushbackInputStream.read();
      if (c == END_OF_STREAM) {
        handleEndOfStream();
      } else {
        pushbackInputStream.unread(c);
      }
    }

    return b;
  }
95354
public void endObject() throws IOException {
    int p = peeked;
    if (p == PEEKED_NONE) {
      p = doPeek();
    }
    if (p == PEEKED_END_OBJECT) {
      stackSize--;
      pathNames[stackSize] = null; // Free the last path name so that it can be garbage collected!
      pathIndices[stackSize - 1]++;
      peeked = PEEKED_NONE;
    } else {
      throw new IllegalStateException("Expected END_OBJECT but was " + peek()
          + " at line " + getLineNumber() + " column " + getColumnNumber() + " path " + getPath());
    }
  }
1366133
public boolean checkComplete() throws IOException {
     JobID jobID = runningJob.getID();
     if (runningJob.isComplete()) {
       // delete job directory
       final String jobdir = jobconf.get(JOB_DIR_LABEL);
       if (jobdir != null) {
         final Path jobpath = new Path(jobdir);
         jobpath.getFileSystem(jobconf).delete(jobpath, true);
       }
       if (runningJob.isSuccessful()) {
         LOG.info("Job Complete(Succeeded): " + jobID);
       } else {
         LOG.info("Job Complete(Failed): " + jobID);
       }
       raidPolicyPathPairList.clear();
       Counters ctrs = runningJob.getCounters();
       if (ctrs != null) {
         RaidNodeMetrics metrics = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID);
         if (ctrs.findCounter(Counter.FILES_FAILED) != null) {
           long filesFailed = ctrs.findCounter(Counter.FILES_FAILED).getValue();
           metrics.raidFailures.inc(filesFailed);
         }
         long slotSeconds = ctrs.findCounter(
          JobInProgress.Counter.SLOTS_MILLIS_MAPS).getValue() / 1000;
         metrics.raidSlotSeconds.inc(slotSeconds);
       }
       return true;
     } else {
       String report =  (" job " + jobID +
         " map " + StringUtils.formatPercent(runningJob.mapProgress(), 0)+
         " reduce " + StringUtils.formatPercent(runningJob.reduceProgress(), 0));
       if (!report.equals(lastReport)) {
         LOG.info(report);
         lastReport = report;
       }
       TaskCompletionEvent[] events =
         runningJob.getTaskCompletionEvents(jobEventCounter);
       jobEventCounter += events.length;
       for(TaskCompletionEvent event : events) {
         if (event.getTaskStatus() ==  TaskCompletionEvent.Status.FAILED) {
           LOG.info(" Job " + jobID + " " + event.toString());
         }
       }
       return false;
     }
   }