/* get replication factor of a block
168688
int getReplication(Block block) {
    BlockInfo storedBlock = blocksMap.getBlockInfo(block);
    INodeFile fileINode = (storedBlock == null) ? null : storedBlock.getINode();
    if (fileINode == null) { // block does not belong to any file
      return 0;
    }
    assert !fileINode.isDirectory() : "Block cannot belong to a directory.";
    return fileINode.getBlockReplication(storedBlock);
  }
1082489
private void adjustReplicationFactorForURI(HttpPut httpPut) throws URISyntaxException{
        String queryString = httpPut.getURI().getQuery();
        if(!StringUtils.isEmpty(queryString) && queryString.contains("op=CREATE") && (queryString.contains("replication=") || !StringUtils.isEmpty(m_blockReplication))){
            rateLimitedLogWarn(m_logger, "Set block replication factor in the target system.");
            if(!StringUtils.isEmpty(m_blockReplication) && !queryString.contains("replication=")){
                StringBuilder builder = new StringBuilder(128);
                builder.append(queryString).append("&replication=").append(m_blockReplication);
                URI oldUri = httpPut.getURI();
                URI newUri = new URI(oldUri.getScheme(), oldUri.getAuthority(),oldUri.getPath(), builder.toString(), oldUri.getFragment());
                httpPut.setURI(newUri);
            }
        }
    }
534352
function getBlock() {
  const type = NodeType.BlockType;
  const token = tokens[pos];
  const line = token.ln;
  const column = token.col;
  const end = tokens[pos].right;
  let content = [];

  // Skip `{`.
  pos++;

  while (pos < end) {
    if (checkBlockdecl(pos)) content = content.concat(getBlockdecl());
    else throwError(pos);
  }

  const end_ = getLastPosition(content, line, column, 1);
  pos = end + 1;

  return newNode(type, content, line, column, end_);
}
462961
@Override
  public Block getPenultimateBlock() {
    if (blocks == null || blocks.length <= 1) {
      return null;
    }
    return blocks[blocks.length - 2];
  }
241228
private boolean cacheBlockFromUfs(long blockId, long blockSize,
      Protocol.OpenUfsBlockOptions openUfsBlockOptions) {
    // Check if the block has been requested in UFS block store
    try {
      if (!mBlockWorker
          .openUfsBlock(Sessions.ASYNC_CACHE_SESSION_ID, blockId, openUfsBlockOptions)) {
        LOG.warn("Failed to async cache block {} from UFS on opening the block", blockId);
        return false;
      }
    } catch (BlockAlreadyExistsException e) {
      // It is already cached
      return true;
    }
    try (BlockReader reader = mBlockWorker
        .readUfsBlock(Sessions.ASYNC_CACHE_SESSION_ID, blockId, 0)) {
      // Read the entire block, caching to block store will be handled internally in UFS block store
      // Note that, we read from UFS with a smaller buffer to avoid high pressure on heap
      // memory when concurrent async requests are received and thus trigger GC.
      long offset = 0;
      while (offset < blockSize) {
        long bufferSize = Math.min(8 * Constants.MB, blockSize - offset);
        reader.read(offset, bufferSize);
        offset += bufferSize;
      }
    } catch (AlluxioException | IOException e) {
      // This is only best effort
      LOG.warn("Failed to async cache block {} from UFS on copying the block: {}", blockId, e);
      return false;
    } finally {
      try {
        mBlockWorker.closeUfsBlock(Sessions.ASYNC_CACHE_SESSION_ID, blockId);
      } catch (AlluxioException | IOException ee) {
        LOG.warn("Failed to close UFS block {}: {}", blockId, ee);
        return false;
      }
    }
    return true;
  }
466710
static long getGenerationStampFromInlineChecksumFile(String blockName)
      throws IOException {
    String[] vals = StringUtils.split(blockName, '_');
    if (vals.length != 6) {
      // blk, blkid, genstamp, version, checksumtype, byte per checksum
      throw new IOException("unidentified block name format: " + blockName);
    }
    return Long.parseLong(vals[2]);
  }
1129695
public Long retrieveChecksum(Block lostBlock, Path p,
      long lostBlockOffset, FileSystem fs, Context context) 
          throws IOException {
    Long oldCRC = null;  
    if (checksumStore != null) { 
      IOException caughtException = null;
      try {
        oldCRC = checksumStore.getChecksum(lostBlock); 
      } catch (IOException ioe) {
        LOG.error(" Fail to get checksum for block " + lostBlock, ioe);
        caughtException = ioe;
      }
      // Checksum is not record, we should report
      if (oldCRC == null) {
        LogUtils.logRaidReconstructionMetrics(LOGRESULTS.FAILURE, 0,
            codec, p, lostBlockOffset,
            LOGTYPES.OFFLINE_RECONSTRUCTION_GET_CHECKSUM,
            fs, caughtException, context);
      }
    }
    return oldCRC;
  }
758535
private void removeStoredBlock(Block block, DatanodeDescriptor node) {
    if (NameNode.stateChangeLog.isDebugEnabled()) {
      NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: "
        + block + " from " + node.getName());
    }
    if (!blocksMap.removeNode(block, node)) {
      if (NameNode.stateChangeLog.isDebugEnabled()) {
        NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeStoredBlock: "
          + block + " has already been removed from node " + node);
      }
      return;
    }

    //
    // if file is being actively written to and it is the last block, 
    // then do not check replication-factor here.
    //
    BlockInfo storedBlock = blocksMap.getStoredBlock(block);
    INodeFile fileINode = (storedBlock == null) ? null : storedBlock.getINode();
    if (fileINode != null && 
        fileINode.isUnderConstruction() && fileINode.isLastBlock(storedBlock)) {
      decrementSafeBlockCount(block);
      return;
    }      

    //
    // It's possible that the block was removed because of a datanode
    // failure.  If the block is still valid, check if replication is
    // necessary.  In that case, put block on a possibly-will-
    // be-replicated list.
    //
    if (fileINode != null) {
      decrementSafeBlockCount(block);
      // handle under replication
      // Use storedBlock here because block maybe a deleted block with size DELETED
      if (isPopulatingReplQueuesInternal()) {
        NumberReplicas num = countNodes(storedBlock);
        int numCurrentReplicas = num.liveReplicas()
            +
        pendingReplications.getNumReplicas(storedBlock);
        updateNeededReplicationQueue(storedBlock, -1, numCurrentReplicas,
            num.decommissionedReplicas, node, 
            fileINode.getBlockReplication(storedBlock));
      }
    }

    //
    // We've removed a block from a node, so it's definitely no longer
    // in "excess" there.
    //
    removeFromExcessReplicateMap(block, node);

    // Remove the replica from corruptReplicas
    corruptReplicas.removeFromCorruptReplicasMap(block, node);
  }
1033574
private boolean checkBlockSize(Block block, INodeFile inode) {
    if (block.getNumBytes() < 0) {
      return false;
    }
    BlockInfo[] blocks = inode.getBlocks();
    if (blocks.length == 0) {
      return false;
    }
    return block.getNumBytes() <= inode.getPreferredBlockSize();
  }
1074497
private void processOverReplicatedBlocksAsync() {

    // blocks should not be scheduled for deletion during safemode
    if (isInSafeMode()) {
      return;
    }
    
    if (delayOverreplicationMonitorTime > now()) {
      LOG.info("Overreplication monitor delayed for "
          + ((delayOverreplicationMonitorTime - now()) / 1000) + " seconds");
      return;
    }
    nameNode.clearOutstandingNodes();
    
    final int nodes = heartbeats.size();
    List<Block> blocksToProcess = new ArrayList<Block>(Math.min(
        overReplicatedBlocks.size(),
        ReplicationConfigKeys.overreplicationWorkMultiplier * nodes));
    
    for (int i = 0; i < ReplicationConfigKeys.overreplicationWorkMultiplier; i++) {
      writeLock();
      try {
        NameNode.getNameNodeMetrics().numOverReplicatedBlocks.set(overReplicatedBlocks.size());
        overReplicatedBlocks.pollNToList(nodes, blocksToProcess);
        if (overReplicatedBlocks.isEmpty()) {
          break;
        }
      } finally {
        writeUnlock();
      }
    }

    for (Block block : blocksToProcess) {
      if (NameNode.stateChangeLog.isDebugEnabled()) {
        NameNode.stateChangeLog
            .debug("BLOCK* NameSystem.processOverReplicatedBlocksAsync: " + block);
      }
      if (block instanceof OverReplicatedBlock) {
        OverReplicatedBlock opb = (OverReplicatedBlock)block;
        processOverReplicatedBlock(block, (short) -1, opb.addedNode, opb.delNodeHint);
      } else {
        processOverReplicatedBlock(block, (short) -1, null, null);
      }
        
    }
  }
461113
private synchronized void readBlockSizeInfo() throws IOException {
    if (!transferBlockSize) {
      return;
    }
    blkLenInfoUpdated = true;
    isBlockFinalized = in.readBoolean();
    updatedBlockLength = in.readLong();
    if (dataTransferVersion >= DataTransferProtocol.READ_PROFILING_VERSION) {
      readDataNodeProfilingData();
    }
    
    if (LOG.isDebugEnabled()) {
      LOG.debug("ifBlockComplete? " + isBlockFinalized + " block size: "
          + updatedBlockLength);
    }      
  }
232122
DatanodeBlockInfo addBlockInfo(Block block,
      DatanodeBlockInfo replicaInfo) {
    return getBlockBucket(block).addBlockInfo(block, replicaInfo);
  }
826082
public S getRepresentative(Block<S, L> block) {
        return block.getStates().choose().getOriginalState();
    }
733636
private DatanodeDescriptor[] chooseTarget(ReplicationWork work) {
    if (!neededReplications.contains(work.block)) {
      return null;
    }
    if (work.blockSize == BlockFlags.NO_ACK) {
      LOG.warn("Block " + work.block.getBlockId() + 
          " of the file " + getFullPathName(work.fileINode) + 
          " is invalidated and cannot be replicated.");
      return null;
    }
    if (work.blockSize == BlockFlags.DELETED) {
      LOG.warn("Block " + work.block.getBlockId() + 
          " of the file " + getFullPathName(work.fileINode) + 
          " is a deleted block and cannot be replicated.");
      return null;
    }
    return replicator.chooseTarget(work.fileINode,
        work.numOfReplicas, work.srcNode,
        work.containingNodes, null, work.blockSize);
  }
262059
def getblock(self, blockhash, decode=False):
        """"""

        if not decode:
            decode = "false"

            return self.req("getblock", [blockhash, decode])

        else:
            return self.req("getblock", [blockhash])
1200429
LocatedBlocksWithMetaInfo updateDatanodeInfo(LocatedBlocks locatedBlocks)
      throws IOException {
    if (locatedBlocks.getLocatedBlocks().size() == 0)
      return new LocatedBlocksWithMetaInfo(locatedBlocks.getFileLength(),
          locatedBlocks.getLocatedBlocks(), false,
          DataTransferProtocol.DATA_TRANSFER_VERSION, getNamespaceId(),
          this.nameNode.getClientProtocolMethodsFingerprint());
    List<LocatedBlock> newBlocks = new ArrayList<LocatedBlock>();

    readLock();
    try {
      for (LocatedBlock locBlock: locatedBlocks.getLocatedBlocks()) {
        Block block = locBlock.getBlock();
        int numNodes = blocksMap.numNodes(block);
        int numCorruptNodes = countNodes(block).corruptReplicas();
        int numCorruptReplicas = corruptReplicas.numCorruptReplicas(block); 

        if (numCorruptNodes != numCorruptReplicas) {
          LOG.warn("Inconsistent number of corrupt replicas for " + 
                   block + "blockMap has " + numCorruptNodes + 
                   " but corrupt replicas map has " + numCorruptReplicas);
        }

        boolean blockCorrupt = numCorruptNodes == numNodes;
        int numMachineSet = blockCorrupt ? numNodes : (numNodes - numCorruptNodes);
        DatanodeDescriptor[] machineSet = new DatanodeDescriptor[numMachineSet];

        if (numMachineSet > 0) {
          numNodes = 0;
          for(Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); 
              it.hasNext();) {
            DatanodeDescriptor dn = it.next();
            boolean replicaCorrupt = corruptReplicas.isReplicaCorrupt(block, dn);
            if (blockCorrupt || (!blockCorrupt && !replicaCorrupt))
              machineSet[numNodes++] = dn;
          }
        }

        // We need to make a copy of the block object before releasing the lock
        // to prevent the state of block is changed after that and before the
        // object is serialized to clients, to avoid potential inconsistency.
        // Further optimization is possible to avoid some object copy. Since it
        // is so far not a critical path. We leave a safe approach here.
        //
        Block blockCopy  = null;
        if (block != null) {
          blockCopy = new Block(block);
        }
        LocatedBlock newBlock = new LocatedBlock(blockCopy, machineSet, 0,
            blockCorrupt);
  newBlocks.add(newBlock);
      }
    } finally {
      readUnlock();
    }

    return new LocatedBlocksWithMetaInfo(locatedBlocks.getFileLength(),
        newBlocks, false, DataTransferProtocol.DATA_TRANSFER_VERSION,
        getNamespaceId(), this.nameNode.getClientProtocolMethodsFingerprint());
  }
714063
static public int getBlockCrc(DataNode datanode, ReplicaToRead ri,
      int namespaceId, Block block) throws IOException {

    InputStream rawStreamIn = null;
    DataInputStream streamIn = null;
    int blockCrc = 0;

    try {
      int bytesPerCRC;
      int checksumSize;

      bytesPerCRC = ri.getBytesPerChecksum();
      int checksumType = ri.getChecksumType();
      if (checksumType != DataChecksum.CHECKSUM_CRC32) {
        throw new IOException("File Checksum now is only supported for CRC32");
      }
      DataChecksum dataChecksum = DataChecksum.newDataChecksum(checksumType,
          bytesPerCRC);
      checksumSize = dataChecksum.getChecksumSize();
      
      rawStreamIn = ri.getBlockInputStream(datanode, 0);
      streamIn = new DataInputStream(new BufferedInputStream(rawStreamIn,
          FSConstants.BUFFER_SIZE));
      IOUtils.skipFully(streamIn, BlockInlineChecksumReader.getHeaderSize());

      long lengthLeft = ((FileInputStream) rawStreamIn).getChannel().size()
          - BlockInlineChecksumReader.getHeaderSize();
      if (lengthLeft == 0) {
        blockCrc = (int) dataChecksum.getValue();
      } else {
        byte[] buffer = new byte[checksumSize];
        boolean firstChecksum = true;

        while (lengthLeft > 0) {
          long dataByteLengh;

          if (lengthLeft >= bytesPerCRC + checksumSize) {
            lengthLeft -= bytesPerCRC + checksumSize;
            dataByteLengh = bytesPerCRC;
          } else if (lengthLeft > checksumSize) {
            dataByteLengh = lengthLeft - checksumSize;
            lengthLeft = 0;
          } else {
            // report to name node the corruption.
            DataBlockScanner.reportBadBlocks(block, namespaceId, datanode);
            throw new IOException("File for namespace " + namespaceId
                + " block " + block + " seems to be corrupted");
          }

          IOUtils.skipFully(streamIn, dataByteLengh);
          IOUtils.readFully(streamIn, buffer, 0, buffer.length);
          int intChecksum = DataChecksum.getIntFromBytes(buffer, 0);
          if (firstChecksum) {
            blockCrc = intChecksum;
            firstChecksum = false;
          } else {
            blockCrc = CrcConcat.concatCrc(blockCrc, intChecksum,
                (int) dataByteLengh);
          }
        }
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
            + ", crc=" + blockCrc);
      }
      return blockCrc;
    } finally {
      IOUtils.closeStream(streamIn);
      IOUtils.closeStream(rawStreamIn);
    }

  }