Calculate CRC Checksum of the whole block. Implemented by concatenating
checksums of all the chunks.

@param datanode
@param ri
@param namespaceId
@param block
@return
@throws IOException
774137
static public int getBlockCrc(DataNode datanode, ReplicaToRead ri,
      int namespaceId, Block block) throws IOException {
    InputStream rawStreamIn = null;
    DataInputStream streamIn = null;
    try {
      int bytesPerCRC;
      int checksumSize;

      long crcPerBlock;

      rawStreamIn = BlockWithChecksumFileReader.getMetaDataInputStream(
          datanode.data, namespaceId, block);
      streamIn = new DataInputStream(new BufferedInputStream(rawStreamIn,
          FSConstants.BUFFER_SIZE));

      final BlockMetadataHeader header = BlockMetadataHeader
          .readHeader(streamIn);
      final DataChecksum checksum = header.getChecksum();
      if (checksum.getChecksumType() != DataChecksum.CHECKSUM_CRC32) {
        throw new IOException("File Checksum now is only supported for CRC32");
      }

      bytesPerCRC = checksum.getBytesPerChecksum();
      checksumSize = checksum.getChecksumSize();
      crcPerBlock = (((BlockWithChecksumFileReader.MetaDataInputStream) rawStreamIn)
          .getLength() - BlockMetadataHeader.getHeaderSize()) / checksumSize;

      int blockCrc = 0;
      byte[] buffer = new byte[checksumSize];
      for (int i = 0; i < crcPerBlock; i++) {
        IOUtils.readFully(streamIn, buffer, 0, buffer.length);
        int intChecksum = ((buffer[0] & 0xff) << 24)
            | ((buffer[1] & 0xff) << 16) | ((buffer[2] & 0xff) << 8)
            | ((buffer[3] & 0xff));

        if (i == 0) {
          blockCrc = intChecksum;
        } else {
          int chunkLength;
          if (i != crcPerBlock - 1 || ri.getBytesVisible() % bytesPerCRC == 0) {
            chunkLength = bytesPerCRC;
          } else {
            chunkLength = (int) ri.getBytesVisible() % bytesPerCRC;
          }
          blockCrc = CrcConcat.concatCrc(blockCrc, intChecksum, chunkLength);
        }
      }
      return blockCrc;
    } finally {
      if (streamIn != null) {
        IOUtils.closeStream(streamIn);
      }
      if (rawStreamIn != null) {
        IOUtils.closeStream(rawStreamIn);
      }
    }
  }
482191
BlocksWithLocations getBlocks(DatanodeID datanode, long size)
    throws IOException {
    readLock();
    try {
      checkSuperuserPrivilege();

      DatanodeDescriptor node = getDatanode(datanode);
      if (node == null) {
        NameNode.stateChangeLog.warn("BLOCK* NameSystem.getBlocks: "
          + "Asking for blocks from an unrecorded node " + datanode.getName());
        throw new IllegalArgumentException(
          "Unexpected exception.  Got getBlocks message for datanode " +
            datanode.getName() + ", but there is no info for it");
      }

      int numBlocks = node.numBlocks();
      if (numBlocks == 0) {
        return new BlocksWithLocations(new BlockWithLocations[0]);
      }
      Iterator<BlockInfo> iter = node.getBlockIterator();
      int startBlock = r.nextInt(numBlocks); // starting from a random block
      // skip blocks
      for (int i = 0; i < startBlock; i++) {
        iter.next();
      }
      List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
      long totalSize = 0;
      while (totalSize < size && iter.hasNext()) {
        totalSize += addBlock(iter.next(), results);
      }
      if (totalSize < size) {
        iter = node.getBlockIterator(); // start from the beginning
        for (int i = 0; i < startBlock && totalSize < size; i++) {
          totalSize += addBlock(iter.next(), results);
        }
      }

      return new BlocksWithLocations(
        results.toArray(new BlockWithLocations[results.size()]));
    } finally {
      readUnlock();
    }
  }
744750
protected final void verifyWriteCData()
        throws XMLStreamException
    {
        // Not legal outside main element tree:
        if (mCheckStructure) {
            if (inPrologOrEpilog()) {
                reportNwfStructure(ErrorConsts.WERR_PROLOG_CDATA);
            }
        }
        // 08-Dec-2005, TSa: validator-based validation?
        if (mVldContent <= XMLValidator.CONTENT_ALLOW_WS) {
            // there's no ignorable white space CDATA...
            reportInvalidContent(CDATA);
        }
    }
171535
public File getBlockFile(int namespaceId, Block b) throws IOException {
    File f = validateBlockFile(namespaceId, b);
    if (f == null) {
      if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
        InterDatanodeProtocol.LOG
            .debug("b=" + b + ", volumeMap=" + volumeMap);
      }
      throw new IOException("Block " + b + ", namespace= " + namespaceId
          + " is not valid.");
    }
    return f;
  }
779257
private void processBlock(List<GenericCriteria> list, byte[] block)
   {
      if (block != null)
      {
         if (MPPUtility.getShort(block, 0) > 0x3E6)
         {
            addCriteria(list, block);
         }
         else
         {
            switch (block[0])
            {
               case (byte) 0x0B:
               {
                  processBlock(list, getChildBlock(block));
                  break;
               }

               case (byte) 0x06:
               {
                  processBlock(list, getListNextBlock(block));
                  break;
               }

               case (byte) 0xED: // EQUALS
               {
                  addCriteria(list, block);
                  break;
               }

               case (byte) 0x19: // AND
               case (byte) 0x1B:
               {
                  addBlock(list, block, TestOperator.AND);
                  break;
               }

               case (byte) 0x1A: // OR
               case (byte) 0x1C:
               {
                  addBlock(list, block, TestOperator.OR);
                  break;
               }
            }
         }
      }
   }
897838
public static BlockReaderLocalBase newBlockReader(
      Configuration conf,
      String file,
      int namespaceid,
      Block blk,
      DatanodeInfo node,
      long startOffset,
      long length,
      DFSClientMetrics metrics,
      boolean verifyChecksum,
      boolean clearOsBuffer,
      boolean positionalReadMode) throws IOException {

    LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node);

    BlockPathInfo pathinfo = localDatanodeInfo.getOrComputePathInfo(namespaceid,
        blk, node, conf);
    
    // Another alternative is for datanode to pass whether it is an inline checksum
    // file and checksum metadata through BlockPathInfo, which is a cleaner approach.
    // However, we need to worry more about protocol compatible issue. We avoid this
    // trouble for now. We can always change to the other approach later.
    //
    boolean isInlineChecksum = Block.isInlineChecksumBlockFilename(new Path(
        pathinfo.getBlockPath()).getName());

    // check to see if the file exists. It may so happen that the
    // HDFS file has been deleted and this block-lookup is occuring
    // on behalf of a new HDFS file. This time, the block file could
    // be residing in a different portion of the fs.data.dir directory.
    // In this case, we remove this entry from the cache. The next
    // call to this method will repopulate the cache.
    try {

      // get a local file system
      FileChannel dataFileChannel;
      FileDescriptor dataFileDescriptor;
      File blkfile = new File(pathinfo.getBlockPath());
      FileInputStream fis = new FileInputStream(blkfile);
      dataFileChannel = fis.getChannel();
      dataFileDescriptor = fis.getFD();
      
      if (LOG.isDebugEnabled()) {
        LOG.debug("New BlockReaderLocal for file " +
            pathinfo.getBlockPath() + " of size " + blkfile.length() +
                  " startOffset " + startOffset +
                  " length " + length);
      }
      
      DataChecksum checksum = null;
      if (isInlineChecksum) {
        GenStampAndChecksum gac = BlockInlineChecksumReader
            .getGenStampAndChecksumFromInlineChecksumFile(new Path(pathinfo
                .getBlockPath()).getName());
        checksum = DataChecksum.newDataChecksum(gac.getChecksumType(),
            gac.getBytesPerChecksum());
        
        if (verifyChecksum) {

          return new BlockReaderLocalInlineChecksum(conf, file, blk,
              startOffset, length, pathinfo, metrics, checksum, verifyChecksum,
              dataFileChannel, dataFileDescriptor, clearOsBuffer,
              positionalReadMode);
        }
        else {
          return new BlockReaderLocalInlineChecksum(conf, file, blk,
              startOffset, length, pathinfo, metrics, checksum,
              dataFileChannel, dataFileDescriptor, clearOsBuffer,
              positionalReadMode);
        }
      } else if (verifyChecksum) {
        FileChannel checksumInChannel = null;
        // get the metadata file
        File metafile = new File(pathinfo.getMetaPath());
        FileInputStream checksumIn = new FileInputStream(metafile);
        checksumInChannel = checksumIn.getChannel();
        // read and handle the common header here. For now just a version
        BlockMetadataHeader header = BlockMetadataHeader.readHeader(
            new DataInputStream(checksumIn), new NativeCrc32());
        short version = header.getVersion();

        if (version != FSDataset.FORMAT_VERSION_NON_INLINECHECKSUM) {
          LOG.warn("Wrong version (" + version + ") for metadata file for "
              + blk + " ignoring ...");
        }
        checksum = header.getChecksum();

        return new BlockReaderLocalWithChecksum(conf, file, blk, startOffset,
            length, pathinfo, metrics, checksum, verifyChecksum,
            dataFileChannel, dataFileDescriptor, checksumInChannel,
            clearOsBuffer, positionalReadMode);
      }
      else {
        return new BlockReaderLocalWithChecksum(conf, file, blk, startOffset,
            length, pathinfo, metrics, dataFileChannel, dataFileDescriptor,
            clearOsBuffer, positionalReadMode);
      }

    } catch (FileNotFoundException e) {
      localDatanodeInfo.removeBlockLocalPathInfo(namespaceid, blk);
      DFSClient.LOG.warn("BlockReaderLoca: Removing " + blk +
          " from cache because local file " +
          pathinfo.getBlockPath() +
          " could not be opened.");
      throw e;
    }
  }
57768
public void sendBlocksBeingWrittenReport(DatanodeProtocol node,
      int namespaceId, DatanodeRegistration nsRegistration) throws IOException {
    Block[] blocks = data.getBlocksBeingWrittenReport(namespaceId);
    if (blocks != null && blocks.length != 0) {
      long[] blocksAsLong =
        BlockListAsLongs.convertToArrayLongs(blocks);
      BlockReport bbwReport = new BlockReport(blocksAsLong);
      node.blocksBeingWrittenReport(nsRegistration, bbwReport);
    }
  }
397945
ActiveFile getOngoingCreates(int namespaceId, Block block) {
    checkBlock(block);
    NamespaceMap nm = getNamespaceMap(namespaceId);
    if (nm == null) {
      return null;
    }
    return nm.getOngoingCreates(block);
  }
1194603
private static int getBlockIdInFile(DistributedFileSystem srcFs, 
      Path srcPath, long blockId) throws IOException {
    FileStatus srcStat = srcFs.getFileStatus(srcPath);
    LocatedBlocks lbs =
        srcFs.getClient().getLocatedBlocks(srcPath.toUri().getPath(), 0, 
            srcStat.getLen());
    
    int i = 0;
    LOG.info("Look for block " + blockId + " in file " + srcPath);
    for (LocatedBlock lb: lbs.getLocatedBlocks()) {
      if (lb.getBlock().getBlockId() == blockId) {
        return i;
      }
      i++;
    }
    
    return -1;
  }
583747
function(inStream, block, length, crc) {
  var pos = 0;
  var lastChar = -1;
  var runLength = 0;
  while (pos < length) {
    if (runLength===4) {
      block[pos++] = 0;
      if (pos >= length) { break; }
    }
    var ch = inStream.readByte();
    if (ch === EOF) {
      break;
    }
    crc.updateCRC(ch);
    if (ch !== lastChar) {
      lastChar = ch;
      runLength = 1;
    } else {
      runLength++;
      if (runLength > 4) {
        if (runLength < 256) {
          block[pos-1]++;
          continue;
        } else {
          runLength = 1;
        }
      }
    }
    block[pos++] = ch;
  }
  return pos;
}
71465
@Override
    boolean checkNodes(final INodeReadTrx paramFirstRtx, final INodeReadTrx paramSecondRtx)
        throws TTIOException {
        assert paramFirstRtx != null;
        assert paramSecondRtx != null;

        boolean found = false;

        if (paramFirstRtx.getNode().getDataKey() == paramSecondRtx.getNode().getDataKey()
            && paramFirstRtx.getNode().equals(paramSecondRtx.getNode())) {
            final long nodeKey = paramFirstRtx.getNode().getDataKey();

            if (paramFirstRtx.getNode().getKind() == IConstants.ELEMENT) {
                if (((ElementNode)paramFirstRtx.getNode()).getNamespaceCount() == 0
                    && ((ElementNode)paramFirstRtx.getNode()).getAttributeCount() == 0
                    && ((ElementNode)paramSecondRtx.getNode()).getAttributeCount() == 0
                    && ((ElementNode)paramSecondRtx.getNode()).getNamespaceCount() == 0) {
                    found = true;
                } else {
                    if (((ElementNode)paramFirstRtx.getNode()).getNamespaceCount() == 0) {
                        found = true;
                    } else {
                        for (int i = 0; i < ((ElementNode)paramFirstRtx.getNode()).getNamespaceCount(); i++) {
                            paramFirstRtx.moveToNamespace(i);
                            for (int j = 0; j < ((ElementNode)paramSecondRtx.getNode()).getNamespaceCount(); j++) {
                                paramSecondRtx.moveToNamespace(i);

                                if (paramFirstRtx.getNode().equals(paramSecondRtx.getNode())) {
                                    found = true;
                                    break;
                                }
                            }
                            paramFirstRtx.moveTo(nodeKey);
                            paramSecondRtx.moveTo(nodeKey);
                        }
                    }

                    if (found) {
                        for (int i = 0; i < ((ElementNode)paramFirstRtx.getNode()).getAttributeCount(); i++) {
                            paramFirstRtx.moveToAttribute(i);
                            for (int j = 0; j < ((ElementNode)paramSecondRtx.getNode()).getAttributeCount(); j++) {
                                paramSecondRtx.moveToAttribute(i);

                                if (paramFirstRtx.getNode().equals(paramSecondRtx.getNode())) {
                                    found = true;
                                    break;
                                }
                            }
                            paramFirstRtx.moveTo(nodeKey);
                            paramSecondRtx.moveTo(nodeKey);
                        }
                    }
                }
            } else {
                found = true;
            }
        }

        return found;
    }
1004358
synchronized void verifiedByClient(int namespaceId, Block block) {
    DataBlockScanner nsScanner = getNSScanner(namespaceId);
    if (nsScanner != null) {
      nsScanner.updateScanStatusUpdateOnly(block,
          DataBlockScanner.ScanType.REMOTE_READ, true);
    } else {
      LOG.warn("No namespace scanner found for namespaceId: " + nsScanner);
    }
  }
916352
int getFileCrc(String src) throws IOException {
    checkOpen();
    return getFileCrc(dataTransferVersion,
        src, namenode, namenodeProtocolProxy, socketFactory, socketTimeout);
  }
588260
public LocatedBlock recoverBlock(int namespaceId, Block block,
      boolean keepLength, DatanodeID[] datanodeids, boolean closeFile,
      long deadline) throws IOException {
    int errorCount = 0;

    // Number of "replicasBeingWritten" in 0.21 parlance - these are replicas
    // on DNs that are still alive from when the write was happening
    int rbwCount = 0;
    // Number of "replicasWaitingRecovery" in 0.21 parlance - these replicas
    // have survived a DN restart, and thus might be truncated (eg if the
    // DN died because of a machine power failure, and when the ext3 journal
    // replayed, it truncated the file
    int rwrCount = 0;

    List<BlockRecord> blockRecords = new ArrayList<BlockRecord>();
    List<InterDatanodeProtocol> datanodeProxies = new ArrayList<InterDatanodeProtocol>();
    // check generation stamps
    for (DatanodeID id : datanodeids) {
      try {
        InterDatanodeProtocol datanode;
        if (dnr!= null && dnr.equals(id)) {
          LOG.info("Skipping IDNPP creation for local id " + id
              + " when recovering " + block);
          datanode = dn;
        } else {
          LOG.info("Creating IDNPP for non-local id " + id + " (dnReg="
              + dnr + ") when recovering "
              + block);
          datanode = DataNode.createInterDataNodeProtocolProxy(id, conf,
              socketTimeout);
          datanodeProxies.add(datanode);
        }
        BlockSyncer.throwIfAfterTime(deadline);
        BlockRecoveryInfo info = datanode
            .startBlockRecovery(namespaceId, block);
        if (info == null) {
          LOG.info("No block metadata found for block " + block
              + " on datanode " + id);
          continue;
        }
        if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
          LOG.info("Only old generation stamp "
              + info.getBlock().getGenerationStamp() + " found on datanode "
              + id + " (needed block=" + block + ")");
          continue;
        }
        blockRecords.add(new BlockRecord(id, datanode, info));

        if (info.wasRecoveredOnStartup()) {
          rwrCount++;
        } else {
          rbwCount++;
        }
      } catch (BlockRecoveryTimeoutException e) {
        throw e;
      } catch (IOException e) {
        ++errorCount;
        InterDatanodeProtocol.LOG.warn(
            "Failed to getBlockMetaDataInfo for block (=" + block
                + ") from datanode (=" + id + ")", e);
      }
    }

    // If we *only* have replicas from post-DN-restart, then we should
    // include them in determining length. Otherwise they might cause us
    // to truncate too short.
    boolean shouldRecoverRwrs = (rbwCount == 0);

    List<BlockRecord> syncList = new ArrayList<BlockRecord>();
    long minlength = Long.MAX_VALUE;

    for (BlockRecord record : blockRecords) {
      BlockRecoveryInfo info = record.info;
      assert (info != null && info.getBlock().getGenerationStamp() >= block
          .getGenerationStamp());
      if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
        LOG.info("Not recovering replica " + record
            + " since it was recovered on "
            + "startup and we have better replicas");
        continue;
      }
      if (keepLength) {
        if (info.getBlock().getNumBytes() == block.getNumBytes()) {
          syncList.add(record);
        }
      } else {
        syncList.add(record);
        if (info.getBlock().getNumBytes() < minlength) {
          minlength = info.getBlock().getNumBytes();
        }
      }
    }

    if (syncList.isEmpty() && errorCount > 0) {
      DataNode.stopAllProxies(datanodeProxies);
      throw new IOException("All datanodes failed: block=" + block
          + ", datanodeids=" + Arrays.asList(datanodeids));
    }
    if (!keepLength) {
      block.setNumBytes(minlength);
    }
    return blockSyncer.syncBlock(block, syncList, closeFile, datanodeProxies,
        deadline);
  }
1362160
File validateBlockFile(int namespaceId, Block b) throws IOException {
    return getValidateBlockFile(namespaceId, b, false);
  }
330871
boolean deleteInternal(String src, INode[] inodes, boolean recursive,
                         boolean enforcePermission) throws IOException {
    ArrayList<BlockInfo> collectedBlocks = new ArrayList<BlockInfo>();
    INode targetNode = null;

    byte[][] components = inodes == null ?
      INodeDirectory.getPathComponents(src) : null;

    writeLock();
    try {
      if (NameNode.stateChangeLog.isDebugEnabled()) {
        NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src);
      }
      if (isInSafeMode()) {
        throw new SafeModeException("Cannot delete " + src, safeMode);
      }
      if (inodes == null) {
        inodes = new INode[components.length];
        dir.rootDir.getExistingPathINodes(components, inodes);
      }
      if (enforcePermission && isPermissionEnabled
          && isPermissionCheckingEnabled(inodes)) {
        checkPermission(src, inodes, false, null, FsAction.WRITE, null, FsAction.ALL);
      }
      if (neverDeletePaths.contains(src)) {
        NameNode.stateChangeLog.warn("DIR* NameSystem.delete: " +
          " Trying to delete a whitelisted path " + src +
          " by user " + getCurrentUGI() +
          " from server " + Server.getRemoteIp());
        throw new IOException("Deleting a whitelisted directory is not allowed. " + src);
      }

      if ((!recursive) && (!dir.isDirEmpty(inodes[inodes.length-1]))) {
        throw new IOException(src + " is non empty");
      }

      targetNode = dir.delete(src, inodes, collectedBlocks,
          BLOCK_DELETION_INCREMENT);
      if (targetNode == null) {
        return false;
      }
    } finally {
      writeUnlock();
    }
    List<INode> removedINodes = new ArrayList<INode>();
    while (targetNode.name != null) {
      // Interatively Remove blocks
      collectedBlocks.clear();
      // sleep to make sure that the lock can be grabbed by another thread
      try {
        Thread.sleep(1);
      } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
      }
      writeLock();
      try {
        int filesRemoved = targetNode.collectSubtreeBlocksAndClear(
            collectedBlocks, BLOCK_DELETION_INCREMENT, removedINodes);
        incrDeletedFileCount(this, filesRemoved);
        removeBlocks(collectedBlocks);
        // remove from inodeMap
        dir.removeFromInodeMap(removedINodes);
      } finally {
        writeUnlock();
      }   
    }
       
    collectedBlocks.clear();
    removedINodes.clear();
    
    return true;
  }
281786
static void checkRc(final int rc) throws LmdbNativeException {
    switch (rc) {
      case MDB_SUCCESS:
        return;
      case Dbi.BadDbiException.MDB_BAD_DBI:
        throw new Dbi.BadDbiException();
      case BadReaderLockException.MDB_BAD_RSLOT:
        throw new BadReaderLockException();
      case BadException.MDB_BAD_TXN:
        throw new BadException();
      case Dbi.BadValueSizeException.MDB_BAD_VALSIZE:
        throw new Dbi.BadValueSizeException();
      case LmdbNativeException.PageCorruptedException.MDB_CORRUPTED:
        throw new LmdbNativeException.PageCorruptedException();
      case Cursor.FullException.MDB_CURSOR_FULL:
        throw new Cursor.FullException();
      case Dbi.DbFullException.MDB_DBS_FULL:
        throw new Dbi.DbFullException();
      case Dbi.IncompatibleException.MDB_INCOMPATIBLE:
        throw new Dbi.IncompatibleException();
      case Env.FileInvalidException.MDB_INVALID:
        throw new Env.FileInvalidException();
      case Dbi.KeyExistsException.MDB_KEYEXIST:
        throw new Dbi.KeyExistsException();
      case Env.MapFullException.MDB_MAP_FULL:
        throw new Env.MapFullException();
      case Dbi.MapResizedException.MDB_MAP_RESIZED:
        throw new Dbi.MapResizedException();
      case Dbi.KeyNotFoundException.MDB_NOTFOUND:
        throw new Dbi.KeyNotFoundException();
      case LmdbNativeException.PageFullException.MDB_PAGE_FULL:
        throw new LmdbNativeException.PageFullException();
      case LmdbNativeException.PageNotFoundException.MDB_PAGE_NOTFOUND:
        throw new LmdbNativeException.PageNotFoundException();
      case LmdbNativeException.PanicException.MDB_PANIC:
        throw new LmdbNativeException.PanicException();
      case Env.ReadersFullException.MDB_READERS_FULL:
        throw new Env.ReadersFullException();
      case LmdbNativeException.TlsFullException.MDB_TLS_FULL:
        throw new LmdbNativeException.TlsFullException();
      case TxFullException.MDB_TXN_FULL:
        throw new TxFullException();
      case Env.VersionMismatchException.MDB_VERSION_MISMATCH:
        throw new Env.VersionMismatchException();
      default:
        break;
    }

    final Constant constant = CONSTANTS.getConstant(rc);
    if (constant == null) {
      throw new IllegalArgumentException("Unknown result code " + rc);
    }
    final String msg = constant.name() + " " + constant.toString();
    throw new LmdbNativeException.ConstantDerviedException(rc, msg);
  }