Populate bytes per checksum and checksum size information from
local checksum object.

@param blockLength
511625
protected void getChecksumInfo(long blockLength) {
    /*
     * If bytesPerChecksum is very large, then the metadata file is mostly
     * corrupted. For now just truncate bytesPerchecksum to blockLength.
     */
    bytesPerChecksum = checksum.getBytesPerChecksum();
    if (bytesPerChecksum > 10 * 1024 * 1024 && bytesPerChecksum > blockLength) {
      checksum = DataChecksum.newDataChecksum(checksum.getChecksumType(),
          Math.max((int) blockLength, 10 * 1024 * 1024));
      bytesPerChecksum = checksum.getBytesPerChecksum();
    }
    checksumSize = checksum.getChecksumSize();
  }
1080783
@Override
    protected final long ciphertextLength(long plaintextLength) {
        long cipherBlockSize = contentCryptoScheme.getBlockSizeInBytes();
        long offset = cipherBlockSize - (plaintextLength % cipherBlockSize);
        return plaintextLength + offset;
    }
251124
public static void dumpBlockData(int headerSize, int blockSize, byte[] data)
   {
      if (data != null)
      {
         System.out.println(ByteArrayHelper.hexdump(data, 0, headerSize, false));
         int index = headerSize;
         while (index < data.length)
         {
            System.out.println(ByteArrayHelper.hexdump(data, index, blockSize, false));
            index += blockSize;
         }
      }
   }
344617
public synchronized  void setLastBlockSize(long blockId, long blockSize) {
    assert blocks.size() > 0;
    LocatedBlock last = blocks.get(blocks.size() - 1);
    if (underConstruction && blockSize > last.getBlockSize()) {
      assert blockId == last.getBlock().getBlockId();
      this.setFileLength(this.getFileLength() + blockSize - last.getBlockSize());
      last.setBlockSize(blockSize);

      if (LOG.isDebugEnabled()) {
        LOG.debug("DFSClient setting last block " + last + " to length "
            + blockSize + " filesize is now " + getFileLength());
      }
    }
  }
240636
void checkLostBlocks(List<Block> blocksLostChecksum,
      List<Block> blocksLostStripe, Path p, Codec codec)
      throws IOException {
    StringBuilder message = new StringBuilder();
    if (blocksLostChecksum.size() > 0) {
      message.append("Lost " + blocksLostChecksum.size() +
          " checksums in blocks:");
      for (Block blk : blocksLostChecksum) {
        message.append(" ");
        message.append(blk.toString());
      }
    }
    if (blocksLostStripe.size() > 0) {
      message.append("Lost " + blocksLostStripe.size() +
          " stripes in blocks:");
      for (Block blk : blocksLostStripe) {
        message.append(" ");
        message.append(blk.toString());
      }
    }
    if (message.length() == 0)
      return;
    message.append(" in file " + p);
    throw new IOException(message.toString());
  }
1358621
protected final byte[] serializeAllocationBlock( int blockIndex )
    {
    	byte[] allocationBlock = new byte[AT_BLOCK_SIZE];

        // Regroup I/O to improve performance
        allocationBlock[AB_FLAGS_OFFSET]       = flags[blockIndex];
        allocationBlock[AB_ALLOCSIZE_OFFSET]   = (byte)((allocatedSize[blockIndex] >>> 24) & 0xFF);
        allocationBlock[AB_ALLOCSIZE_OFFSET+1] = (byte)((allocatedSize[blockIndex] >>> 16) & 0xFF);
        allocationBlock[AB_ALLOCSIZE_OFFSET+2] = (byte)((allocatedSize[blockIndex] >>>  8) & 0xFF);
        allocationBlock[AB_ALLOCSIZE_OFFSET+3] = (byte)((allocatedSize[blockIndex] >>>  0) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET]   = (byte)((previousBlock[blockIndex] >>> 24) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET+1] = (byte)((previousBlock[blockIndex] >>> 16) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET+2] = (byte)((previousBlock[blockIndex] >>>  8) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET+3] = (byte)((previousBlock[blockIndex] >>>  0) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET]   = (byte)((nextBlock[blockIndex] >>> 24) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET+1] = (byte)((nextBlock[blockIndex] >>> 16) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET+2] = (byte)((nextBlock[blockIndex] >>>  8) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET+3] = (byte)((nextBlock[blockIndex] >>>  0) & 0xFF);
        
        return allocationBlock;
    }
366861
private void computePartialChunkCrc(long blkoff, int bytesPerChecksum,
      DataChecksum checksum) throws IOException {

    // find offset of the beginning of partial chunk.
    //
    int sizePartialChunk = (int) (blkoff % bytesPerChecksum);
    int checksumSize = checksum.getChecksumSize();
    long fileOff = BlockInlineChecksumReader.getPosFromBlockOffset(blkoff
        - sizePartialChunk, bytesPerChecksum, checksumSize);
    LOG.info("computePartialChunkCrc sizePartialChunk " + sizePartialChunk
        + " block " + block + " offset in block " + blkoff);

    // create an input stream from the block file
    // and read in partial crc chunk into temporary buffer
    //
    byte[] buf = new byte[sizePartialChunk];
    byte[] crcbuf = new byte[checksumSize];
    // FileInputStream dataIn = null;
    
      /*
      RandomAccessFile blockInFile = new RandomAccessFile(blockFile, "r");
      dataIn = new FileInputStream(blockInFile.getFD());

      if (fileOff > 0) {
        blockInFile.seek(fileOff);
      }
      IOUtils.readFully(dataIn, buf, 0, sizePartialChunk);

      // open meta file and read in crc value computer earlier
      IOUtils.readFully(dataIn, crcbuf, 0, crcbuf.length);
      */
    BlockDataFile.Reader blockReader = blockDataFile.getReader(datanode);
    blockReader.readFully(buf, 0, sizePartialChunk, fileOff, true);
    blockReader.readFully(crcbuf, 0, crcbuf.length, fileOff + sizePartialChunk,
        true);


    // compute crc of partial chunk from data read in the block file.
    Checksum partialCrc = new CRC32();
    partialCrc.update(buf, 0, sizePartialChunk);
    LOG.info("Read in partial CRC chunk from disk for block " + block);

    // paranoia! verify that the pre-computed crc matches what we
    // recalculated just now
    if (partialCrc.getValue() != FSInputChecker.checksum2long(crcbuf)) {
      String msg = "Partial CRC " + partialCrc.getValue()
          + " does not match value computed the "
          + " last time file was closed "
          + FSInputChecker.checksum2long(crcbuf);
      throw new IOException(msg);
    }
    partialCrcInt = (int) partialCrc.getValue();
  }
248953
function (data, blockSize) {
            // Shortcut
            var blockSizeBytes = blockSize * 4;

            // Count padding bytes
            var nPaddingBytes = blockSizeBytes - data.sigBytes % blockSizeBytes;

            // Create padding word
            var paddingWord = (nPaddingBytes << 24) | (nPaddingBytes << 16) | (nPaddingBytes << 8) | nPaddingBytes;

            // Create padding
            var paddingWords = [];
            for (var i = 0; i < nPaddingBytes; i += 4) {
                paddingWords.push(paddingWord);
            }
            var padding = WordArray.create(paddingWords, nPaddingBytes);

            // Add padding
            data.concat(padding);
        }
889661
public synchronized void allocate(RegularFile file, int count) throws IOException {
    int newAllocatedBlockCount = allocatedBlockCount + count;
    if (newAllocatedBlockCount > maxBlockCount) {
      throw new IOException("out of disk space");
    }

    int newBlocksNeeded = Math.max(count - blockCache.blockCount(), 0);

    for (int i = 0; i < newBlocksNeeded; i++) {
      file.addBlock(new byte[blockSize]);
    }

    if (newBlocksNeeded != count) {
      blockCache.transferBlocksTo(file, count - newBlocksNeeded);
    }

    allocatedBlockCount = newAllocatedBlockCount;
  }
510201
def _checksum(in_file, block_size=65536):
    
    cs = hashlib.sha256()
    with open(in_file, "rb") as f:
        for block in iter(lambda: f.read(block_size), b''):
            cs.update(block)
    return cs.hexdigest()
166426
def paddingLength(str_len, blocksize=AES_blocksize):
  ''' '''
  assert 0 < blocksize < 255, 'blocksize must be between 0 and 255'
  assert str_len > 0 , 'string length should be non-negative'
  
  'If the last block is already full, append an extra block of padding'
  pad_len = blocksize - (str_len % blocksize)
  
  return pad_len
247852
public void add(StorageDirView dir, long blockId, long blockSizeBytes) {
    Pair<List<Long>, Long> candidate;
    if (mDirCandidates.containsKey(dir)) {
      candidate = mDirCandidates.get(dir);
    } else {
      candidate = new Pair<List<Long>, Long>(new ArrayList<Long>(), 0L);
      mDirCandidates.put(dir, candidate);
    }

    candidate.getFirst().add(blockId);
    long blockBytes = candidate.getSecond() + blockSizeBytes;
    candidate.setSecond(blockBytes);

    long sum = blockBytes + dir.getAvailableBytes();
    if (mMaxBytes < sum) {
      mMaxBytes = sum;
      mDirWithMaxBytes = dir;
    }
  }
1129695
public Long retrieveChecksum(Block lostBlock, Path p,
      long lostBlockOffset, FileSystem fs, Context context) 
          throws IOException {
    Long oldCRC = null;  
    if (checksumStore != null) { 
      IOException caughtException = null;
      try {
        oldCRC = checksumStore.getChecksum(lostBlock); 
      } catch (IOException ioe) {
        LOG.error(" Fail to get checksum for block " + lostBlock, ioe);
        caughtException = ioe;
      }
      // Checksum is not record, we should report
      if (oldCRC == null) {
        LogUtils.logRaidReconstructionMetrics(LOGRESULTS.FAILURE, 0,
            codec, p, lostBlockOffset,
            LOGTYPES.OFFLINE_RECONSTRUCTION_GET_CHECKSUM,
            fs, caughtException, context);
      }
    }
    return oldCRC;
  }
714063
static public int getBlockCrc(DataNode datanode, ReplicaToRead ri,
      int namespaceId, Block block) throws IOException {

    InputStream rawStreamIn = null;
    DataInputStream streamIn = null;
    int blockCrc = 0;

    try {
      int bytesPerCRC;
      int checksumSize;

      bytesPerCRC = ri.getBytesPerChecksum();
      int checksumType = ri.getChecksumType();
      if (checksumType != DataChecksum.CHECKSUM_CRC32) {
        throw new IOException("File Checksum now is only supported for CRC32");
      }
      DataChecksum dataChecksum = DataChecksum.newDataChecksum(checksumType,
          bytesPerCRC);
      checksumSize = dataChecksum.getChecksumSize();
      
      rawStreamIn = ri.getBlockInputStream(datanode, 0);
      streamIn = new DataInputStream(new BufferedInputStream(rawStreamIn,
          FSConstants.BUFFER_SIZE));
      IOUtils.skipFully(streamIn, BlockInlineChecksumReader.getHeaderSize());

      long lengthLeft = ((FileInputStream) rawStreamIn).getChannel().size()
          - BlockInlineChecksumReader.getHeaderSize();
      if (lengthLeft == 0) {
        blockCrc = (int) dataChecksum.getValue();
      } else {
        byte[] buffer = new byte[checksumSize];
        boolean firstChecksum = true;

        while (lengthLeft > 0) {
          long dataByteLengh;

          if (lengthLeft >= bytesPerCRC + checksumSize) {
            lengthLeft -= bytesPerCRC + checksumSize;
            dataByteLengh = bytesPerCRC;
          } else if (lengthLeft > checksumSize) {
            dataByteLengh = lengthLeft - checksumSize;
            lengthLeft = 0;
          } else {
            // report to name node the corruption.
            DataBlockScanner.reportBadBlocks(block, namespaceId, datanode);
            throw new IOException("File for namespace " + namespaceId
                + " block " + block + " seems to be corrupted");
          }

          IOUtils.skipFully(streamIn, dataByteLengh);
          IOUtils.readFully(streamIn, buffer, 0, buffer.length);
          int intChecksum = DataChecksum.getIntFromBytes(buffer, 0);
          if (firstChecksum) {
            blockCrc = intChecksum;
            firstChecksum = false;
          } else {
            blockCrc = CrcConcat.concatCrc(blockCrc, intChecksum,
                (int) dataByteLengh);
          }
        }
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
            + ", crc=" + blockCrc);
      }
      return blockCrc;
    } finally {
      IOUtils.closeStream(streamIn);
      IOUtils.closeStream(rawStreamIn);
    }

  }
450332
private void readColumnBlock(int startIndex, int blockLength) throws Exception
   {
      int endIndex = startIndex + blockLength;
      List<Integer> blocks = new ArrayList<Integer>();
      for (int index = startIndex; index < endIndex - 11; index++)
      {
         if (matchChildBlock(index))
         {
            int childBlockStart = index - 2;
            blocks.add(Integer.valueOf(childBlockStart));
         }
      }
      blocks.add(Integer.valueOf(endIndex));

      int childBlockStart = -1;
      for (int childBlockEnd : blocks)
      {
         if (childBlockStart != -1)
         {
            int childblockLength = childBlockEnd - childBlockStart;
            try
            {
               readColumn(childBlockStart, childblockLength);
            }
            catch (UnexpectedStructureException ex)
            {
               logUnexpectedStructure();
            }
         }
         childBlockStart = childBlockEnd;
      }
   }
832784
function (block, buffer, offset) {
    var start = offset % block
    var length = buffer.readUInt16LE(start)
    if(length === block - 1)
      return null
    else {
      var _length = buffer.readUInt16LE(start+2+length)
      if(_length != length)
        throw new Error('expected matching length at end, expected:'+length+', was:'+_length)
      result.offset = offset
      result.start = start+2
      result.length = length
      return result
    }
  }
1227513
function roll(data, checksums, blockSize) {
  var results = [];
  var hashtable = createHashtable(checksums);
  var length = data.length;
  var start = 0;
  var end = blockSize > length ? length : blockSize;
  // Updated when a block matches
  var lastMatchedEnd = 0;
  // This gets updated every iteration with the previous weak 32bit hash
  var prevRollingWeak = null;
  var weak;
  var weak16;
  var match;
  var d;
  var len;
  var mightMatch;
  var chunk;
  var strong;
  var hashtable_weak16;
  var hashtable_weak16i;

  for (; end <= length; start++, end++) {
    weak = calcWeak32(data, prevRollingWeak, start, end);
    weak16 = calcWeak16(weak.sum);
    match = false;
    d = null;
    prevRollingWeak = weak;
    hashtable_weak16 = hashtable[weak16];

    if (hashtable_weak16) {
      len = hashtable_weak16.length;
      for (var i = 0; i < len; i++) {
        hashtable_weak16i = hashtable_weak16[i];
        if (hashtable_weak16i.weak === weak.sum) {
          mightMatch = hashtable_weak16i;
          chunk = data.slice(start, end);
          strong = md5sum(chunk);

          if (mightMatch.strong === strong) {
            match = mightMatch;
            break;
          }
        }
      }
    }
    if (match) {
      if(start < lastMatchedEnd) {
        d = data.slice(lastMatchedEnd - 1, end);
        results.push({
          data: d,
          index: match.index
        });
      } else if (start - lastMatchedEnd > 0) {
        d = data.slice(lastMatchedEnd, start);
        results.push({
          data: d,
          index: match.index
        });
      } else {
        results.push({
          index: match.index
        });
      }
      lastMatchedEnd = end;
    } else if (end === length) {
      // No match and last block
      d = data.slice(lastMatchedEnd);
      results.push({
        data: d
      });
    }
  }
  return results;
}