given the path to a local file being streamed, returns back the MD5 hash to quickly check if a file's contents have changed.
510966
function getFileHash(filePath, callback) {
  var stream = fs.ReadStream(filePath);
  var md5sum = crypto.createHash('md5');

  stream.on('data', function(data) {
    md5sum.update(data);
  });

  stream.on('end', function() {
    callback(md5sum.digest('hex'));
  });
}
118503
def _post_md5_skip_on_check(self, key, filename, size, md5_match):
        # type: (Downloader, str, str, int, bool) -> None
        
        with self._md5_meta_lock:
            rfile = self._md5_map.pop(key)
        lpath = pathlib.Path(filename)
        if md5_match:
            if size is None:
                size = lpath.stat().st_size
            with self._transfer_lock:
                self._transfer_set.remove(
                    blobxfer.operations.download.Downloader.
                    create_unique_transfer_operation_id(rfile))
                self._download_total -= 1
                self._download_bytes_total -= size
            if self._general_options.dry_run:
                logger.info('[DRY RUN] MD5 match, skipping: {} -> {}'.format(
                    rfile.path, lpath))
        else:
            if self._general_options.dry_run:
                with self._transfer_lock:
                    self._transfer_set.remove(
                        blobxfer.operations.download.Downloader.
                        create_unique_transfer_operation_id(rfile))
                    self._download_total -= 1
                    self._download_bytes_total -= size
                logger.info(
                    '[DRY RUN] MD5 mismatch, download: {} -> {}'.format(
                        rfile.path, lpath))
            else:
                self._add_to_download_queue(lpath, rfile)
1244269
def _pre_md5_skip_on_check(self, src, rfile):
        # type: (Uploader, blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity) -> None
        
        md5 = blobxfer.models.metadata.get_md5_from_metadata(rfile)
        key = blobxfer.operations.upload.Uploader.create_unique_id(src, rfile)
        with self._md5_meta_lock:
            self._md5_map[key] = (src, rfile)
        self._md5_offload.add_localfile_for_md5_check(
            key, None, str(src.absolute_path), md5, rfile.mode, src.view)
786326
def file_hash(path, hash_type='md5'):
    
    if os.path.exists(path):
        h = getattr(hashlib, hash_type)()
        with open(path, 'rb') as source:
            h.update(source.read())
        return h.hexdigest()
    else:
        return None
700487
def _get_file_md5(filename):
    """"""
    md5_data = md5()
    with open(filename, 'rb') as f:
        for chunk in iter(lambda: f.read(128*md5_data.block_size), b''):
            md5_data.update(chunk)
    return md5_data.hexdigest()
739607
def ReceiveFileHash(self, responses):
    """"""
    index = responses.request_data["index"]
    if not responses.success:
      self.Log("Failed to hash file: %s", responses.status)
      self.state.pending_hashes.pop(index, None)
      # Report the error.
      self._FileFetchFailed(index, "HashFile")
      return

    self.state.files_hashed += 1
    response = responses.First()
    if response.HasField("hash"):
      hash_obj = response.hash
    else:
      # Deprecate this method of returning hashes.
      hash_obj = rdf_crypto.Hash()

      if len(response.results) < 1 or response.results[0]["name"] != "generic":
        self.Log("Failed to hash file: %s", self.state.indexed_pathspecs[index])
        self.state.pending_hashes.pop(index, None)
        return

      result = response.results[0]

      try:
        for hash_type in ["md5", "sha1", "sha256"]:
          value = result.GetItem(hash_type)
          setattr(hash_obj, hash_type, value)
      except AttributeError:
        self.Log("Failed to hash file: %s", self.state.indexed_pathspecs[index])
        self.state.pending_hashes.pop(index, None)
        return

    try:
      tracker = self.state.pending_hashes[index]
    except KeyError:
      # Hashing the file failed, but we did stat it.
      self._FileFetchFailed(index, responses.request.request.name)
      return

    tracker["hash_obj"] = hash_obj
    tracker["bytes_read"] = response.bytes_read

    self.state.files_hashed_since_check += 1
    if self.state.files_hashed_since_check >= self.MIN_CALL_TO_FILE_STORE:
      self._CheckHashesWithFileStore()
854788
def has_changed (filename):
    """"""
    key = os.path.abspath(filename)
    mtime = get_mtime(key)
    if key not in _mtime_cache:
        _mtime_cache[key] = mtime
        return True
    return mtime > _mtime_cache[key]
1035621
def _up_to_date(md5, fn):
    
    if os.path.exists(fn):
        if hashlib.md5(open(fn).read()).hexdigest() == md5:
            logger.info('md5sum match for %s' % fn)
            return True
        else:
            logger.info('wrong md5sum for %s' % fn)
            os.unlink(fn)
739902
def compare_checksums(self):
        
        is_match = True
        reference_checksums = self.parse_checksum_file(
            self.files['checksum']['file'])
        for md5, file in reference_checksums.items():
            if os.path.isfile('/'.join((self.rawdir, file))):
                if self.get_file_md5(self.rawdir, file) != md5:
                    is_match = False
                    LOG.warning('%s was not downloaded completely', file)
                    return is_match

        return is_match
190839
def checksum(path):
    """"""
    hasher = hashlib.sha1()
    with open(path, 'rb') as stream:
        buf = stream.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = stream.read(BLOCKSIZE)
    return hasher.hexdigest()
1271036
def _hash_of_file(path, algorithm):
    """"""
    with open(path, 'rb') as archive:
        hash = hashlib.new(algorithm)
        for chunk in read_chunks(archive):
            hash.update(chunk)
    return hash.hexdigest()
468890
def md5(filename):
    '''  '''

    if not os.path.exists(filename):
        return None
    digest = _md5()
    blocksize = 64 * 1024
    infile = open(filename, 'rb')
    block = infile.read(blocksize)
    while block:
        digest.update(block)
        block = infile.read(blocksize)
    infile.close()
    return digest.hexdigest()
966183
def replace_if_changed(localfile, jottapath, JFS):
    """"""
    jf = JFS.getObject(jottapath)
    lf_hash = getxattrhash(localfile) # try to read previous hash, stored in xattr
    if lf_hash is None:               # no valid hash found in xattr,
        with open(localfile) as lf:
            lf_hash = calculate_md5(lf) # (re)calculate it
    if type(jf) == JFSIncompleteFile:
        log.debug("Local file %s is incompletely uploaded, continue", localfile)
        return resume(localfile, jf, JFS)
    elif jf.md5 == lf_hash: # hashes are the same
        log.debug("hash match (%s), file contents haven't changed", lf_hash)
        setxattrhash(localfile, lf_hash)
        return jf         # return the version from jottaclouds
    else:
        setxattrhash(localfile, lf_hash)
        return new(localfile, jottapath, JFS)
1248504
def changed(self, filename='.md5', glob=None):
        
        if glob is not None:
            filename += '.glob-' + ''.join(ch.lower()
                                           for ch in glob if ch.isalpha())
        return changed(self, filename, glob=glob)
1030498
def md5_checksum(file_path, chunk_bytes=4194304):
    """"""

    with open(file_path, "rb") as infile:
        checksum = hashlib.md5()
        while 1:
            data = infile.read(chunk_bytes)
            if not data:
                break
            checksum.update(data)

    return checksum.hexdigest()
1128515
def get_file_hash(self, *algorithms: str):
        '''
        
        '''
        from .hashs import hashfile_hexdigest
        return hashfile_hexdigest(self._path, algorithms)
483677
def hash_file(path, digest=None):
  
  digest = digest or hashlib.sha1()
  with open(path, 'rb') as fd:
    s = fd.read(8192)
    while s:
      digest.update(s)
      s = fd.read(8192)
  return digest.hexdigest() if PY3 else digest.hexdigest().decode('utf-8')