Verify that the actual file contents match the recorded hashes stored in the manifest files
511107
def _validate_entries(self, processes):
        
        errors = list()

        # First we'll make sure there's no mismatch between the filesystem
        # and the list of files in the manifest(s)
        only_in_manifests, only_on_fs = self.compare_manifests_with_fs()
        for path in only_in_manifests:
            e = FileMissing(path)
            logger.warning(str(e))
            errors.append(e)
        for path in only_on_fs:
            e = UnexpectedFile(path)
            logger.warning(str(e))
            errors.append(e)

        # To avoid the overhead of reading the file more than once or loading
        # potentially massive files into memory we'll create a dictionary of
        # hash objects so we can open a file, read a block and pass it to
        # multiple hash objects

        available_hashers = set()
        for alg in self.algs:
            try:
                hashlib.new(alg)
                available_hashers.add(alg)
            except ValueError:
                logger.warning("Unable to validate file contents using unknown %s hash algorithm", alg)

        if not available_hashers:
            raise RuntimeError("%s: Unable to validate bag contents: none of the hash algorithms in %s are supported!" % (self, self.algs))

        def _init_worker():
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        args = ((self.path, rel_path, hashes, available_hashers) for rel_path, hashes in list(self.entries.items()))

        try:
            if processes == 1:
                hash_results = list(map(_calc_hashes, args))
            else:
                try:
                    pool = multiprocessing.Pool(processes if processes else None, _init_worker)
                    hash_results = pool.map(_calc_hashes, args)
                finally:
                    try:
                        pool.terminate()
                    except:
                        # we really don't care about any exception in terminate()
                        pass
        # Any unhandled exceptions are probably fatal
        except:
            logger.exception("unable to calculate file hashes for %s", self)
            raise

        for rel_path, f_hashes, hashes in hash_results:
            for alg, computed_hash in list(f_hashes.items()):
                stored_hash = hashes[alg]
                if stored_hash.lower() != computed_hash:
                    e = ChecksumMismatch(rel_path, alg, stored_hash.lower(), computed_hash)
                    logger.warning(str(e))
                    errors.append(e)

        if errors:
            raise BagValidationError("invalid bag", errors)
1031825
public static void verifySavedMD5(File dataFile, MD5Hash expectedMD5)
      throws IOException {
    MD5Hash storedHash = readStoredMd5ForFile(dataFile);
    // Check the hash itself
    if (!expectedMD5.equals(storedHash)) {
      throw new IOException(
          "File " + dataFile + " did not match stored MD5 checksum " +
          " (stored: " + storedHash + ", computed: " + expectedMD5);
    }
  }
241074
function(opts) {
	var opts = opts || {};
	var shouldHash = typeof opts.hash != 'undefined' ? opts.hash : defaults.hash;
	var prefix = _.flatten([defaults.prefix]);
	var shouldPrefix = typeof opts.shouldPrefix != 'undefined' ? opts.shouldPrefix : true;
	return through.obj(function(file, enc, cb) {
		var originalPath = file.path;

		// Get hash of contents
		var hash = md5(opts, file.contents.toString());

		// Construct new filename
		var ext = path.extname(file.path);
		var basePath = path.basename(file.path, ext);
		var filename = typeof hash !== 'undefined' ? basePath + '-' + hash + ext : basePath + ext;
		file.path = path.join(path.dirname(file.path), filename);

		// Add to manifest
		var base = path.join(file.cwd, defaults.src);
		var key = originalPath.replace(base, '');

		// @TODO: Instead of this it could use "glob" module to regex delete files
		// Check for existing value and whether cleanup is set
		var existing = manifest[key];
		if (existing && existing.src && defaults.cleanup) {
			// Delete the file
			fs.unlink(path.join(file.cwd, defaults.dest, existing.src));
		} else if (defaults.cleanup && shouldHash) {
			// Check if cleanup and hash enabled then we can remove any non hashed version from dest directory
			var nonHashPath = path.join(path.dirname(originalPath), basePath + ext).replace(base, '');
			var absPath = path.join(file.cwd, defaults.dest, nonHashPath);
			fs.exists(absPath, function(exists) {
				if (!exists) return;
				fs.unlink(absPath);
			});
		}

		var filePrefix = shouldPrefix ? prefix[index % prefix.length] : '';

		// Finally add new value to manifest
		var src = file.path.replace(base, '');
		manifest[key] = {
			index: index++,
			src: src,
			dest: filePrefix + src
		};

		// Write manifest file
		writeManifest();

		// Return and continue
		this.push(file);
		cb();
	});
}
1166991
def check_fast(self, reproduce=False, **args):
        
        hashvals = {}

        fast_check = self.check_file(
            filepaths=self.data.keys(),
            hashvals=hashvals,
            hashfn=fast_hashes,
            shortcircuit=True,
            **args
        )

        if not fast_check:

            # Save all the fast hashes for failed files that we've already
            # calculated
            for filepath in hashvals:
                for hash, val in hashvals[filepath].items():
                    self.data[filepath]['hashes'][hash] = val

            if reproduce:
                for filepath in hashvals:
                    print('Check failed for {0} {1}'
                          ''.format(filepath, hashvals[filepath]))
                    tmphash = {}
                    full_check = self.check_file(
                        filepaths=filepath,
                        hashfn=full_hashes,
                        hashvals=tmphash,
                        shortcircuit=False,
                        **args
                    )

                    if full_check:
                        # File is still ok, so replace fast hashes
                        print('Full hashes ({0}) checked ok'
                              ''.format(full_hashes))
                        print('Updating fast hashes for {0} in {1}'
                              ''.format(filepath, self.path))
                        self.add_fast(filepath, force=True)
                        print('Saving updated manifest')
                        self.needsync = True
                    else:
                        sys.stderr.write(
                            'Run cannot reproduce: manifest {0} is not '
                            'correct\n'.format(self.path)
                        )
                        for path, hashdict in tmphash.items():
                            print('    {0}:'.format(path))
                            for hash, val in hashdict.items():
                                hash_table = self.data[path]['hashes']
                                hash_table_val = hash_table.get(hash, None)
                                print('        {0}: {1} != {2}'
                                      ''.format(hash, val, hash_table_val))
                        sys.exit(1)
            else:
                # Not relevant if full hashes are correct. Regenerate full
                # hashes for all filepaths that failed fast check.
                print('Updating full hashes for {0} files in {1}'
                      ''.format(len(hashvals), self.path))

                # Add all full hashes at once -- much faster. Definitely want
                # to force the full hash to be updated. In the specific case of
                # an empty hash the value will be None, without force it will
                # be written as null.
                self.add(
                    filepaths=list(hashvals.keys()),
                    hashfn=full_hashes,
                    force=True,
                    fullpaths=[self.fullpath(fpath) for fpath
                               in list(hashvals.keys())]
                )

                # Flag need to update version on disk
                self.needsync = True
1333370
def _write_output_manifest(self, manifest, filestore_root):
        
        output = os.path.basename(manifest)
        fieldnames, source_manifest = self._parse_manifest(manifest)
        if 'file_path' not in fieldnames:
            fieldnames.append('file_path')
        with atomic_write(output, overwrite=True) as f:
            delimiter = b'\t' if USING_PYTHON2 else '\t'
            writer = csv.DictWriter(f, fieldnames, delimiter=delimiter, quoting=csv.QUOTE_NONE)
            writer.writeheader()
            for row in source_manifest:
                row['file_path'] = self._file_path(row['file_sha256'], filestore_root)
                writer.writerow(row)
            if os.path.isfile(output):
                logger.warning('Overwriting manifest %s', output)
        logger.info('Rewrote manifest %s with additional column containing path to downloaded files.', output)
345923
def verify(path):
        """"""
        valid = False
        try:
            h5 = h5py.File(path, mode="r")
        except (OSError, IsADirectoryError):
            pass
        else:
            if ("file_format" in h5.attrs and
                h5.attrs["file_format"].lower() == "hyperspy" and
                    "Experiments" in h5):
                valid = True
        return valid
1124631
def verify_files(files, user):
    '''
    
    '''
    if salt.utils.platform.is_windows():
        return True
    import pwd  # after confirming not running Windows
    try:
        pwnam = pwd.getpwnam(user)
        uid = pwnam[2]
    except KeyError:
        err = ('Failed to prepare the Salt environment for user '
               '{0}. The user is not available.\n').format(user)
        sys.stderr.write(err)
        sys.exit(salt.defaults.exitcodes.EX_NOUSER)

    for fn_ in files:
        dirname = os.path.dirname(fn_)
        try:
            if dirname:
                try:
                    os.makedirs(dirname)
                except OSError as err:
                    if err.errno != errno.EEXIST:
                        raise
            if not os.path.isfile(fn_):
                with salt.utils.files.fopen(fn_, 'w'):
                    pass

        except IOError as err:
            if os.path.isfile(dirname):
                msg = 'Failed to create path {0}, is {1} a file?'.format(fn_, dirname)
                raise SaltSystemExit(msg=msg)
            if err.errno != errno.EACCES:
                raise
            msg = 'No permissions to access "{0}", are you running as the correct user?'.format(fn_)
            raise SaltSystemExit(msg=msg)

        except OSError as err:
            msg = 'Failed to create path "{0}" - {1}'.format(fn_, err)
            raise SaltSystemExit(msg=msg)

        stats = os.stat(fn_)
        if uid != stats.st_uid:
            try:
                os.chown(fn_, uid, -1)
            except OSError:
                pass
    return True
82178
private void checkHashes(Dependency dependency, String sha1, String sha256, String md5) throws FileNotFoundException {
        final String md5sum = dependency.getMd5sum();
        if (!md5.equals(md5sum)) {
            throw new FileNotFoundException("Artifact found by API is not matching the md5 "
                    + "of the artifact (repository hash is " + md5 + WHILE_ACTUAL_IS + md5sum + ") !");
        }
        final String sha1sum = dependency.getSha1sum();
        if (!sha1.equals(sha1sum)) {
            throw new FileNotFoundException("Artifact found by API is not matching the SHA1 "
                    + "of the artifact (repository hash is " + sha1 + WHILE_ACTUAL_IS + sha1sum + ") !");
        }
        final String sha256sum = dependency.getSha256sum();
        if (sha256 != null && !sha256.equals(sha256sum)) {
            throw new FileNotFoundException("Artifact found by API is not matching the SHA-256 "
                    + "of the artifact (repository hash is " + sha256 + WHILE_ACTUAL_IS + sha256sum + ") !");
        }
    }
354315
def valid_checksum(file, hash_file):
    
    bits = 4096
    # calc md5 hash
    hash_md5 = hashlib.md5()
    with open(file, "rb") as f:
        for chunk in iter(lambda: f.read(bits), b""):
            hash_md5.update(chunk)
    # locate hash signature for file, validate
    with open(hash_file) as c:
        for line in c.readlines():
            if line.strip():
                check_list = line.split()
                if file == check_list[1]:
                    if check_list[0] == hash_md5.hexdigest():
                        return True
    return False
1095803
def has_data_file(self, sha1hash):  # type: (str) -> bool
        """"""
        folder = os.path.join(self.folder, DATA, sha1hash[0:2])
        hash_path = os.path.join(folder, sha1hash)
        return os.path.isfile(hash_path)
1173667
def sha_match?(file, gav, use_sha_file=false)
      if File.exists?(file)
        if gav.sha1.nil?
          gav.sha1 = gav_data(gav)['sha1']
        end
        sha(file,use_sha_file) == gav.sha1
      else
        false
      end
    end
1208123
def parse_manifest(template_lines):
    """"""
    manifest_files = distutils.filelist.FileList()
    for line in template_lines:
        if line.strip():
            manifest_files.process_template_line(line)
    return manifest_files.files
684204
def action_verify_checksum(self, ids):
        """"""
        try:
            count = 0
            for file_id in ids:
                f = FileInstance.query.filter_by(
                    id=uuid.UUID(file_id)).one_or_none()
                if f is None:
                    raise ValueError(_("Cannot find file instance."))
                verify_checksum.delay(file_id)
                count += 1
            if count > 0:
                flash(_('Fixity check(s) sent to queue.'), 'success')
        except Exception as exc:
            if not self.handle_view_exception(exc):
                raise
            current_app.logger.exception(str(exc))  # pragma: no cover
            flash(_('Failed to run fixity checks.'),
                  'error')
1166014
def file_hash(load, fnd):
    '''
    
    '''
    if 'env' in load:
        # "env" is not supported; Use "saltenv".
        load.pop('env')

    ret = {}

    if 'saltenv' not in load:
        return ret

    if 'path' not in fnd or 'bucket' not in fnd or not fnd['path']:
        return ret

    cached_file_path = _get_cached_file_name(
            fnd['bucket'],
            load['saltenv'],
            fnd['path'])

    if os.path.isfile(cached_file_path):
        ret['hsum'] = salt.utils.hashutils.get_hash(cached_file_path)
        ret['hash_type'] = 'md5'

    return ret
150706
def _check_missing_manifests(self, segids):
    """"""
    manifest_paths = [ self._manifest_path(segid) for segid in segids ]
    with Storage(self.vol.layer_cloudpath, progress=self.vol.progress) as stor:
      exists = stor.files_exist(manifest_paths)

    dne = []
    for path, there in exists.items():
      if not there:
        (segid,) = re.search(r'(\d+):0$', path).groups()
        dne.append(segid)
    return dne
794347
def verify_file_exists(file_name, file_location):
    
    return __os.path.isfile(__os.path.join(file_location, file_name))
804961
def clean!
      files_to_keep = cache.keys.concat(cache.values)

      if (sprockets_manifest = Dir.entries("#{base_dir}").detect { |entry| entry =~ SPROCKETS_MANIFEST_REGEX })
        files_to_keep.concat(JSON.parse(File.read("#{base_dir}/#{sprockets_manifest}")).fetch("files", {}).keys)
      end

      Dir["#{base_dir}/**/*"].each do |path|
        next if File.directory?(path) || files_to_keep.include?(Pathname(path).relative_path_from(base_dir).to_s)

        FileUtils.rm(path)
      end
    end