Find missing files listed in datasets.
508306
def check_missing_files(client):
    """"""
    missing = defaultdict(list)

    for path, dataset in client.datasets.items():
        for file in dataset.files:
            filepath = (path.parent / file)
            if not filepath.exists():
                missing[str(
                    path.parent.relative_to(client.renku_datasets_path)
                )].append(
                    os.path.normpath(str(filepath.relative_to(client.path)))
                )

    if not missing:
        return True

    click.secho(
        WARNING + 'There are missing files in datasets.'
        # '\n  (use "renku dataset clean <name>" to clean them)'
    )

    for dataset, files in missing.items():
        click.secho(
            '\n\t' + click.style(dataset, fg='yellow') + ':\n\t  ' +
            '\n\t  '.join(click.style(path, fg='red') for path in files)
        )

    return False
1257738
def find_clean_dirs(self):
        """"""

        for folder in glob.glob(self.glob_pattern):
            if re.match(self.re_pattern, folder):
                yield folder
        for folder in glob.glob(self.orig_glob_pattern):
            if re.match(self.orig_re_pattern, folder):
                yield folder
405767
def exclude_data_files(self, package, src_dir, files):
        """'"""
        files = list(files)
        patterns = self._get_platform_patterns(
            self.exclude_package_data,
            package,
            src_dir,
        )
        match_groups = (
            fnmatch.filter(files, pattern)
            for pattern in patterns
        )
        # flatten the groups of matches into an iterable of matches
        matches = itertools.chain.from_iterable(match_groups)
        bad = set(matches)
        keepers = (
            fn
            for fn in files
            if fn not in bad
        )
        # ditch dupes
        return list(_unique_everseen(keepers))
782918
def available_datasets(self):
        """"""
        res = self.resolution
        coordinates = ['pixel_longitude', 'pixel_latitude']
        for var_name, val in self.file_content.items():
            if isinstance(val, netCDF4.Variable):
                ds_info = {
                    'file_type': self.filetype_info['file_type'],
                    'resolution': res,
                }
                if not self.is_geo:
                    ds_info['coordinates'] = coordinates
                yield DatasetID(name=var_name, resolution=res), ds_info
102183
def check_for_reqd_cols(data, reqd_cols):
    
    missing = []
    for col in reqd_cols:
        if col not in data[0]:
            missing.append(col)
    return missing
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
557200
def available_datasets(self):
        """"""
        sensor = self.get_sensor(self['/attr/sensor'])
        nadir_resolution = self.get_nadir_resolution(sensor)
        for var_name, val in self.file_content.items():
            if isinstance(val, SDS):
                ds_info = {
                    'file_type': self.filetype_info['file_type'],
                    'resolution': nadir_resolution,
                }
                if self._is_polar():
                    ds_info['coordinates'] = ['longitude', 'latitude']
                yield DatasetID(name=var_name, resolution=nadir_resolution), ds_info
956405
def dataset(ctx, client, revision, datadir, format):
    """"""
    ctx.meta['renku.datasets.datadir'] = datadir

    if ctx.invoked_subcommand is not None:
        return

    if revision is None:
        datasets = client.datasets.values()
    else:
        datasets = client.datasets_from_commit(client.repo.commit(revision))

    DATASETS_FORMATS[format](client, datasets)
5897
def ListFiles(self, ext_attrs=None):
    """"""
    del ext_attrs  # Unused.

    if not self.IsDirectory():
      return

    if self.hive is None:
      for name in dir(winreg):
        if name.startswith("HKEY_"):
          response = rdf_client_fs.StatEntry(st_mode=stat.S_IFDIR)
          response_pathspec = self.pathspec.Copy()
          response_pathspec.last.path = utils.JoinPath(
              response_pathspec.last.path, name)
          response.pathspec = response_pathspec

          yield response
      return

    try:
      with OpenKey(self.hive, self.local_path) as key:
        (self.number_of_keys, self.number_of_values,
         self.last_modified) = QueryInfoKey(key)

        # First keys - These will look like directories.
        for i in range(self.number_of_keys):
          try:
            name = EnumKey(key, i)
            key_name = utils.JoinPath(self.local_path, name)

            try:
              # Store the default value in the stat response for values.
              with OpenKey(self.hive, key_name) as subkey:
                value, value_type = QueryValueEx(subkey, "")
            except OSError:
              value, value_type = None, None

            response = self._Stat(name, value, value_type)
            # Keys look like Directories in the VFS.
            response.st_mode = stat.S_IFDIR

            yield response
          except OSError:
            pass

        # Now Values - These will look like files.
        for i in range(self.number_of_values):
          try:
            name, value, value_type = EnumValue(key, i)
            response = self._Stat(name, value, value_type)

            # Values look like files in the VFS.
            response.st_mode = stat.S_IFREG

            yield response

          except OSError:
            pass
    except OSError as e:
      raise IOError("Unable to list key %s: %s" % (self.key_name, e))
466694
def files(self, paths, access=None, extensions=None, minsize=None):
        """"""
        self.failures = [path for path in paths if not
                         isvalid(path, access, extensions, 'file', minsize)]

        return not self.failures
1196013
def CheckForUnknownFiles(self):
		""""""
		unknown_files = self.GetUnknownFiles()
		if unknown_files:
			print "The following files are not added to version control:"
			for line in unknown_files:
				print line
			prompt = "Are you sure to continue?(y/N) "
			answer = raw_input(prompt).strip()
			if answer != "y":
				ErrorExit("User aborted")
166232
def reconstruct_files(input_dir):
    ''''''
    input_dir = input_dir.rstrip('/')
    with nl.notify('Attempting to organize/reconstruct directory'):
        # Some datasets start with a ".", which confuses many programs
        for r,ds,fs in os.walk(input_dir):
            for f in fs:
                if f[0]=='.':
                    shutil.move(os.path.join(r,f),os.path.join(r,'i'+f))
        nl.dicom.organize_dir(input_dir)
        output_dir = '%s-sorted' % input_dir
        if os.path.exists(output_dir):
            with nl.run_in(output_dir):
                for dset_dir in os.listdir('.'):
                    with nl.notify('creating dataset from %s' % dset_dir):
                        nl.dicom.create_dset(dset_dir)
        else:
            nl.notify('Warning: failed to auto-organize directory %s' % input_dir,level=nl.level.warning)
1129159
@Override
  public Set<Dataset> findDistinctDatasets() throws IOException {
    Set<Dataset> datasets = Sets.newHashSet();
    Path inputPath = new Path(this.inputDir);
    Path inputLatePath = new Path(inputPath, MRCompactor.COMPACTION_LATE_DIR_SUFFIX);
    Path outputPath = new Path(this.destDir);
    Path outputLatePath = new Path(outputPath, MRCompactor.COMPACTION_LATE_DIR_SUFFIX);
    Dataset dataset =
        new Dataset.Builder().withPriority(this.getDatasetPriority(inputPath.getName()))
            .addInputPath(this.recompactDatasets ? outputPath : inputPath)
            .addInputLatePath(this.recompactDatasets ? outputLatePath : inputLatePath).withOutputPath(outputPath)
            .withOutputLatePath(outputLatePath).withOutputTmpPath(new Path(this.tmpOutputDir)).build();
    datasets.add(dataset);
    return datasets;
  }
1321547
def check_dataset_metadata(client):
    """"""
    # Find pre 0.3.4 metadata files.
    old_metadata = list(_dataset_metadata_pre_0_3_4(client))

    if not old_metadata:
        return True

    click.secho(
        WARNING + 'There are metadata files in the old location.'
        '\n  (use "renku migrate datasets" to move them)\n\n\t' + '\n\t'.join(
            click.style(str(path.relative_to(client.path)), fg='yellow')
            for path in old_metadata
        ) + '\n'
    )

    return False
1267536
def _read_image_list(self, skip_image_ids=None):
    """"""
    if skip_image_ids is None:
      skip_image_ids = []
    images = self._storage_client.list_blobs(
        prefix=os.path.join('dataset', self._dataset_name) + '/')
    zip_files = [i for i in images if i.endswith('.zip')]
    if len(zip_files) == 1:
      # we have a zip archive with images
      zip_name = zip_files[0]
      logging.info('Reading list of images from zip file %s', zip_name)
      blob = self._storage_client.get_blob(zip_name)
      buf = BytesIO()
      logging.info('Downloading zip')
      blob.download_to_file(buf)
      buf.seek(0)
      logging.info('Reading content of the zip')
      with zipfile.ZipFile(buf) as f:
        images = [os.path.join(zip_name, os.path.basename(n))
                  for n in f.namelist() if n.endswith('.png')]
      buf.close()
      logging.info('Found %d images', len(images))
    else:
      # we have just a directory with images, filter non-PNG files
      logging.info('Reading list of images from png files in storage')
      images = [i for i in images if i.endswith('.png')]
      logging.info('Found %d images', len(images))
    # filter images which should be skipped
    images = [i for i in images
              if os.path.basename(i)[:-4] not in skip_image_ids]
    # assign IDs to images
    images = [(DATASET_IMAGE_ID_PATTERN.format(idx), i)
              for idx, i in enumerate(sorted(images))]
    return images
634001
def files(self):
    """"""
    all_files = set()
    for label in self.filesets:
      all_files.update(self.filesets[label])
    return all_files
735158
def clear_file_systems(self):
        """"""

        self._source_url = None
        self.dataset.config.library.source.url = None
        self._source_fs = None

        self._build_url = None
        self.dataset.config.library.build.url = None
        self._build_fs = None

        self.dataset.commit()