If ``ds`` is a ``skorch.dataset.Dataset`` or a
    ``skorch.dataset.Dataset`` nested inside a
    ``torch.utils.data.Subset`` and uses
    y as a placeholder, return ``True``.
179581
def uses_placeholder_y(ds):
    """"""

    if isinstance(ds, torch.utils.data.Subset):
        return uses_placeholder_y(ds.dataset)
    return isinstance(ds, Dataset) and hasattr(ds, "y") and ds.y is None
652567
def dict_is_subset(subset: Any, full_set: Any) -> bool:
    

    if not isinstance(subset, full_set.__class__):
        return False
    elif isinstance(subset, dict):
        for k, v in subset.items():
            if k not in full_set or not dict_is_subset(v, full_set[k]):
                return False

        return True
    elif isinstance(subset, list):
        if len(subset) != len(full_set):
            return False

        for a, b in zip(subset, full_set):
            if not dict_is_subset(a, b):
                return False

        return True
    else:
        return subset == full_set
807210
def _make_dataset(cls, coords):
        
        class Slice(cls._SliceType):
            extra_coords = coords

        Slice.__name__ = '%s.slice(%s)' % (
            cls.__name__,
            ', '.join('%s=%r' % item for item in coords.items()),
        )
        return Slice
808571
def select_mask(cls, dataset, selection):
        
        select_mask = None
        for dim, k in selection.items():
            if isinstance(k, tuple):
                k = slice(*k)
            masks = []
            alias = dataset.get_dimension(dim).name
            series = dataset.data[alias]
            if isinstance(k, slice):
                if k.start is not None:
                    # Workaround for dask issue #3392
                    kval = util.numpy_scalar_to_python(k.start)
                    masks.append(kval <= series)
                if k.stop is not None:
                    kval = util.numpy_scalar_to_python(k.stop)
                    masks.append(series < kval)
            elif isinstance(k, (set, list)):
                iter_slc = None
                for ik in k:
                    mask = series == ik
                    if iter_slc is None:
                        iter_slc = mask
                    else:
                        iter_slc |= mask
                masks.append(iter_slc)
            elif callable(k):
                masks.append(k(series))
            else:
                masks.append(series == k)
            for mask in masks:
                if select_mask is not None:
                    select_mask &= mask
                else:
                    select_mask = mask
        return select_mask
9534
def index_library_datasets(self, tick_f=None):
        

        dataset_n = 0
        partition_n = 0

        def tick(d, p):
            if tick_f:
                tick_f('datasets: {} partitions: {}'.format(d, p))

        for dataset in self.library.datasets:

            if self.backend.dataset_index.index_one(dataset):
                # dataset added to index
                dataset_n += 1
                tick(dataset_n, partition_n)
                for partition in dataset.partitions:
                    self.backend.partition_index.index_one(partition)
                    partition_n += 1
                    tick(dataset_n, partition_n)
            else:
                # dataset already indexed
                pass
813785
def _validate_dataset(dataset):
    
    if not (isinstance(dataset, _SFrame)):
        raise TypeError("Input 'dataset' must be an SFrame.")

    if dataset.num_rows() == 0 or dataset.num_columns() == 0:
        raise ValueError("Input 'dataset' has no data.")
1356386
def select(cls, dataset, selection_mask=None, **selection):
        
        selection = {k: slice(*sel) if isinstance(sel, tuple) else sel
                     for k, sel in selection.items()}
        coords = tuple(selection[kd.name] if kd.name in selection else slice(None)
                       for kd in dataset.kdims)
        if not any([isinstance(el, slice) for el in coords]):
            return dataset.data[dataset.sheet2matrixidx(*coords)]

        # Apply slices
        xidx, yidx = coords
        l, b, r, t = dataset.bounds.lbrt()
        if isinstance(xidx, slice):
            l = l if xidx.start is None else max(l, xidx.start)
            r = r if xidx.stop is None else min(r, xidx.stop)
        if isinstance(yidx, slice):
            b = b if yidx.start is None else max(b, yidx.start)
            t = t if yidx.stop is None else min(t, yidx.stop)
        bounds = BoundingBox(points=((l, b), (r, t)))
        slc = Slice(bounds, dataset)
        return slc.submatrix(dataset.data)
306606
def relpath(dataset_uri, item_identifier):
    
    dataset = dtoolcore.DataSet.from_uri(dataset_uri)
    try:
        props = dataset.item_properties(item_identifier)
    except KeyError:
        click.secho(
            "No such item in dataset: {}".format(item_identifier),
            fg="red",
            err=True
        )
        sys.exit(21)
    click.secho(props["relpath"])
1165027
def _gotitem(self, key, ndim, subset=None):
        
        # create a new object to prevent aliasing
        if subset is None:
            subset = self.obj

        # we need to make a shallow copy of ourselves
        # with the same groupby
        kwargs = {attr: getattr(self, attr) for attr in self._attributes}

        # Try to select from a DataFrame, falling back to a Series
        try:
            groupby = self._groupby[key]
        except IndexError:
            groupby = self._groupby

        self = self.__class__(subset,
                              groupby=groupby,
                              parent=self,
                              **kwargs)
        self._reset_cache()
        if subset.ndim == 2:
            if is_scalar(key) and key in subset or is_list_like(key):
                self._selection = key
        return self
312912
def copy_dataset(self, ds, incver=False, cb=None, **kwargs):
        
        from ambry.orm import Table, Column, Partition, File, ColumnStat, Code, \
            DataSource, SourceTable, SourceColumn

        tables = [Table, Column, Partition, File, ColumnStat, Code, SourceTable, SourceColumn, DataSource]

        return self._copy_dataset_copy(ds, tables, incver, cb, **kwargs)
230433
def get_dataset(dataset_id,**kwargs):
    

    user_id = int(kwargs.get('user_id'))

    if dataset_id is None:
        return None
    try:
        dataset_rs = db.DBSession.query(Dataset.id,
                Dataset.type,
                Dataset.unit_id,
                Dataset.name,
                Dataset.hidden,
                Dataset.cr_date,
                Dataset.created_by,
                DatasetOwner.user_id,
                null().label('metadata'),
                case([(and_(Dataset.hidden=='Y', DatasetOwner.user_id is not None), None)],
                        else_=Dataset.value).label('value')).filter(
                Dataset.id==dataset_id).outerjoin(DatasetOwner,
                                    and_(DatasetOwner.dataset_id==Dataset.id,
                                    DatasetOwner.user_id==user_id)).one()

        rs_dict = dataset_rs._asdict()

        #convert the value row into a string as it is returned as a binary
        if dataset_rs.value is not None:
            rs_dict['value'] = str(dataset_rs.value)

        if dataset_rs.hidden == 'N' or (dataset_rs.hidden == 'Y' and dataset_rs.user_id is not None):
            metadata = db.DBSession.query(Metadata).filter(Metadata.dataset_id==dataset_id).all()
            rs_dict['metadata'] = metadata
        else:
            rs_dict['metadata'] = []

    except NoResultFound:
        raise HydraError("Dataset %s does not exist."%(dataset_id))


    dataset = namedtuple('Dataset', rs_dict.keys())(**rs_dict)

    return dataset
1170174
def _subset(subset, superset):
    

    result = True
    for k in subset:
        result = k in superset and subset[k] == superset[k]
        if not result:
            break
    return result
658197
def _canonize_input(self, dataset):
        
        unpack = lambda x: x
        if isinstance(dataset, _tc.SArray):
            dataset = _tc.SFrame({self.feature: dataset})
        elif isinstance(dataset, _tc.Image):
            dataset = _tc.SFrame({self.feature: [dataset]})
            unpack = lambda x: x[0]
        return dataset, unpack
752495
def _slice_datasets(self, dataset_ids, slice_key, new_area, area_only=True):
        """"""
        new_datasets = {}
        datasets = (self[ds_id] for ds_id in dataset_ids)
        for ds, parent_ds in dataset_walker(datasets):
            ds_id = DatasetID.from_dict(ds.attrs)
            # handle ancillary variables
            pres = None
            if parent_ds is not None:
                pres = new_datasets[DatasetID.from_dict(parent_ds.attrs)]
            if ds_id in new_datasets:
                replace_anc(ds, pres)
                continue
            if area_only and ds.attrs.get('area') is None:
                new_datasets[ds_id] = ds
                replace_anc(ds, pres)
                continue

            if not isinstance(slice_key, dict):
                # match dimension name to slice object
                key = dict(zip(ds.dims, slice_key))
            else:
                key = slice_key
            new_ds = ds.isel(**key)
            if new_area is not None:
                new_ds.attrs['area'] = new_area

            new_datasets[ds_id] = new_ds
            if parent_ds is None:
                # don't use `__setitem__` because we don't want this to
                # affect the existing wishlist/dep tree
                self.datasets[ds_id] = new_ds
            else:
                replace_anc(new_ds, pres)
661488
def exists(self, dataset_id):
        
        from google.api_core.exceptions import NotFound

        try:
            self.client.get_dataset(self.client.dataset(dataset_id))
            return True
        except NotFound:
            return False
        except self.http_error as ex:
            self.process_http_error(ex)
312591
def check_dataset(dataset):
    """"""
    if isinstance(dataset, numpy.ndarray) and not len(dataset.shape) == 4:
        check_dataset_shape(dataset)
        check_dataset_range(dataset)
    else:  # must be a list of arrays or a 4D NumPy array
        for i, d in enumerate(dataset):
            if not isinstance(d, numpy.ndarray):
                raise ValueError(
                    'Requires a NumPy array (rgb x rows x cols) '
                    'with integer values in the range [0, 255].'
                )
            try:
                check_dataset_shape(d)
                check_dataset_range(d)
            except ValueError as err:
                raise ValueError(
                    '{}\nAt position {} in the list of arrays.'
                    .format(err, i)
                )
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data