Given a Dataset object and a dictionary with dimension keys and
        selection keys (i.e tuple ranges, slices, sets, lists or literals)
        return a boolean mask over the rows in the Dataset object that
        have been selected.
774630
def select_mask(cls, dataset, selection):
        
        mask = np.ones(len(dataset), dtype=np.bool)
        for dim, k in selection.items():
            if isinstance(k, tuple):
                k = slice(*k)
            arr = cls.values(dataset, dim)
            if isinstance(k, slice):
                with warnings.catch_warnings():
                    warnings.filterwarnings('ignore', r'invalid value encountered')
                    if k.start is not None:
                        mask &= k.start <= arr
                    if k.stop is not None:
                        mask &= arr < k.stop
            elif isinstance(k, (set, list)):
                iter_slcs = []
                for ik in k:
                    with warnings.catch_warnings():
                        warnings.filterwarnings('ignore', r'invalid value encountered')
                        iter_slcs.append(arr == ik)
                mask &= np.logical_or.reduce(iter_slcs)
            elif callable(k):
                mask &= k(arr)
            else:
                index_mask = arr == k
                if dataset.ndims == 1 and np.sum(index_mask) == 0:
                    data_index = np.argmin(np.abs(arr - k))
                    mask = np.zeros(len(dataset), dtype=np.bool)
                    mask[data_index] = True
                else:
                    mask &= index_mask
        return mask
338574
def boolMasksToImage(masks):
    '''
    
    '''
    assert len(masks) <= 8, 'can only transform up to 8 masks into image'
    masks = np.asarray(masks, dtype=np.uint8)
    assert masks.ndim == 3, 'layers need to be stack of 2d arrays'
    return np.packbits(masks, axis=0)[0].T
848513
def mask_xdata_with_shape(self, shape: DataAndMetadata.ShapeType) -> DataAndMetadata.DataAndMetadata:
        
        mask = self._graphic.get_mask(shape)
        return DataAndMetadata.DataAndMetadata.from_data(mask)
1312643
def split(cls, dataset, start, end, datatype, **kwargs):
        
        objs = []
        if datatype is None:
            for d in dataset.data[start: end]:
                objs.append(dataset.clone(d, datatype=cls.subtypes))
            return objs
        elif not dataset.data:
            return objs
        ds = cls._inner_dataset_template(dataset)
        for d in dataset.data:
            ds.data = d
            if datatype == 'array':
                obj = ds.array(**kwargs)
            elif datatype == 'dataframe':
                obj = ds.dframe(**kwargs)
            elif datatype == 'columns':
                if ds.interface.datatype == 'dictionary':
                    obj = dict(ds.data)
                else:
                    obj = ds.columns(**kwargs)
            else:
                raise ValueError("%s datatype not support" % datatype)
            objs.append(obj)
        return objs
1121112
def get_dataset_key(self, key, **kwargs):
        
        return get_key(key, self.ids.keys(), **kwargs)
725103
public static MultiDataSet toMultiDataSet(DataSet dataSet) {
        INDArray f = dataSet.getFeatures();
        INDArray l = dataSet.getLabels();
        INDArray fMask = dataSet.getFeaturesMaskArray();
        INDArray lMask = dataSet.getLabelsMaskArray();

        INDArray[] fNew = f == null ? null : new INDArray[] {f};
        INDArray[] lNew = l == null ? null : new INDArray[] {l};
        INDArray[] fMaskNew = (fMask != null ? new INDArray[] {fMask} : null);
        INDArray[] lMaskNew = (lMask != null ? new INDArray[] {lMask} : null);

        return new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
    }
1025226
def indexed(cls, dataset, selection):
        
        selected = list(selection.keys())
        all_scalar = all((not isinstance(sel, (tuple, slice, set, list))
                          and not callable(sel)) for sel in selection.values())
        all_kdims = all(d in selected for d in dataset.kdims)
        return all_scalar and all_kdims
681690
def select_paths(cls, dataset, selection):
        
        return [s[0] for s in np.array([{0: p} for p in dataset.data])[selection]]
1362709
def datasetItems(self):
        
        out = {}
        for item in self.items():
            if isinstance(item, XChartDatasetItem):
                out[item.dataset()] = item
        return out
119926
def _chunked_selector_output_shape(  # pylint: disable=invalid-name
    input_shapes, selector=None, **unused_kwargs):
  """"""
  # Read the main function below first, the shape logic just follows the ops.
  selector = selector or (lambda x: [] if x < 1 else [x-1])
  triples, _ = zip(*input_shapes)
  (query_shapes, key_shapes, value_shapes) = zip(*triples)
  result = []
  for i in range(len(input_shapes)):
    selected = selector(i)
    cur_key_shape, cur_value_shape = key_shapes[i], value_shapes[i]
    # Since keys and values are [batch, length, depth] we concatenate on axis=1.
    new_key_len = sum([key_shapes[j][1] for j in selected]) + cur_key_shape[1]
    new_key_shape = (cur_key_shape[0], new_key_len, cur_key_shape[2])
    new_value_len = sum(
        [value_shapes[j][1] for j in selected]) + cur_value_shape[1]
    new_value_shape = (cur_value_shape[0], new_value_len, cur_value_shape[2])
    # Masks are (1, query-len, key-len).
    new_mask_shape = (1, query_shapes[i][1], new_key_len)
    new_shape = ((query_shapes[i], new_key_shape, new_value_shape),
                 new_mask_shape)
    result.append(new_shape)
  return tuple(result)
831877
def idat(self, rows, packed=False):
        """"""
        # http://www.w3.org/TR/PNG/#11IDAT
        filt = Filter(self.bitdepth * self.planes,
                      self.interlace, self.height)
        data = bytearray()

        def byteextend(rowbytes):
            """Default extending data with bytes. Applying filter"""
            data.extend(filt.do_filter(self.filter_type, rowbytes))

        # Choose an extend function based on the bitdepth.  The extend
        # function packs/decomposes the pixel values into bytes and
        # stuffs them onto the data array.
        if self.bitdepth == 8 or packed:
            extend = byteextend
        elif self.bitdepth == 16:
            def extend(sl):
                """Decompose into bytes before byteextend"""
                fmt = '!%dH' % len(sl)
                byteextend(bytearray(struct.pack(fmt, *sl)))
        else:
            # Pack into bytes
            assert self.bitdepth < 8
            # samples per byte
            spb = 8 // self.bitdepth

            def extend(sl):
                """Pack into bytes before byteextend"""
                a = bytearray(sl)
                # Adding padding bytes so we can group into a whole
                # number of spb-tuples.
                l = float(len(a))
                extra = math.ceil(l / float(spb)) * spb - l
                a.extend([0] * int(extra))
                # Pack into bytes
                l = group(a, spb)
                l = [reduce(lambda x, y: (x << self.bitdepth) + y, e)
                     for e in l]
                byteextend(l)

        # Build the first row, testing mostly to see if we need to
        # changed the extend function to cope with NumPy integer types
        # (they cause our ordinary definition of extend to fail, so we
        # wrap it).  See
        # http://code.google.com/p/pypng/issues/detail?id=44
        enumrows = enumerate(rows)
        del rows

        # :todo: Certain exceptions in the call to ``.next()`` or the
        # following try would indicate no row data supplied.
        # Should catch.
        i, row = next(enumrows)
        try:
            # If this fails...
            extend(row)
        except:
            # ... try a version that converts the values to int first.
            # Not only does this work for the (slightly broken) NumPy
            # types, there are probably lots of other, unknown, "nearly"
            # int types it works for.
            def wrapmapint(f):
                return lambda sl: f([int(x) for x in sl])
            extend = wrapmapint(extend)
            del wrapmapint
            extend(row)

        for i, row in enumrows:
            extend(row)
            if len(data) > self.chunk_limit:
                yield bytearray_to_bytes(data)
                # Because of our very witty definition of ``extend``,
                # above, we must re-use the same ``data`` object.  Hence
                # we use ``del`` to empty this one, rather than create a
                # fresh one (which would be my natural FP instinct).
                del data[:]
        if len(data):
            yield bytearray_to_bytes(data)
        self.irows = i + 1
380294
def table_mask(self):
        """"""
        margin = compress_pruned(
            self._slice.margin(
                axis=None,
                weighted=False,
                include_transforms_for_dims=self._hs_dims,
                prune=self._prune,
            )
        )
        mask = margin < self._size

        if margin.shape == self._shape:
            return mask

        if self._slice.dim_types[0] == DT.MR:
            # If the margin is a column vector - broadcast it's mask to the array shape
            return np.logical_or(np.zeros(self._shape, dtype=bool), mask[:, None])

        return np.logical_or(np.zeros(self._shape, dtype=bool), mask)
726078
def polygon_to_mask(coords, dims, z=None):
    

    bounds = array(coords).astype('int')
    path = Path(bounds)

    grid = meshgrid(range(dims[1]), range(dims[0]))
    grid_flat = zip(grid[0].ravel(), grid[1].ravel())

    mask = path.contains_points(grid_flat).reshape(dims[0:2]).astype('int')

    if z is not None:
        if len(dims) < 3:
            raise Exception('Dims must have three-dimensions for embedding z-index')
        if z >= dims[2]:
            raise Exception('Z-index %g exceeds third dimension %g' % (z, dims[2]))
        tmp = zeros(dims)
        tmp[:, :, z] = mask
        mask = tmp

    return mask
374906
def _get_bolean_mask(self, mask):
        """"""
        if mask.dtype.kind == 'f':
            warnings.warn("mask image should be unsigned byte between 0"
                          " and 255. Got a float array")
        if mask.ndim == 2:
            boolean_mask = mask == 255
        elif mask.ndim == 3:
            # if all channels are white, mask out
            boolean_mask = np.all(mask[:, :, :3] == 255, axis=-1)
        else:
            raise ValueError("Got mask of invalid shape: %s" % str(mask.shape))
        return boolean_mask
372559
def segmentation_to_mask(polys, height, width):
    
    polys = [p.flatten().tolist() for p in polys]
    assert len(polys) > 0, "Polygons are empty!"

    import pycocotools.mask as cocomask
    rles = cocomask.frPyObjects(polys, height, width)
    rle = cocomask.merge(rles)
    return cocomask.decode(rle)
417458
def to_dict(self, trim=True):
        """"""
        if trim:
            return self._to_trimmed_dict()
        else:
            return dict(zip(DATASET_KEYS, self))
230498
def masked(a, b):
    

    if np.any([a.dtype.kind.startswith(c) for c in ['i', 'u', 'f', 'c']]):
        n = np.array([np.nan for i in range(len(a))])
    else:
        n = np.array([None for i in range(len(a))])
        # a = a.astype(object)
    return np.where(b, a, n)