Return a bundle build on a dataset, with the given vid or id reference
343834
def bundle(self, ref, capture_exceptions=False):
        """"""
        from ..orm.exc import NotFoundError

        if isinstance(ref, Dataset):
            ds = ref
        else:
            try:
                ds = self._db.dataset(ref)
            except NotFoundError:
                ds = None

        if not ds:
            try:
                p = self.partition(ref)
                ds = p._bundle.dataset
            except NotFoundError:
                ds = None

        if not ds:
            raise NotFoundError('Failed to find dataset for ref: {}'.format(ref))

        b = Bundle(ds, self)
        b.capture_exceptions = capture_exceptions

        return b
1231927
def get_image(dataset):
    """"""
    dim, nrow, ncol = dataset.shape
    uint8_dataset = dataset.astype('uint8')
    if not (uint8_dataset == dataset).all():
        message = (
            "\nYour image was cast to a `uint8` (`<img>.astype(uint8)`), "
            "but some information was lost.\nPlease check your gif and "
            "convert to uint8 beforehand if the gif looks wrong."
        )
        warnings.warn(message)
    image = [[
            struct.pack(
                'BBB',
                uint8_dataset[0, i, j],
                uint8_dataset[1, i, j],
                uint8_dataset[2, i, j]
            )
            for j in range(ncol)]
        for i in range(nrow)]
    return image
45528
def load(self, d3mds):
        """"""
        X, y = d3mds.get_data()

        resource_columns = d3mds.get_related_resources(self.data_modality)
        for resource_column in resource_columns:
            X = self.load_resources(X, resource_column, d3mds)

        context = self.get_context(X, y)

        return Dataset(d3mds.dataset_id, X, y, context=context)
254311
function Dataset(sc, dependencies) {
  this.id = sc.datasetIdCounter++;
  this.dependencies = dependencies || [];
  this.persistent = false;
  this.sc = sc;
}
421419
def get_bundle(self, current_bundle, url_kwargs, context_kwargs):
        
        if self.bundle_attr:
            if self.bundle_attr == PARENT:
                return current_bundle.parent

            view, name = current_bundle.get_view_and_name(self.bundle_attr)
            return view

        return current_bundle
752262
def get_dataset(self):
        # type: () -> hdx.data.dataset.Dataset
        
        package_id = self.data.get('package_id')
        if package_id is None:
            raise HDXError('Resource has no package id!')
        return hdx.data.dataset.Dataset.read_from_hdx(package_id)
629553
def package_dataset(self):
        """"""

        return self.session.query(Dataset).filter(Dataset.vid != ROOT_CONFIG_NAME_V).one()
176764
def _get_vqa_v2_image_raw_dataset(directory, image_root_url, image_urls):
  """"""
  for url in image_urls:
    filename = os.path.basename(url)
    download_url = os.path.join(image_root_url, url)
    path = generator_utils.maybe_download(directory, filename, download_url)
    unzip_dir = os.path.join(directory, filename.strip(".zip"))
    if not tf.gfile.Exists(unzip_dir):
      zipfile.ZipFile(path, "r").extractall(directory)
1117244
def canonical_dataset_to_grib(dataset, path, mode='wb', no_warn=False, grib_keys={}, **kwargs):
    # type: (xr.Dataset, str, str, bool, T.Dict[str, T.Any] T.Any) -> None
    
    if not no_warn:
        warnings.warn("GRIB write support is experimental, DO NOT RELY ON IT!", FutureWarning)

    # validate Dataset keys, DataArray names, and attr keys/values
    xr.backends.api._validate_dataset_names(dataset)
    xr.backends.api._validate_attrs(dataset)

    real_grib_keys = {k[5:]: v for k, v in dataset.attrs.items() if k[:5] == 'GRIB_'}
    real_grib_keys.update(grib_keys)

    with open(path, mode=mode) as file:
        for data_var in dataset.data_vars.values():
            canonical_dataarray_to_grib(data_var, file, grib_keys=real_grib_keys, **kwargs)
193137
def hydrate_bundles(bundles_field, glob_match_error_behavior):
  """"""
  path_globs_with_match_errors = [
    pg.copy(glob_match_error_behavior=glob_match_error_behavior)
    for pg in bundles_field.path_globs_list
  ]
  snapshot_list = yield [Get(Snapshot, PathGlobs, pg) for pg in path_globs_with_match_errors]

  spec_path = bundles_field.address.spec_path

  bundles = []
  zipped = zip(bundles_field.bundles,
               bundles_field.filespecs_list,
               snapshot_list)
  for bundle, filespecs, snapshot in zipped:
    rel_spec_path = getattr(bundle, 'rel_path', spec_path)
    kwargs = bundle.kwargs()
    # NB: We `include_dirs=True` because bundle filesets frequently specify directories in order
    # to trigger a (deprecated) default inclusion of their recursive contents. See the related
    # deprecation in `pants.backend.jvm.tasks.bundle_create`.
    kwargs['fileset'] = _eager_fileset_with_spec(rel_spec_path,
                                                 filespecs,
                                                 snapshot,
                                                 include_dirs=True)
    bundles.append(BundleAdaptor(**kwargs))
  yield HydratedField('bundles', bundles)
602305
function getReaddirAttributes(bundle) {
		debugLS('in getReaddirAttributes');
		var defered = q.defer();
		self.iReadMultiple([
			'FILE_IO_NAME_READ_LEN',
			'FILE_IO_ATTRIBUTES',
			'FILE_IO_SIZE',
		])
		.then(function(results) {
			debugLS('in getReaddirAttributes res', results);
			results.forEach(function(result) {
				if(!result.isErr) {
					bundle[result.data.name] = result.data;
				}
			});
			// console.log("HERE!@'");
			// bundle.FILE_IO_NAME_READ_LEN = 0;
			defered.resolve(bundle);
		}, function(err) {
			debugLS('in getReaddirAttributes err', err);
			bundle.isError = true;
			bundle.errorStep = 'getReaddirAttributes';
			bundle.error = modbusMap.getErrorInfo(err);
			bundle.errorCode = err;
			defered.reject(bundle);
		});
		return defered.promise;
	}
1167175
def load(app_id):
    """"""
    path_to_data = find_experiment_export(app_id)
    if path_to_data is None:
        raise IOError("Dataset {} could not be found.".format(app_id))

    return Data(path_to_data)
128448
def partition_by_vid(self, ref):
        """"""
        from ambry.orm import Partition

        p = self.session.query(Partition).filter(Partition.vid == str(ref)).first()
        if p:
            return self.wrap_partition(p)
        else:
            return None
33281
def download_attachments(self, dataset_identifier, content_type="json",
                             download_dir="~/sodapy_downloads"):
        '''
        
        '''
        metadata = self.get_metadata(dataset_identifier, content_type=content_type)
        files = []
        attachments = metadata['metadata'].get("attachments")
        if not attachments:
            logging.info("No attachments were found or downloaded.")
            return files

        download_dir = os.path.join(os.path.expanduser(download_dir), dataset_identifier)
        if not os.path.exists(download_dir):
            os.makedirs(download_dir)

        for attachment in attachments:
            file_path = os.path.join(download_dir, attachment["filename"])
            has_assetid = attachment.get("assetId", False)
            if has_assetid:
                base = _format_old_api_request(dataid=dataset_identifier)
                assetid = attachment["assetId"]
                resource = "{0}/files/{1}?download=true&filename={2}"\
                           .format(base, assetid, attachment["filename"])
            else:
                base = "/api/assets"
                assetid = attachment["blobId"]
                resource = "{0}/{1}?download=true".format(base, assetid)

            uri = "{0}{1}{2}".format(self.uri_prefix, self.domain, resource)
            _download_file(uri, file_path)
            files.append(file_path)

        logging.info("The following files were downloaded:\n\t{0}".format("\n\t".join(files)))
        return files
695317
def checkin_bundle(self, db_path, replace=True, cb=None):
        """"""
        from ambry.orm.exc import NotFoundError

        db = Database('sqlite:///{}'.format(db_path))
        db.open()

        if len(db.datasets) == 0:
            raise NotFoundError("Did not get a dataset in the {} bundle".format(db_path))


        ds = db.dataset(db.datasets[0].vid)  # There should only be one

        assert ds is not None
        assert ds._database

        try:
            b = self.bundle(ds.vid)
            self.logger.info(
                "Removing old bundle before checking in new one of same number: '{}'"
                .format(ds.vid))
            self.remove(b)
        except NotFoundError:
            pass

        try:
            self.dataset(ds.vid)  # Skip loading bundles we already have
        except NotFoundError:
            self.database.copy_dataset(ds, cb=cb)

        b = self.bundle(ds.vid)  # It had better exist now.
        # b.state = Bundle.STATES.INSTALLED
        b.commit()

        #self.search.index_library_datasets(tick)

        self.search.index_bundle(b)

        return b
115392
def build_from_source(version, **kwargs):
        
        mvn = os.path.join(Spark.svm_version_path(version), 'build', 'mvn')
        Spark.chmod_add_excute(mvn)
        p = subprocess.Popen([mvn, '-DskipTests', 'clean', 'package'], cwd=Spark.svm_version_path(version))
        p.wait()
        return p.returncode
400253
def include(context, bundle_name, version):
    
    store = Store(context.obj['database'], context.obj['root'])
    if version:
        version_obj = store.Version.get(version)
        if version_obj is None:
            click.echo(click.style('version not found', fg='red'))
    else:
        bundle_obj = store.bundle(bundle_name)
        if bundle_obj is None:
            click.echo(click.style('bundle not found', fg='red'))
        version_obj = bundle_obj.versions[0]

    try:
        include_version(context.obj['root'], version_obj)
    except VersionIncludedError as error:
        click.echo(click.style(error.message, fg='red'))
        context.abort()

    version_obj.included_at = dt.datetime.now()
    store.commit()
    click.echo(click.style('included all files!', fg='green'))