Generate data for a given dataset.
357645
def download_and_prepare(builder):
  """"""
  print("download_and_prepare for dataset {}...".format(builder.info.full_name))

  dl_config = download_config()

  if isinstance(builder, tfds.core.BeamBasedBuilder):
    beam = tfds.core.lazy_imports.apache_beam
    # TODO(b/129149715): Restore compute stats. Currently skipped because not
    # beam supported.
    dl_config.compute_stats = tfds.download.ComputeStatsMode.SKIP
    dl_config.beam_options = beam.options.pipeline_options.PipelineOptions()

  builder.download_and_prepare(
      download_dir=FLAGS.download_dir,
      download_config=dl_config,
  )
  termcolor.cprint(str(builder.info.as_proto), attrs=["bold"])

  if FLAGS.debug:
    dataset = builder.as_dataset(split=tfds.Split.TRAIN)
    pdb.set_trace()
    del dataset
746584
def prepare_data(problem, hparams, params, config):
  """"""
  input_fn = problem.make_estimator_input_fn(
      tf.estimator.ModeKeys.EVAL, hparams, force_repeat=True)
  dataset = input_fn(params, config)
  features, _ = dataset.make_one_shot_iterator().get_next()
  inputs, labels = features["targets"], features["inputs"]
  inputs = tf.to_float(inputs)
  input_shape = inputs.shape.as_list()
  inputs = tf.reshape(inputs, [hparams.batch_size] + input_shape[1:])
  labels = tf.reshape(labels, [hparams.batch_size])
  return inputs, labels, features
1160255
def get_dataset(self, key, info):
        """"""

        angles = self._get_coarse_dataset(key, info)
        if angles is None:
            return

        # Fill gaps at edges of swath
        darr = DataArray(angles, dims=['y', 'x'])
        darr = darr.bfill('x')
        darr = darr.ffill('x')
        angles = darr.data

        res = self.interpolate_angles(angles, key.resolution)

        proj = DataArray(res, dims=['y', 'x'])
        proj.attrs = info.copy()
        proj.attrs['units'] = 'degrees'
        proj.attrs['platform_name'] = self.platform_name
        return proj
1256354
def save_datasets(self, writer="geotiff", datasets=None, compute=True, **kwargs):
        """"""
        if datasets is not None:
            datasets = [self[ds] for ds in datasets]
        else:
            datasets = [self.datasets.get(ds) for ds in self.wishlist]
            datasets = [ds for ds in datasets if ds is not None]
        if not datasets:
            raise RuntimeError("None of the requested datasets have been "
                               "generated or could not be loaded. Requested "
                               "composite inputs may need to have matching "
                               "dimensions (eg. through resampling).")
        writer, save_kwargs = load_writer(writer, ppp_config_dir=self.ppp_config_dir, **kwargs)
        return writer.save_datasets(datasets, compute=compute, **save_kwargs)
932212
def run(self, dataset_path):
        """"""
        features = self._generate_features(self._feature_extractors)
        features.to_csv(dataset_path)
1320440
def open_dataset(self, service):
        """"""

        if not self.dataset:
            path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
            self.dataset = netCDF4.Dataset(path, 'r')
        return self.dataset
1184000
def dataset(db=nil)
      db ||= self.class.default_db
      builder = Database::DatasetBuilder.new(db, self)
      builder.select(@columns)
      builder.filter(@filters)
      builder.order(@order)
      builder.limit(@limit) if @limit
      builder.dataset
    end
782918
def available_datasets(self):
        """"""
        res = self.resolution
        coordinates = ['pixel_longitude', 'pixel_latitude']
        for var_name, val in self.file_content.items():
            if isinstance(val, netCDF4.Variable):
                ds_info = {
                    'file_type': self.filetype_info['file_type'],
                    'resolution': res,
                }
                if not self.is_geo:
                    ds_info['coordinates'] = coordinates
                yield DatasetID(name=var_name, resolution=res), ds_info
406066
def properties(dataset_uri, item_identifier):
    """"""
    dataset = dtoolcore.DataSet.from_uri(dataset_uri)
    try:
        props = dataset.item_properties(item_identifier)
    except KeyError:
        click.secho(
            "No such item in dataset: {}".format(item_identifier),
            fg="red",
            err=True
        )
        sys.exit(20)

    json_lines = [
        '{',
        '  "relpath": "{}",'.format(props["relpath"]),
        '  "size_in_bytes": {},'.format(props["size_in_bytes"]),
        '  "utc_timestamp": {},'.format(props["utc_timestamp"]),
        '  "hash": "{}"'.format(props["hash"]),
        '}',
    ]
    formatted_json = "\n".join(json_lines)
    colorful_json = pygments.highlight(
        formatted_json,
        pygments.lexers.JsonLexer(),
        pygments.formatters.TerminalFormatter())
    click.secho(colorful_json, nl=False)
1286915
def addDataset(self, dataset):
        
        id_ = dataset.getId()
        self._datasetIdMap[id_] = dataset
        self._datasetNameMap[dataset.getLocalId()] = dataset
        self._datasetIds.append(id_)
657498
def generate_matching_datasets(self, data_slug):
        """"""
        tag = Tag.objects.filter(slug=data_slug)
        try:
            return tag[0].dataset_set.all().order_by('-uploaded_by')
        except IndexError:
            return None
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
1237390
def make_data():
    """"""
    a = { (1,1):.25, (1,2):.15, (1,3):.2,
          (2,1):.3,  (2,2):.3,  (2,3):.1,
          (3,1):.15, (3,2):.65, (3,3):.05,
          (4,1):.1,  (4,2):.05,  (4,3):.8
          }
    epsilon = 0.01
    I,p = multidict({1:5, 2:6, 3:8, 4:20})
    K,LB = multidict({1:.2, 2:.3, 3:.2})
    return I,K,a,p,epsilon,LB
1003029
def create_dataset(self, in_stream):
        
        assert XLSX_IMPORT
        from io import BytesIO
        xlsx_book = openpyxl.load_workbook(BytesIO(in_stream), read_only=True)

        dataset = tablib.Dataset()
        sheet = xlsx_book.active

        # obtain generator
        rows = sheet.rows
        dataset.headers = [cell.value for cell in next(rows)]

        for row in rows:
            row_values = [cell.value for cell in row]
            dataset.append(row_values)
        return dataset
427548
def CreateDataset(self):
    """"""
    body = {
        "datasetReference": {
            "datasetId": self.dataset_id,
            "description": "Data exported from GRR",
            "friendlyName": "GRRExportData",
            "projectId": self.project_id
        }
    }
    result = self.service.datasets().insert(
        projectId=self.project_id, body=body).execute()
    self.datasets[self.dataset_id] = result
    return result
74834
def generate(size, output, schema):
    """"""
    pii_data = randomnames.NameList(size)

    if schema is not None:
        raise NotImplementedError

    randomnames.save_csv(
        pii_data.names,
        [f.identifier for f in pii_data.SCHEMA.fields],
        output)
1135533
def generate_dataset(self, cdl_path):
        '''
        
        '''
        if '.cdl' in cdl_path:  # it's possible the filename doesn't have the .cdl extension
            ds_str = cdl_path.replace('.cdl', '.nc')
        else:
            ds_str = cdl_path + '.nc'
        subprocess.call(['ncgen', '-o', ds_str, cdl_path])
        return ds_str