Create MR job properties for a {@link Dataset}.
764875
protected Optional<Dataset> createJobProps(Dataset dataset) throws IOException {
    if (this.recompactFromOutputPaths && (!latePathsFound(dataset))) {
      LOG.info(String.format("Skipping recompaction for %s since there is no late data in %s",
          new Object[] { dataset.inputPaths(), dataset.inputLatePaths() }));
      return Optional.absent();
    }

    State jobProps = new State();
    jobProps.addAll(this.state);
    jobProps.setProp(MRCompactor.COMPACTION_ENABLE_SUCCESS_FILE, false);
    jobProps.setProp(MRCompactor.COMPACTION_INPUT_DEDUPLICATED, this.inputDeduplicated);
    jobProps.setProp(MRCompactor.COMPACTION_OUTPUT_DEDUPLICATED, this.outputDeduplicated);
    jobProps.setProp(MRCompactor.COMPACTION_SHOULD_DEDUPLICATE, !this.inputDeduplicated && this.outputDeduplicated);

    if (this.recompactFromOutputPaths || !MRCompactor.datasetAlreadyCompacted(this.fs, dataset, renameSourceDirEnabled)) {
      if (renameSourceDirEnabled) {
        Set<Path> newUnrenamedDirs = MRCompactor.getDeepestLevelUnrenamedDirsWithFileExistence(this.fs, dataset.inputPaths());
        if (getAllFilePathsRecursively(newUnrenamedDirs).isEmpty()) {
          return Optional.absent();
        }
        LOG.info ("[{}] has unprocessed directories for first time compaction: {}", dataset.getDatasetName(), newUnrenamedDirs);
        dataset.overwriteInputPaths(newUnrenamedDirs);
        dataset.setRenamePaths(newUnrenamedDirs);
      } else {
        addInputLateFilesForFirstTimeCompaction(jobProps, dataset);
      }

      LOG.info(String.format("Created MR job properties for input %s and output %s.", dataset.inputPaths(),
              dataset.outputPath()));
      dataset.setJobProps(jobProps);
      return Optional.of(dataset);
    } else {
      return obtainDatasetWithJobProps (jobProps, dataset);
    }
  }
51884
@Override
  JobDesc create(QualifiedJobId qualifiedJobId, long submitTimeMillis,
      Configuration jobConf) {
    // TODO: Get the actual values appropriate for the plain Hadoop jobs.

    String appId = getAppId(jobConf);

    long appSubmitTimeMillis = jobConf.getLong(Constants.MR_RUN_CONF_KEY,
        submitTimeMillis);


    return create(qualifiedJobId, jobConf, appId, Constants.UNKNOWN,
        Framework.NONE, appSubmitTimeMillis);
  }
980335
private MRCompactorJobRunner getMRCompactorJobRunner(Dataset dataset) {
    try {
      @SuppressWarnings("unchecked")
      Class<? extends MRCompactorJobRunner> cls = (Class<? extends MRCompactorJobRunner>) Class
          .forName(this.state.getProp(COMPACTION_JOB_RUNNER_CLASS, DEFAULT_COMPACTION_JOB_RUNNER_CLASS));
      return cls.getDeclaredConstructor(Dataset.class, FileSystem.class).newInstance(dataset, this.fs);
    } catch (Exception e) {
      throw new RuntimeException("Cannot instantiate MRCompactorJobRunner", e);
    }
  }
134033
protected final void setUp(DataSet dataSet, Set<Integer> categoricalToRemove, Set<Integer> numericalToRemove)
    {
        for(int i : categoricalToRemove)
            if (i >= dataSet.getNumCategoricalVars())
                throw new RuntimeException("The data set does not have a categorical value " + i + " to remove");
        for(int i : numericalToRemove)
            if (i >= dataSet.getNumNumericalVars())
                throw new RuntimeException("The data set does not have a numercal value " + i + " to remove");
        
        catIndexMap = new int[dataSet.getNumCategoricalVars()-categoricalToRemove.size()];
        newCatHeader = new CategoricalData[catIndexMap.length];
        numIndexMap = new int[dataSet.getNumNumericalVars()-numericalToRemove.size()];
        int k = 0;
        for(int i = 0; i < dataSet.getNumCategoricalVars(); i++)
        {
            if(categoricalToRemove.contains(i))
                continue;
            newCatHeader[k] = dataSet.getCategories()[i].clone();
            catIndexMap[k++] = i;
        }
        k = 0;
        for(int i = 0; i < dataSet.getNumNumericalVars(); i++)
        {
            if(numericalToRemove.contains(i))
                continue;
            numIndexMap[k++] = i;
        }
    }
975030
public void setJobs(java.util.Collection<JobSummary> jobs) {
        if (jobs == null) {
            this.jobs = null;
            return;
        }

        this.jobs = new java.util.ArrayList<JobSummary>(jobs);
    }
199055
public DataSetInfo create(int dataSet) throws InvalidDataSetException {
		DataSetInfo info = dataSets.get(createKey(dataSet));
		if (info == null) {
			int recordNumber = (dataSet >> 8) & 0xFF;
			int dataSetNumber = dataSet & 0xFF;
			throw new UnsupportedDataSetException(recordNumber + ":" + dataSetNumber);
			// info = super.create(dataSet);
		}
		return info;
	}
1122393
public Job copyTableId(String dataset, String tableName) throws BigQueryException {
    // [START bigquery_copy_table]
    TableId destinationId = TableId.of(dataset, tableName);
    JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL);
    Job job = table.copy(destinationId, options);
    // Wait for the job to complete.
    try {
      Job completedJob =
          job.waitFor(
              RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
              RetryOption.totalTimeout(Duration.ofMinutes(3)));
      if (completedJob != null && completedJob.getStatus().getError() == null) {
        // Job completed successfully.
      } else {
        // Handle error case.
      }
    } catch (InterruptedException e) {
      // Handle interrupted wait
    }
    // [END bigquery_copy_table]
    return job;
  }
574669
public DataSet createDataSet(final Message response, final MessageType messageType) {
        try {
            if (response.getPayload() instanceof DataSet) {
                return response.getPayload(DataSet.class);
            } else if (isReadyToMarshal(response, messageType)) {
                return marshalResponse(response, messageType);
            } else {
                return new DataSet();
            }
        } catch (final SQLException e) {
            throw new CitrusRuntimeException("Failed to read dataSet from response message", e);
        }
    }
134411
private static JobConf createJobConf(Configuration conf) {
    JobConf jobconf = new JobConf(conf, DistRaid.class);
    jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
    jobconf.setUser(RaidNode.JOBUSER);
    jobconf.setJobName(jobName);
    jobconf.setMapSpeculativeExecution(false);
    RaidUtils.parseAndSetOptions(jobconf, SCHEDULER_OPTION_LABEL);

    jobconf.setJarByClass(DistRaid.class);
    jobconf.setInputFormat(DistRaidInputFormat.class);
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(Text.class);

    jobconf.setMapperClass(DistRaidMapper.class);
    jobconf.setNumReduceTasks(0);
    return jobconf;
  }
858598
def create(cls, train_ds:Dataset, valid_ds:Dataset, test_ds:Optional[Dataset]=None, path:PathOrStr='.', bs:int=64,
               val_bs:int=None, num_workers:int=defaults.cpus, dl_tfms:Optional[Collection[Callable]]=None,
               device:torch.device=None, collate_fn:Callable=data_collate, no_check:bool=False, **dl_kwargs)->'DataBunch':
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        dls = [DataLoader(d, b, shuffle=s, drop_last=s, num_workers=num_workers, **dl_kwargs) for d,b,s in
               zip(datasets, (bs,val_bs,val_bs,val_bs), (True,False,False,False)) if d is not None]
        return cls(*dls, path=path, device=device, dl_tfms=dl_tfms, collate_fn=collate_fn, no_check=no_check)
984208
@Override
  public void setupHadoopJobProperties() {
    if (getJobProps().containsKey(HADOOP_GLOBAL_OPTS)) {
      String hadoopGlobalOps = getJobProps().getString(HADOOP_GLOBAL_OPTS);
      if (getJobProps().containsKey(HADOOP_OPTS)) {
        String hadoopOps = getJobProps().getString(HADOOP_OPTS);
        getJobProps().put(HADOOP_OPTS, String.format("%s %s", hadoopOps, hadoopGlobalOps));
      } else {
        getJobProps().put(HADOOP_OPTS, hadoopGlobalOps);
      }
    }
  }
653347
def populateFromRow(self, dataset):
        
        self._description = dataset.description
        self.setAttributesJson(dataset.attributes)
86557
@SneakyThrows
    public DataSource createDataSource() {
        DataSource result = (DataSource) Class.forName(dataSourceClassName).newInstance();
        Method[] methods = result.getClass().getMethods();
        for (Entry<String, Object> entry : properties.entrySet()) {
            if (SKIPPED_PROPERTY_NAMES.contains(entry.getKey())) {
                continue;
            }
            Optional<Method> setterMethod = findSetterMethod(methods, entry.getKey());
            if (setterMethod.isPresent()) {
                setterMethod.get().invoke(result, entry.getValue());
            }
        }
        return result;
    }
956405
def dataset(ctx, client, revision, datadir, format):
    """"""
    ctx.meta['renku.datasets.datadir'] = datadir

    if ctx.invoked_subcommand is not None:
        return

    if revision is None:
        datasets = client.datasets.values()
    else:
        datasets = client.datasets_from_commit(client.repo.commit(revision))

    DATASETS_FORMATS[format](client, datasets)
427548
def CreateDataset(self):
    """"""
    body = {
        "datasetReference": {
            "datasetId": self.dataset_id,
            "description": "Data exported from GRR",
            "friendlyName": "GRRExportData",
            "projectId": self.project_id
        }
    }
    result = self.service.datasets().insert(
        projectId=self.project_id, body=body).execute()
    self.datasets[self.dataset_id] = result
    return result
587463
def write_data_set(file_path, dataset, output_dir, color_array_info, new_name=None, compress=True):
    """"""
    fileName = new_name if new_name else os.path.basename(file_path)
    dataset_dir = os.path.join(output_dir, fileName)
    data_dir = os.path.join(dataset_dir, 'data')

    if not os.path.exists(data_dir):
        os.makedirs(data_dir)

    root = {}
    root['metadata'] = {}
    root['metadata']['name'] = fileName

    writer = writer_mapping[dataset.GetClassName()]
    if writer:
        writer(dataset_dir, data_dir, dataset, color_array_info, root, compress)
    else:
        print(dataObject.GetClassName(), 'is not supported')

    with open(os.path.join(dataset_dir, "index.json"), 'w') as f:
        f.write(json.dumps(root, indent=2))

    return dataset_dir
52280
public static SequenceBatchCSVRecord fromDataSet(MultiDataSet dataSet) {
        SequenceBatchCSVRecord batchCSVRecord = new SequenceBatchCSVRecord();
        for (int i = 0; i < dataSet.numFeatureArrays(); i++) {
            batchCSVRecord.add(Arrays.asList(BatchCSVRecord.fromDataSet(new DataSet(dataSet.getFeatures(i),dataSet.getLabels(i)))));
        }

        return batchCSVRecord;
    }