Create a preprocessed dataset file by applying `preprocessing_queue`
       to `path_to_data`. The result will be stored in `outputpath`.
162352
def create_preprocessed_dataset(path_to_data, outputpath, preprocessing_queue):
    """"""
    # Log everything
    logging.info("Data soure %s", path_to_data)
    logging.info("Output will be stored in %s", outputpath)
    tmp = "Preprocessing Queue:\n"
    for preprocessing_class in preprocessing_queue:
        tmp += str(preprocessing_class) + "\n"
    logging.info(tmp)
    # Load from pickled file
    if not os.path.isfile(path_to_data):
        logging.info(("'%s' does not exist. Please either abort this script "
                      "or update the data location."), path_to_data)
        raw_dataset_path = utils.choose_raw_dataset()
        # Get project-relative path
        raw_dataset_path = "raw-datasets" + \
                           raw_dataset_path.split("raw-datasets")[1]
        print(raw_dataset_path)
        sys.exit()  # TODO: Update model!
    logging.info("Start loading data...")
    loaded = pickle.load(open(path_to_data, "rb"))
    raw_datasets = loaded['handwriting_datasets']
    logging.info("Start applying preprocessing methods")
    start_time = time.time()
    for i, raw_dataset in enumerate(raw_datasets):
        if i % 10 == 0 and i > 0:
            utils.print_status(len(raw_datasets), i, start_time)
        # Do the work
        raw_dataset['handwriting'].preprocessing(preprocessing_queue)
    sys.stdout.write("\r%0.2f%% (done)\033[K\n" % (100))
    print("")
    pickle.dump({'handwriting_datasets': raw_datasets,
                 'formula_id2latex': loaded['formula_id2latex'],
                 'preprocessing_queue': preprocessing_queue},
                open(outputpath, "wb"),
                2)
61553
def compile_data(tmp_dir, datasets, filename, datatypes_to_clean=None):
  """"""
  datatypes_to_clean = datatypes_to_clean or []
  filename = os.path.join(tmp_dir, filename)
  lang1_fname = filename + ".lang1"
  lang2_fname = filename + ".lang2"
  if tf.gfile.Exists(lang1_fname) and tf.gfile.Exists(lang2_fname):
    tf.logging.info("Skipping compile data, found files:\n%s\n%s", lang1_fname,
                    lang2_fname)
    return filename
  with tf.gfile.GFile(lang1_fname, mode="w") as lang1_resfile:
    with tf.gfile.GFile(lang2_fname, mode="w") as lang2_resfile:
      for dataset in datasets:
        url = dataset[0]
        compressed_filename = os.path.basename(url)
        compressed_filepath = os.path.join(tmp_dir, compressed_filename)
        if url.startswith("http"):
          generator_utils.maybe_download(tmp_dir, compressed_filename, url)

        if dataset[1][0] == "tmx":
          cleaning_requested = "tmx" in datatypes_to_clean
          tmx_filename = os.path.join(tmp_dir, dataset[1][1])
          if tmx_filename.endswith(".gz"):
            with gzip.open(tmx_filename, "rb") as tmx_file:
              _tmx_to_source_target(tmx_file, lang1_resfile, lang2_resfile,
                                    do_cleaning=cleaning_requested)
          else:
            with tf.gfile.Open(tmx_filename) as tmx_file:
              _tmx_to_source_target(tmx_file, lang1_resfile, lang2_resfile,
                                    do_cleaning=cleaning_requested)

        elif dataset[1][0] == "tsv":
          _, src_column, trg_column, glob_pattern = dataset[1]
          filenames = tf.gfile.Glob(os.path.join(tmp_dir, glob_pattern))
          if not filenames:
            # Capture *.tgz and *.tar.gz too.
            mode = "r:gz" if compressed_filepath.endswith("gz") else "r"
            with tarfile.open(compressed_filepath, mode) as corpus_tar:
              corpus_tar.extractall(tmp_dir)
            filenames = tf.gfile.Glob(os.path.join(tmp_dir, glob_pattern))
          for tsv_filename in filenames:
            if tsv_filename.endswith(".gz"):
              new_filename = tsv_filename.strip(".gz")
              generator_utils.gunzip_file(tsv_filename, new_filename)
              tsv_filename = new_filename
            with tf.gfile.Open(tsv_filename) as tsv_file:
              for line in tsv_file:
                if line and "\t" in line:
                  parts = line.split("\t")
                  source, target = parts[src_column], parts[trg_column]
                  source, target = source.strip(), target.strip()
                  clean_pairs = [(source, target)]
                  if "tsv" in datatypes_to_clean:
                    clean_pairs = cleaner_en_xx.clean_en_xx_pairs(clean_pairs)
                  for source, target in clean_pairs:
                    if source and target:
                      lang1_resfile.write(source)
                      lang1_resfile.write("\n")
                      lang2_resfile.write(target)
                      lang2_resfile.write("\n")

        else:
          lang1_filename, lang2_filename = dataset[1]
          lang1_filepath = os.path.join(tmp_dir, lang1_filename)
          lang2_filepath = os.path.join(tmp_dir, lang2_filename)
          is_sgm = (
              lang1_filename.endswith("sgm") and lang2_filename.endswith("sgm"))

          if not (tf.gfile.Exists(lang1_filepath) and
                  tf.gfile.Exists(lang2_filepath)):
            # For .tar.gz and .tgz files, we read compressed.
            mode = "r:gz" if compressed_filepath.endswith("gz") else "r"
            with tarfile.open(compressed_filepath, mode) as corpus_tar:
              corpus_tar.extractall(tmp_dir)
          if lang1_filepath.endswith(".gz"):
            new_filepath = lang1_filepath.strip(".gz")
            generator_utils.gunzip_file(lang1_filepath, new_filepath)
            lang1_filepath = new_filepath
          if lang2_filepath.endswith(".gz"):
            new_filepath = lang2_filepath.strip(".gz")
            generator_utils.gunzip_file(lang2_filepath, new_filepath)
            lang2_filepath = new_filepath

          for example in text_problems.text2text_txt_iterator(
              lang1_filepath, lang2_filepath):
            line1res = _preprocess_sgm(example["inputs"], is_sgm)
            line2res = _preprocess_sgm(example["targets"], is_sgm)
            clean_pairs = [(line1res, line2res)]
            if "txt" in datatypes_to_clean:
              clean_pairs = cleaner_en_xx.clean_en_xx_pairs(clean_pairs)
            for line1res, line2res in clean_pairs:
              if line1res and line2res:
                lang1_resfile.write(line1res)
                lang1_resfile.write("\n")
                lang2_resfile.write(line2res)
                lang2_resfile.write("\n")

  return filename
1141125
def prepare_to_generate(self, data_dir, tmp_dir):
    """"""
    self.get_or_create_vocab(data_dir, tmp_dir)
    self.train_text_filepaths(tmp_dir)
    self.dev_text_filepaths(tmp_dir)
830916
def upload(self, file_path, dataset=None, public=False):
        """"""

        upload_status = self.upload_file(file_path)
        err_msg = 'Dataset has not been uploaded to the remote host'
        if not upload_status.successful:
            msg = '{}, because of the following error: {}'.format(err_msg, upload_status.error)
            raise ValueError(msg)

        err_msg = 'File has not been verified'
        upload_ver_status = self.upload_verify(upload_status.properties.location, dataset)
        if not upload_ver_status.successful:
            ver_err = '\r\n'.join(upload_ver_status.errors)
            msg = '{}, because of the following error(s): {}'.format(err_msg, ver_err)
            raise ValueError(msg)

        ds_upload = definition.DatasetUpload(upload_ver_status, upload_status, dataset, public)
        ds_upload_submit_result = self.upload_submit(ds_upload)
        err_msg = 'Dataset has not been saved to the database'
        if ds_upload_submit_result.status == 'failed':
            ver_err = '\r\n'.join(ds_upload_submit_result.errors)
            msg = '{}, because of the following error(s): {}'.format(err_msg, ver_err)
            raise ValueError(msg)

        ds_upload_result = None
        while True:
            ds_upload_result = self.upload_status(ds_upload_submit_result.submit_id)
            if ds_upload_result.status == 'pending' or ds_upload_result.status == 'processing':
                time.sleep(5)
            else:
                break

        if ds_upload_result.status != 'successful':
            ver_err = '\r\n'.join(ds_upload_result.errors)
            msg = '{}, because of the following error(s): {}'.format(err_msg, ver_err)
            raise ValueError(msg)

        return ds_upload_result.dataset
1243393
public static <E> View<E> viewForPath(Dataset<E> dataset, Path path) {
    if (dataset instanceof FileSystemDataset) {
      return ((FileSystemDataset<E>) dataset).viewForUri(path.toUri());
    }
    throw new IllegalArgumentException("Not a file system dataset: " + dataset);
  }
668697
def preprocess_dataset(dataset, labels):
    
    start = time.time()
    with mp.Pool() as pool:
        # Each sample is processed in an asynchronous manner.
        dataset = gluon.data.SimpleDataset(list(zip(dataset, labels)))
        lengths = gluon.data.SimpleDataset(pool.map(get_length, dataset))
    end = time.time()
    logging.info('Done! Preprocessing Time={:.2f}s, #Sentences={}'
                 .format(end - start, len(dataset)))
    return dataset, lengths
395365
def data_preprocess(data, size=None, min_axis=None, batch=False):
    
    if batch:
        return [data_preprocess(el, size=size, min_axis=min_axis, batch=False) for el in data]

    if isinstance(data, string_types):
        if file_exists(data):
            # probably a path to an image
            preprocessed = Image.open(data)
        else:
            # base 64 encoded image data, a url, or raw content
            # send raw data to the server and let the server infer type
            b64_or_url = re.sub('^data:image/.+;base64,', '', data)
            return b64_or_url

    elif isinstance(data, Image.Image):
        # data is image from PIL
        preprocessed = data

    elif type(data).__name__ == "ndarray":
        # data is likely image from numpy/scipy
        if "float" in str(data.dtype) and data.min() >= 0 and data.max() <= 1:
            data *= 255.
        try:
            preprocessed = Image.fromarray(data.astype("uint8"))
        except TypeError:
            raise IndicoError(
                "Please ensure the numpy array is in a format by PIL. "
                "Values must be between 0 and 1 or between 0 and 255 in greyscale, rgb, or rgba format."
            )

    else:
        # at this point we are unsure of the type -- it could be malformatted text or image data.
        raise IndicoError(
            "Invalid input datatype: `{}`. "
            "Ensure input data is one of the following types: "
            "`str`, `unicode`, `PIL.Image`, `np.ndarray`.".format(
                data.__class__.__name__
            )
        )

    #
    if size or min_axis:
        preprocessed = resize_image(preprocessed, size, min_axis)

    # standardize on b64 encoding for sending image data over the wire
    temp_output = BytesIO()
    preprocessed.save(temp_output, format='PNG')
    temp_output.seek(0)
    output_s = temp_output.read()
    return base64.b64encode(output_s).decode('utf-8') if PY3 else base64.b64encode(output_s)
578618
protected boolean configureInputAndOutputPaths(Job job, FileSystemDataset dataset) throws IOException {
    boolean emptyDirectoryFlag = false;

    String mrOutputBase = this.state.getProp(MRCompactor.COMPACTION_JOB_DIR);
    CompactionPathParser parser = new CompactionPathParser(this.state);
    CompactionPathParser.CompactionParserResult rst = parser.parse(dataset);
    this.mrOutputPath = concatPaths(mrOutputBase, rst.getDatasetName(), rst.getDstSubDir(), rst.getTimeString());

    log.info("Cleaning temporary MR output directory: " + mrOutputPath);
    this.fs.delete(mrOutputPath, true);

    this.mapReduceInputPaths = getGranularInputPaths(dataset.datasetRoot());
    if (this.mapReduceInputPaths.isEmpty()) {
      this.mapReduceInputPaths.add(dataset.datasetRoot());
      emptyDirectoryFlag = true;
    }

    for (Path path : mapReduceInputPaths) {
      FileInputFormat.addInputPath(job, path);
    }

    FileOutputFormat.setOutputPath(job, mrOutputPath);
    return emptyDirectoryFlag;
  }
294346
def prep_samples_and_config(run_folder, ldetails, fastq_dir, config):
    
    fastq_final_dir = utils.safe_makedir(os.path.join(fastq_dir, "merged"))
    cores = utils.get_in(config, ("algorithm", "num_cores"), 1)
    ldetails = joblib.Parallel(cores)(joblib.delayed(_prep_sample_and_config)(x, fastq_dir, fastq_final_dir)
                                      for x in _group_same_samples(ldetails))
    config_file = _write_sample_config(run_folder, [x for x in ldetails if x])
    return config_file, fastq_final_dir
932964
def preprocess(train_dataset, output_dir, eval_dataset, checkpoint):
    """"""

    import apache_beam as beam
    from google.datalab.utils import LambdaJob
    from . import _preprocess

    if checkpoint is None:
      checkpoint = _util._DEFAULT_CHECKPOINT_GSURL
    job_id = ('preprocess-image-classification-' +
              datetime.datetime.now().strftime('%y%m%d-%H%M%S'))
    # Project is needed for bigquery data source, even in local run.
    options = {
        'project': _util.default_project(),
    }
    opts = beam.pipeline.PipelineOptions(flags=[], **options)
    p = beam.Pipeline('DirectRunner', options=opts)
    _preprocess.configure_pipeline(p, train_dataset, eval_dataset, checkpoint, output_dir, job_id)
    job = LambdaJob(lambda: p.run().wait_until_finish(), job_id)
    return job
542980
function preprocessorcery(infile, info, callback) {
  applicable(infile, info, (err, preprocessors) => {
    if (err) return callback(err);

    const q = queue(1);
    preprocessors.forEach((preprocessor) => {
      const outfile = newfile(infile);
      q.defer((next) => {
        preprocessor(infile, outfile, (err) => {
          if (err) return next(err);
          infile = outfile;
          next();
        });
      });
    });

    q.await((err) => {
      if (err) return callback(err);

      // infile has been changed to the output file by this point
      callback(null, infile);
    });
  });
}
1159025
def save(self, path="chain.dat"):
        
        # create dictionary for data
        dic = {}
        dic['chain'] = self._chain.tolist()
        dic['step_count'] = self._step_count.tolist()
        dic['n_samples'] = self._n_samples
        dic['n_accepted'] = self._n_accepted
        dic['x'] = self._X['x'].tolist()
        dic['f'] = self._X['f'].tolist()
        dic['J'] = self._X['J'].tolist()

        # write data to file
        file = open(path, 'w')
        json.dump(dic, file)
        file.close()
761322
def get_dataset_path(*args):
    
    d = os.environ.get('TENSORPACK_DATASET', None)
    if d is None:
        d = os.path.join(os.path.expanduser('~'), 'tensorpack_data')
        if execute_only_once():
            logger.warn("Env var $TENSORPACK_DATASET not set, using {} for datasets.".format(d))
        if not os.path.isdir(d):
            mkdir_p(d)
            logger.info("Created the directory {}.".format(d))
    assert os.path.isdir(d), d
    return os.path.join(d, *args)
1346240
def open_dataset(path, **kwargs):
    # type: (str, T.Any) -> xr.Dataset
    
    if 'engine' in kwargs and kwargs['engine'] != 'cfgrib':
        raise ValueError("only engine=='cfgrib' is supported")
    kwargs['engine'] = 'cfgrib'
    return xr.backends.api.open_dataset(path, **kwargs)
672199
def _dataset_create_resources(self):
        # type: () -> None
        

        if 'resources' in self.data:
            self.old_data['resources'] = self._copy_hdxobjects(self.resources, hdx.data.resource.Resource, 'file_to_upload')
            self.init_resources()
            self.separate_resources()
932146
def generate_dataset(path, variant='20m', outputpath="."):
    
    filename = os.path.join(outputpath, "movielens_%s.hdf5" % variant)

    if variant == '20m':
        ratings, movies = _read_dataframes_20M(path)
    elif variant == '100k':
        ratings, movies = _read_dataframes_100k(path)
    else:
        ratings, movies = _read_dataframes(path)

    _hfd5_from_dataframe(ratings, movies, filename)
712683
def _save_files(self, data, dtype_out_time):
        """"""
        path = self.path_out[dtype_out_time]
        if not os.path.isdir(self.dir_out):
            os.makedirs(self.dir_out)
        if 'reg' in dtype_out_time:
            try:
                reg_data = xr.open_dataset(path)
            except (EOFError, RuntimeError, IOError):
                reg_data = xr.Dataset()
            reg_data.update(data)
            data_out = reg_data
        else:
            data_out = data
        if isinstance(data_out, xr.DataArray):
            data_out = xr.Dataset({self.name: data_out})
        data_out.to_netcdf(path, engine='netcdf4', format='NETCDF3_64BIT')