Helper function for diskwarp of multiple input GDAL Datasets
511053
def diskwarp_multi(src_ds_list, res='first', extent='intersection', t_srs='first', r='cubic', verbose=True, outdir=None, dst_ndv=None):
    
    return warp_multi(src_ds_list, res, extent, t_srs, r, verbose=verbose, warptype=diskwarp, outdir=outdir, dst_ndv=dst_ndv)
720851
def _convert_slice_incement_inconsistencies(dicom_input):
    

    #   Estimate the "first" slice increment based on the 2 first slices
    increment = numpy.array(dicom_input[0].ImagePositionPatient) - numpy.array(dicom_input[1].ImagePositionPatient)

    # Create as many volumes as many changes in slice increment. NB Increments might be repeated in different volumes
    max_slice_increment = 0
    slice_incement_groups = []
    current_group = [dicom_input[0], dicom_input[1]]
    previous_image_position = numpy.array(dicom_input[1].ImagePositionPatient)
    for dicom in dicom_input[2:]:
        current_image_position = numpy.array(dicom.ImagePositionPatient)
        current_increment = previous_image_position - current_image_position
        max_slice_increment = max(max_slice_increment, numpy.linalg.norm(current_increment))
        if numpy.allclose(increment, current_increment, rtol=0.05, atol=0.1):
            current_group.append(dicom)
        if not numpy.allclose(increment, current_increment, rtol=0.05, atol=0.1):
            slice_incement_groups.append(current_group)
            current_group = [current_group[-1], dicom]
            increment = current_increment
        previous_image_position = current_image_position
    slice_incement_groups.append(current_group)

    # Create nibabel objects for each volume based on the corresponding headers
    slice_incement_niftis = []
    for dicom_slices in slice_incement_groups:
        data = common.get_volume_pixeldata(dicom_slices)
        affine, _ = common.create_affine(dicom_slices)
        slice_incement_niftis.append(nibabel.Nifti1Image(data, affine))

    nifti_volume = resample.resample_nifti_images(slice_incement_niftis)

    return nifti_volume, max_slice_increment
211451
def gdaOnes(shape, dtype, numGhosts=1):
    
    res = GhostedDistArray(shape, dtype)
    res.setNumberOfGhosts(numGhosts)
    res[:] = 1
    return res
1149667
def distorted_inputs():
  
  if not FLAGS.data_dir:
    raise ValueError('Please supply a data_dir')
  data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
  images, labels = cifar10_input.distorted_inputs(data_dir=data_dir,
                                                  batch_size=FLAGS.batch_size)
  if FLAGS.use_fp16:
    images = tf.cast(images, tf.float16)
    labels = tf.cast(labels, tf.float16)
  return images, labels
127999
def _resample(self, arrays, ji_windows):
        """"""
        # get a destination array template
        win_dst = ji_windows[self.dst_res]
        aff_dst = self._layer_meta[self._res_indices[self.dst_res][0]]["transform"]
        arrays_dst = list()
        for i, array in enumerate(arrays):
            arr_dst = np.zeros((int(win_dst.height), int(win_dst.width)))
            if self._layer_resolution[i] > self.dst_res:
                resampling = getattr(Resampling, self.upsampler)
            elif self._layer_resolution[i] < self.dst_res:
                resampling = getattr(Resampling, self.downsampler)
            else:
                arrays_dst.append(array.copy())
                continue
            reproject(array, arr_dst,  # arr_dst[0, :, :, i],
                      src_transform=self._layer_meta[i]["transform"],
                      dst_transform=aff_dst,
                      src_crs=self._layer_meta[0]["crs"],
                      dst_crs=self._layer_meta[0]["crs"],
                      resampling=resampling)
            arrays_dst.append(arr_dst.copy())
        arrays_dst = np.stack(arrays_dst, axis=2) # n_images x n x m x 10 would be the synergise format
        return arrays_dst
931696
def _incomplete_files(filenames):
  """"""
  tmp_files = [get_incomplete_path(f) for f in filenames]
  try:
    yield tmp_files
    for tmp, output in zip(tmp_files, filenames):
      tf.io.gfile.rename(tmp, output)
  finally:
    for tmp in tmp_files:
      if tf.io.gfile.exists(tmp):
        tf.io.gfile.remove(tmp)
456269
def _get_input_files(samples, base_dir, tx_out_dir):
    
    in_files = collections.defaultdict(list)
    for data in samples:
        sum_qc = tz.get_in(["summary", "qc"], data, {})
        if sum_qc in [None, "None"]:
            sum_qc = {}
        elif isinstance(sum_qc, six.string_types):
            sum_qc = {dd.get_algorithm_qc(data)[0]: sum_qc}
        elif not isinstance(sum_qc, dict):
            raise ValueError("Unexpected summary qc: %s" % sum_qc)
        for program, pfiles in sum_qc.items():
            if isinstance(pfiles, dict):
                pfiles = [pfiles["base"]] + pfiles.get("secondary", [])
            # CWL: presents output files as single file plus associated secondary files
            elif isinstance(pfiles, six.string_types):
                if os.path.exists(pfiles):
                    pfiles = [os.path.join(basedir, f) for basedir, subdir, filenames in os.walk(os.path.dirname(pfiles)) for f in filenames]
                else:
                    pfiles = []
            in_files[(dd.get_sample_name(data), program)].extend(pfiles)
    staged_files = []
    for (sample, program), files in in_files.items():
        cur_dir = utils.safe_makedir(os.path.join(base_dir, "inputs", sample, program))
        for f in files:
            if _check_multiqc_input(f) and _is_good_file_for_multiqc(f):
                if _in_temp_directory(f) or any([cwlutils.is_cwl_run(d) for d in samples]):
                    staged_f = os.path.join(cur_dir, os.path.basename(f))
                    shutil.copy(f, staged_f)
                    staged_files.append(staged_f)
                else:
                    staged_files.append(f)
    staged_files.extend(get_qsig_multiqc_files(samples))
    # Back compatible -- to migrate to explicit specifications in input YAML
    if not any([cwlutils.is_cwl_run(d) for d in samples]):
        staged_files += ["trimmed", "htseq-count/*summary"]
        # Add in created target_info file
        if os.path.isfile(os.path.join(base_dir, "report", "metrics", "target_info.yaml")):
            staged_files += [os.path.join(base_dir, "report", "metrics", "target_info.yaml")]
    return sorted(list(set(staged_files)))
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
573103
def _calc_concat_over(datasets, dim, data_vars, coords):
    
    # Return values
    concat_over = set()
    equals = {}

    if dim in datasets[0]:
        concat_over.add(dim)
    for ds in datasets:
        concat_over.update(k for k, v in ds.variables.items()
                           if dim in v.dims)

    def process_subset_opt(opt, subset):
        if isinstance(opt, str):
            if opt == 'different':
                # all nonindexes that are not the same in each dataset
                for k in getattr(datasets[0], subset):
                    if k not in concat_over:
                        # Compare the variable of all datasets vs. the one
                        # of the first dataset. Perform the minimum amount of
                        # loads in order to avoid multiple loads from disk
                        # while keeping the RAM footprint low.
                        v_lhs = datasets[0].variables[k].load()
                        # We'll need to know later on if variables are equal.
                        computed = []
                        for ds_rhs in datasets[1:]:
                            v_rhs = ds_rhs.variables[k].compute()
                            computed.append(v_rhs)
                            if not v_lhs.equals(v_rhs):
                                concat_over.add(k)
                                equals[k] = False
                                # computed variables are not to be re-computed
                                # again in the future
                                for ds, v in zip(datasets[1:], computed):
                                    ds.variables[k].data = v.data
                                break
                        else:
                            equals[k] = True

            elif opt == 'all':
                concat_over.update(set(getattr(datasets[0], subset)) -
                                   set(datasets[0].dims))
            elif opt == 'minimal':
                pass
            else:
                raise ValueError("unexpected value for %s: %s" % (subset, opt))
        else:
            invalid_vars = [k for k in opt
                            if k not in getattr(datasets[0], subset)]
            if invalid_vars:
                if subset == 'coords':
                    raise ValueError(
                        'some variables in coords are not coordinates on '
                        'the first dataset: %s' % (invalid_vars,))
                else:
                    raise ValueError(
                        'some variables in data_vars are not data variables '
                        'on the first dataset: %s' % (invalid_vars,))
            concat_over.update(opt)

    process_subset_opt(data_vars, 'data_vars')
    process_subset_opt(coords, 'coords')
    return concat_over, equals
321420
def _diff_image(slice1, slice2,
                abs_value=True,
                cmap='gray',
                **kwargs):
    """"""

    diff = slice1 - slice2

    if abs_value:
        diff = np.abs(diff)

    return diff, cmap
65421
function prettifyDiffOutput(diffArr) {
    const res = [];
    for (const diff of diffArr) {
        res.push(prettifyDiffObj(diff));
    }
    return res;
}
897338
def writeMNIST(sc, input_images, input_labels, output, format, num_partitions):
  """"""
  # load MNIST gzip into memory
  with open(input_images, 'rb') as f:
    images = numpy.array(mnist.extract_images(f))

  with open(input_labels, 'rb') as f:
    if format == "csv2":
      labels = numpy.array(mnist.extract_labels(f, one_hot=False))
    else:
      labels = numpy.array(mnist.extract_labels(f, one_hot=True))

  shape = images.shape
  print("images.shape: {0}".format(shape))          # 60000 x 28 x 28
  print("labels.shape: {0}".format(labels.shape))   # 60000 x 10

  # create RDDs of vectors
  imageRDD = sc.parallelize(images.reshape(shape[0], shape[1] * shape[2]), num_partitions)
  labelRDD = sc.parallelize(labels, num_partitions)

  output_images = output + "/images"
  output_labels = output + "/labels"

  # save RDDs as specific format
  if format == "pickle":
    imageRDD.saveAsPickleFile(output_images)
    labelRDD.saveAsPickleFile(output_labels)
  elif format == "csv":
    imageRDD.map(toCSV).saveAsTextFile(output_images)
    labelRDD.map(toCSV).saveAsTextFile(output_labels)
  elif format == "csv2":
    imageRDD.map(toCSV).zip(labelRDD).map(lambda x: str(x[1]) + "|" + x[0]).saveAsTextFile(output)
  else:  # format == "tfr":
    tfRDD = imageRDD.zip(labelRDD).map(lambda x: (bytearray(toTFExample(x[0], x[1])), None))
    # requires: --jars tensorflow-hadoop-1.0-SNAPSHOT.jar
    tfRDD.saveAsNewAPIHadoopFile(output, "org.tensorflow.hadoop.io.TFRecordFileOutputFormat",
                                 keyClass="org.apache.hadoop.io.BytesWritable",
                                 valueClass="org.apache.hadoop.io.NullWritable")
1312137
def compress(samples, run_parallel):
    
    to_cram = []
    finished = []
    for data in [x[0] for x in samples]:
        if "cram" in dd.get_archive(data) or "cram-lossless" in dd.get_archive(data):
            to_cram.append([data])
        else:
            finished.append([data])
    crammed = run_parallel("archive_to_cram", to_cram)
    return finished + crammed
594673
def dataset_to_stream(dataset, input_name, num_chunks=0, append_targets=False):
  """"""
  for example in tfds.as_numpy(dataset):
    inp, out = example[0][input_name], example[1]
    if len(out.shape) > 1 and out.shape[-1] == 1:
      out = np.squeeze(out, axis=-1)
    if num_chunks > 0:
      inp = np.split(inp, num_chunks, axis=1)
      out = np.split(out, num_chunks, axis=1)
    if append_targets:
      inp = (inp, out)
    yield inp, out
968518
def merge(self, datasets=None, separate_datasets=False):
        """"""
        self.logger.info("merging")
        if separate_datasets:
            warnings.warn("The option seperate_datasets=True is"
                          "not implemented yet. Performing merging, but"
                          "neglecting the option.")
        else:
            if datasets is None:
                datasets = list(range(len(self.datasets)))
            first = True
            for dataset_number in datasets:
                if first:
                    dataset = self.datasets[dataset_number]
                    first = False
                else:
                    dataset = self._append(dataset, self.datasets[dataset_number])
                    for raw_data_file, file_size in zip(self.datasets[dataset_number].raw_data_files,
                                                        self.datasets[dataset_number].raw_data_files_length):
                        dataset.raw_data_files.append(raw_data_file)
                        dataset.raw_data_files_length.append(file_size)
            self.datasets = [dataset]
            self.number_of_datasets = 1
        return self
1114635
def rm_compressed(ctx, dataset, kwargs):
    ""

    kwargs = parse_kwargs(kwargs)
    data(dataset, **ctx.obj).rm_compressed(**kwargs)
160629
def makecsvdiffs(thediffs, dtls, n1, n2):
    """"""
    def ishere(val):
        if val == None:
            return "not here"
        else:
            return "is here"
    rows = []
    rows.append(['file1 = %s' % (n1, )])
    rows.append(['file2 = %s' % (n2, )])
    rows.append('')
    rows.append(theheader(n1, n2))
    keys = list(thediffs.keys()) # ensures sorting by Name
    keys.sort()
    # sort the keys in the same order as in the idd
    dtlssorter = DtlsSorter(dtls)
    keys = sorted(keys, key=dtlssorter.getkey)
    for key in keys:
        if len(key) == 2:
            rw2 = [''] + [ishere(i) for i in thediffs[key]]
        else:
            rw2 = list(thediffs[key])
        rw1 = list(key)
        rows.append(rw1 + rw2)
    return rows