Concatenate HDF5 Files
7866
def ptconcat(output_file, input_files, overwrite=False):
    """"""
    filt = tb.Filters(
        complevel=5, shuffle=True, fletcher32=True, complib='zlib'
    )
    out_tabs = {}
    dt_file = input_files[0]
    log.info("Reading data struct '%s'..." % dt_file)
    h5struc = tb.open_file(dt_file, 'r')
    log.info("Opening output file '%s'..." % output_file)
    if overwrite:
        outmode = 'w'
    else:
        outmode = 'a'
    h5out = tb.open_file(output_file, outmode)

    for node in h5struc.walk_nodes('/', classname='Table'):
        path = node._v_pathname
        log.debug(path)
        dtype = node.dtype
        p, n = os.path.split(path)
        out_tabs[path] = h5out.create_table(
            p, n, description=dtype, filters=filt, createparents=True
        )
    h5struc.close()
    for fname in input_files:
        log.info('Reading %s...' % fname)
        h5 = tb.open_file(fname)
        for path, out in out_tabs.items():
            tab = h5.get_node(path)
            out.append(tab[:])
        h5.close()
    h5out.close()
1138298
def _add_combine_info(output, combine_map, file_key):
    
    files_per_output = collections.defaultdict(list)
    for part_file, out_file in combine_map.items():
        files_per_output[out_file].append(part_file)
    out_by_file = collections.defaultdict(list)
    out = []
    for data in output:
        # Do not pass along nochrom, noanalysis regions
        if data["region"][0] not in ["nochrom", "noanalysis"]:
            cur_file = data[file_key]
            # If we didn't process, no need to add combine information
            if cur_file in combine_map:
                out_file = combine_map[cur_file]
                if "combine" not in data:
                    data["combine"] = {}
                data["combine"][file_key] = {"out": out_file,
                                             "extras": files_per_output.get(out_file, [])}
                out_by_file[out_file].append(data)
            elif cur_file:
                out_by_file[cur_file].append(data)
            else:
                out.append([data])
    for samples in out_by_file.values():
        regions = [x["region"] for x in samples]
        region_bams = [x["work_bam"] for x in samples]
        assert len(regions) == len(region_bams)
        if len(set(region_bams)) == 1:
            region_bams = [region_bams[0]]
        data = samples[0]
        data["region_bams"] = region_bams
        data["region"] = regions
        data = dd.set_mark_duplicates(data, data["config"]["algorithm"]["orig_markduplicates"])
        del data["config"]["algorithm"]["orig_markduplicates"]
        out.append([data])
    return out
1053270
def fast_combine_pairs(files, force_single, full_name, separators):
    
    files = sort_filenames(files)
    chunks = tz.sliding_window(10, files)
    pairs = [combine_pairs(chunk, force_single, full_name, separators) for chunk in chunks]
    pairs = [y for x in pairs for y in x]
    longest = defaultdict(list)
    # for each file, save the longest pair it is in
    for pair in pairs:
        for file in pair:
            if len(longest[file]) < len(pair):
                longest[file] = pair
    # keep only unique pairs
    longest = {tuple(sort_filenames(x)) for x in longest.values()}
    # ensure filenames are R1 followed by R2
    return [sort_filenames(list(x)) for x in longest]
134895
def _save(self, hdf5, model, positives, negatives):
    """"""
    # write the model and the training set indices to the given HDF5 file
    hdf5.set("PositiveIndices", sorted(list(positives)))
    hdf5.set("NegativeIndices", sorted(list(negatives)))
    hdf5.create_group("Model")
    hdf5.cd("Model")
    model.save(hdf5)
    del hdf5
200580
def smartDumpDictHdf5(RV,o):
    
    for key in list(RV.keys()):
        if type(RV[key])==dict:
            g = o.create_group(key)
            smartDumpDictHdf5(RV[key],g)
        else:
            o.create_dataset(name=key,data=SP.array(RV[key]),chunks=True,compression='gzip')
1242686
def save_dict_to_hdf5(dic, filename):
    
    with h5py.File(filename, 'w') as h5file:
        recursively_save_dict_contents_to_group(h5file, '/', dic)
612026
def write_hdf5_cells(E, filename):
    ''' '''

    E = E.tocsr()

    hf = h5py.File(filename, 'w')
    counts_group = hf.create_group('counts')
    gix_group = hf.create_group('gene_ix')

    hf.attrs['ncells'] = E.shape[0]
    hf.attrs['ngenes'] = E.shape[1]

    for iC in range(E.shape[0]):
        counts = E[iC,:].A.squeeze()
        gene_ix = np.nonzero(counts)[0]
        counts = counts[gene_ix]
        counts_group.create_dataset(str(iC), data = counts)
        gix_group.create_dataset(str(iC), data = gene_ix)

    hf.close()
800685
def hdf5(self):
        """"""
        if self._rundir['hdf5'] is UNDETERMINED:
            h5_folder = self.path / self.par['ioin']['hdf5_output_folder']
            if (h5_folder / 'Data.xmf').is_file():
                self._rundir['hdf5'] = h5_folder
            else:
                self._rundir['hdf5'] = None
        return self._rundir['hdf5']
587618
def combine_count_files(files, out_file=None, ext=".fpkm"):
    
    files = list(files)
    if not files:
        return None
    assert all([file_exists(x) for x in files]), \
        "Some count files in %s do not exist." % files
    for f in files:
        assert file_exists(f), "%s does not exist or is empty." % f
    col_names = [os.path.basename(x.replace(ext, "")) for x in files]
    if not out_file:
        out_dir = os.path.join(os.path.dirname(files[0]))
        out_file = os.path.join(out_dir, "combined.counts")

    if file_exists(out_file):
        return out_file
    logger.info("Combining count files into %s." % out_file)
    row_names = []
    col_vals = defaultdict(list)
    for i, f in enumerate(files):
        vals = []
        if i == 0:
            with open(f) as in_handle:
                for line in in_handle:
                    rname, val = line.strip().split("\t")
                    row_names.append(rname)
                    vals.append(val)
        else:
            with open(f) as in_handle:
                for line in in_handle:
                    _, val = line.strip().split("\t")
                    vals.append(val)
        col_vals[col_names[i]] = vals

    df = pd.DataFrame(col_vals, index=row_names)
    df.to_csv(out_file, sep="\t", index_label="id")
    return out_file
80230
def copyFileToHdfs(localFilePath, hdfsFilePath, hdfsClient, override=True):
    ''''''
    if not os.path.exists(localFilePath):
        raise Exception('Local file Path does not exist!')
    if os.path.isdir(localFilePath):
        raise Exception('localFile should not a directory!')
    if hdfsClient.exists(hdfsFilePath):
        if override:
            hdfsClient.delete(hdfsFilePath)
        else:
            return False
    try:
        hdfsClient.copy_from_local(localFilePath, hdfsFilePath)
        return True
    except Exception as exception:
        nni_log(LogType.Error, 'Copy local file {0} to hdfs file {1} error: {2}'.format(localFilePath, hdfsFilePath, str(exception)))
        return False
591727
def combine_bam(in_files, out_file, config):
    
    runner = broad.runner_from_path("picard", config)
    runner.run_fn("picard_merge", in_files, out_file)
    for in_file in in_files:
        save_diskspace(in_file, "Merged into {0}".format(out_file), config)
    bam.index(out_file, config)
    return out_file
1084440
def concat(self, *dss, **kwargs):
        
        try:
            df = pd.DataFrame()
            for dsx in dss:
                df = pd.concat([df, dsx.df], **kwargs)
            self.df = df
        except Exception as e:
            self.err(e, "Can not concatenate data")
342851
def _load_h5(filename):
    ''''''
    from neurom.io import hdf5
    return hdf5.read(filename,
                     remove_duplicates=False,
                     data_wrapper=DataWrapper)
868647
def hdf5_read_in(self):
        

        with h5py.File(self.WORKING_DIRECTORY + '/' + self.file_name) as f:

            # read in
            data = f['data']

            # find number of distinct x and y points.
            num_x_pts = len(np.unique(data[self.x_column_label][:]))
            num_y_pts = len(np.unique(data[self.y_column_label][:]))

            # create 2D arrays of x,y,z
            self.xvals = np.reshape(data[self.x_column_label][:], (num_y_pts, num_x_pts))
            self.yvals = np.reshape(data[self.y_column_label][:], (num_y_pts, num_x_pts))
            self.zvals = np.reshape(data[self.z_column_label][:], (num_y_pts, num_x_pts))
        return
1151259
def _combine_out_files(chr_files, work_dir, data):
    
    out_file = "%s.bed" % sshared.outname_from_inputs(chr_files)
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                for chr_file in chr_files:
                    with open(chr_file) as in_handle:
                        is_empty = in_handle.readline().startswith("track name=empty")
                    if not is_empty:
                        with open(chr_file) as in_handle:
                            shutil.copyfileobj(in_handle, out_handle)
    return out_file
1315678
def _merge_files(self, input_files, output_file):
        """"""
        # we assume that all the input files have the same charset
        with open(output_file, mode='wb') as out:
            for input_file in input_files:
                out.write(open(input_file, mode='rb').read())
405451
def combine_and_copy(f, files, group):
    
    f[group] = np.concatenate([fi[group][:] if group in fi else \
                                   np.array([], dtype=np.uint32) for fi in files])