Open a GRIB file groupping incompatible hypercubes to different datasets via simple heuristics.
10054
def open_datasets(path, backend_kwargs={}, no_warn=False, **kwargs):
    # type: (str, T.Dict[str, T.Any], bool, T.Any) -> T.List[xr.Dataset]
    
    if not no_warn:
        warnings.warn("open_datasets is an experimental API, DO NOT RELY ON IT!", FutureWarning)

    fbks = []
    datasets = []
    try:
        datasets.append(open_dataset(path, backend_kwargs=backend_kwargs, **kwargs))
    except DatasetBuildError as ex:
        fbks.extend(ex.args[2])
    # NOTE: the recursive call needs to stay out of the exception handler to avoid showing
    #   to the user a confusing error message due to exception chaining
    for fbk in fbks:
        bks = backend_kwargs.copy()
        bks['filter_by_keys'] = fbk
        datasets.extend(open_datasets(path, backend_kwargs=bks, no_warn=True, **kwargs))
    return datasets
314623
def prep_fastq_inputs(in_files, data):
    
    if len(in_files) == 1 and _is_bam_input(in_files):
        out = _bgzip_from_bam(in_files[0], data["dirs"], data)
    elif len(in_files) == 1 and _is_cram_input(in_files):
        out = _bgzip_from_cram(in_files[0], data["dirs"], data)
    elif len(in_files) in [1, 2] and _ready_gzip_fastq(in_files, data):
        out = _symlink_in_files(in_files, data)
    else:
        if len(in_files) > 2:
            fpairs = fastq.combine_pairs(in_files)
            pair_types = set([len(xs) for xs in fpairs])
            assert len(pair_types) == 1
            fpairs.sort(key=lambda x: os.path.basename(x[0]))
            organized = [[xs[0] for xs in fpairs]]
            if len(fpairs[0]) > 1:
                organized.append([xs[1] for xs in fpairs])
            in_files = organized
        parallel = {"type": "local", "num_jobs": len(in_files),
                    "cores_per_job": max(1, data["config"]["algorithm"]["num_cores"] // len(in_files))}
        inputs = [{"in_file": x, "read_num": i, "dirs": data["dirs"], "config": data["config"],
                   "is_cwl": "cwl_keys" in data,
                   "rgnames": data["rgnames"]}
                  for i, x in enumerate(in_files) if x]
        out = run_multicore(_bgzip_from_fastq_parallel, [[d] for d in inputs], data["config"], parallel)
    return out
1010731
def _create_group_assignment(self, mesh_axes):
    """"""

    partitioning = {}
    for logical_pnum in xrange(self.size):
      group = mtf.pnum_to_group(self.shape, mesh_axes, logical_pnum)
      if group not in partitioning:
        partitioning[group] = []
      partitioning[group].append(self.l2p(logical_pnum))
    group_assignment = []
    for group, physical_pnums in partitioning.items():
      group_assignment.append(physical_pnums)
    return group_assignment
536711
def pairwise(iterable):
    ""
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)
883936
def cluster_labels_B(hdf5_file, s_reduced, lock, I, ii, iix, rows_slice):
    

    with Worker.hdf5_lock:
        with tables.open_file(hdf5_file, 'r+') as fileh:
            S = fileh.root.aff_prop_group.similarities
            s = S[rows_slice, ...]
           
    s = s[:, ii]
    s = s[iix[rows_slice]]
    
    with lock:                
        s_reduced += s[:].sum(axis = 0)
        
    del s
785461
def combine_dfs(dfs, names, method):
    """"""
    if method == "track":
        res = list()
        for df, identifier in zip(dfs, names):
            df["dataset"] = identifier
            res.append(df)
        return pd.concat(res, ignore_index=True)
    elif method == "simple":
        return pd.concat(dfs, ignore_index=True)
1196906
def from_data( name, coors, ngroups, conns, mat_ids, descs, igs = None ):
        
        if igs is None:
            igs = range( len( conns ) )
        mesh = Mesh(name)
        mesh._set_data(coors = coors,
                       ngroups = ngroups,
                       conns = [conns[ig] for ig in igs],
                       mat_ids = [mat_ids[ig] for ig in igs],
                       descs = [descs[ig] for ig in igs])
        mesh._set_shape_info()
        return mesh
470459
def _make_ltcube_file_list(ltsumfile, num_files):
    
    outbasename = os.path.basename(ltsumfile)
    lt_list_file = ltsumfile.replace('fits', 'lst')
    outfile = open(lt_list_file, 'w')
    for i in range(num_files):
        split_key = "%06i" % i
        output_dir = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes', split_key)
        filepath = os.path.join(output_dir, outbasename.replace('.fits', '_%s.fits' % split_key))
        outfile.write(filepath)
        outfile.write("\n")
    outfile.close()
    return '@' + lt_list_file
3026
def zinb_ll(data, P, R, Z):
    
    lls = nb_ll(data, P, R)
    clusters = P.shape[1]
    for c in range(clusters):
        pass
    return lls
733603
def _broadcast_shapes(s1, s2):
    """"""

    n1 = len(s1)
    n2 = len(s2)
    n = max(n1, n2)
    res = [1] * n

    for i in range(n):
        if i >= n1:
            c1 = 1
        else:
            c1 = s1[n1-1-i]

        if i >= n2:
            c2 = 1
        else:
            c2 = s2[n2-1-i]

        if c1 == 1:
            rc = c2
        elif c2 == 1 or c1 == c2:
            rc = c1
        else:
            raise ValueError('array shapes %r and %r are not compatible' % (s1, s2))

        res[n-1-i] = rc

    return tuple(res)
544426
def globs_in_single_dir(cls, spec_dir_path, address_mapper):
    """"""
    return [os.path.join(spec_dir_path, pat) for pat in address_mapper.build_patterns]
566874
def wc_lha2dict(lha):
    """"""
    C = OrderedDict()
    # try to read all WCs with 0, 2, or 4 fermions; if not found, set to zero
    for k, (block, i) in WC_dict_0f.items():
        try:
            C[k] = dict(lha['BLOCK'][block]['values'])[i]
        except KeyError:
            C[k] = 0
    for k in definitions.WC_keys_2f:
        try:
            C[k] = lha2matrix(lha['BLOCK']['WC' + k.upper()]['values'], (3,3)).real
        except KeyError:
            C[k] = np.zeros((3,3))
        try: # try to add imaginary part
            C[k] = C[k] + 1j*lha2matrix(lha['BLOCK']['IMWC' + k.upper()]['values'], (3,3))
        except KeyError:
            pass
    for k in definitions.WC_keys_4f:
        try:
            C[k] = lha2matrix(lha['BLOCK']['WC' + k.upper()]['values'], (3,3,3,3))
        except KeyError:
            C[k] = np.zeros((3,3,3,3))
        try: # try to add imaginary part
            C[k] = C[k] + 1j*lha2matrix(lha['BLOCK']['IMWC' + k.upper()]['values'], (3,3,3,3))
        except KeyError:
            pass
    return C
106752
private static void svdlibcSparseBinaryToMatlab(File input, File output,
                                                    boolean transpose) 
            throws IOException {
        
        DataInputStream dis = new DataInputStream(
            new BufferedInputStream(new FileInputStream(input)));

        PrintWriter pw = new PrintWriter(output);

        int rows = dis.readInt();
        int cols = dis.readInt();
        int entries = dis.readInt();

        // SVDLIBC sparse binary is organized as column data.  
        int entriesSeen = 0;
        int col = 0;
        for (; entriesSeen < entries; ++col) {
            int nonZero = dis.readInt();

            for (int i = 0; i < nonZero; ++i, ++entriesSeen) {
                int row = dis.readInt();
                float val = dis.readFloat();
                if (transpose)
                    pw.println((1 + col) + " " + (1 + row) + " " + val);
                else
                    pw.println((1 + row) + " " + (1 + col) + " " + val);
            }
        }
        
        dis.close();
        pw.close();
    }
86896
def _unpack_v1(h5file):
    ''''''
    points = np.array(h5file['points'])
    groups = np.array(h5file['structure'])
    return points, groups
301848
def to_simple_rdd(sc, features, labels):
    
    pairs = [(x, y) for x, y in zip(features, labels)]
    return sc.parallelize(pairs)
831687
def permuted_copy(self, partition=None):
        
        def take(n, iterable):
            return [next(iterable) for _ in range(n)]

        if partition is None:
            partition = Partition([1] * len(self))

        index_tuples = partition.get_membership()

        alignments = []
        for ix in index_tuples:
            concat = Concatenation(self, ix)
            sites = concat.alignment.get_sites()
            random.shuffle(sites)
            d = dict(zip(concat.alignment.get_names(), [iter(x) for x in zip(*sites)]))
            new_seqs = [[(k, ''.join(take(l, d[k]))) for k in d] for l in concat.lengths]

            for seqs, datatype, name in zip(new_seqs, concat.datatypes, concat.names):
                alignment = Alignment(seqs, datatype)
                alignment.name = name
                alignments.append(alignment)

        return self.__class__(records=sorted(alignments, key=lambda x: SORT_KEY(x.name)))
987372
def _bgzip_from_cram_sambamba(cram_file, dirs, data):
    
    raise NotImplementedError("sambamba doesn't yet support retrieval from CRAM by BED file")
    region_file = (tz.get_in(["config", "algorithm", "variant_regions"], data)
                   if tz.get_in(["config", "algorithm", "coverage_interval"], data) in ["regional", "exome"]
                   else None)
    base_name = utils.splitext_plus(os.path.basename(cram_file))[0]
    work_dir = utils.safe_makedir(os.path.join(dirs["work"], "align_prep",
                                               "%s-parts" % base_name))
    f1, f2, o1, o2, si = [os.path.join(work_dir, "%s.fq" % x) for x in ["match1", "match2", "unmatch1", "unmatch2",
                                                                        "single"]]
    ref_file = dd.get_ref_file(data)
    region = "-L %s" % region_file if region_file else ""
    cmd = ("sambamba view -f bam -l 0 -C {cram_file} -T {ref_file} {region} | "
           "bamtofastq F={f1} F2={f2} S={si} O={o1} O2={o2}")
    do.run(cmd.format(**locals()), "Convert CRAM to fastq in regions")