%prog mergemat *.npy

    Combine counts from multiple .npy data files.
507268
def mergemat(args):
    
    p = OptionParser(mergemat.__doc__)
    p.set_outfile(outfile="out")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    npyfiles = args
    A = np.load(npyfiles[0])
    logging.debug("Load `{}`: matrix of shape {}:; sum={}"
                  .format(npyfiles[0], A.shape, A.sum()))
    for npyfile in npyfiles[1:]:
        B = np.load(npyfile)
        A += B
        logging.debug("Load `{}`: sum={}"
                      .format(npyfiles[0], A.sum()))

    pf = opts.outfile
    np.save(pf, A)
    logging.debug("Combined {} files into `{}.npy`".format(len(npyfiles), pf))
443013
def prepare(args):
    
    p = OptionParser(prepare.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    counts, families = args
    countfiles = glob(op.join(counts, "*.count"))
    countsdb = defaultdict(list)
    for c in countfiles:
        rs = RiceSample(c)
        countsdb[(rs.tissue, rs.ind)].append(rs)

    # Merge duplicates - data sequenced in different batches
    key = lambda x: (x.label, x.rep)
    for (tissue, ind), rs in sorted(countsdb.items()):
        rs.sort(key=key)
        nrs = len(rs)
        for i in xrange(nrs):
            ri = rs[i]
            if not ri.working:
                continue
            for j in xrange(i + 1, nrs):
                rj = rs[j]
                if key(ri) != key(rj):
                    continue
                ri.merge(rj)
                rj.working = False
        countsdb[(tissue, ind)] = [x for x in rs if x.working]

    # Group into families
    mkdir("families")
    for (tissue, ind), r in sorted(countsdb.items()):
        r = list(r)
        if r[0].label != "F1":
            continue
        P1, P2 = r[0].P1, r[0].P2
        P1, P2 = countsdb[(tissue, P1)], countsdb[(tissue, P2)]
        rs = P1 + P2 + r
        groups = [1] * len(P1) + [2] * len(P2) + [3] * len(r)
        assert len(rs) == len(groups)

        outfile = "-".join((tissue, ind))
        merge_counts(rs, op.join(families, outfile))
        groupsfile = outfile + ".groups"
        fw = open(op.join(families, groupsfile), "w")
        print(",".join(str(x) for x in groups), file=fw)
        fw.close()
1151259
def _combine_out_files(chr_files, work_dir, data):
    
    out_file = "%s.bed" % sshared.outname_from_inputs(chr_files)
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                for chr_file in chr_files:
                    with open(chr_file) as in_handle:
                        is_empty = in_handle.readline().startswith("track name=empty")
                    if not is_empty:
                        with open(chr_file) as in_handle:
                            shutil.copyfileobj(in_handle, out_handle)
    return out_file
866451
def _merge(dts):
    
    df = pd.concat(dts)

    ma = df.pivot(index='isomir', columns='sample', values='counts')
    ma_mirna = ma
    ma = ma.fillna(0)
    ma_mirna['mirna'] = [m.split(":")[0] for m in ma.index.values]
    ma_mirna = ma_mirna.groupby(['mirna']).sum()
    ma_mirna = ma_mirna.fillna(0)
    return ma, ma_mirna
1030031
def merge(args):
    
    from jcvi.formats.gff import merge as gmerge

    p = OptionParser(merge.__doc__)
    p.set_home("maker")
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    outdir, outputgff = args
    fsnames, suffix = get_fsnames(outdir)
    nfs = len(fsnames)
    cmd = op.join(opts.maker_home, "bin/gff3_merge")

    outfile = "merge.sh"
    write_file(outfile, mergesh.format(suffix, cmd))

    # Generate per split directory
    # Note that gff3_merge write to /tmp, so I limit processes here to avoid
    # filling up disk space
    sh("parallel -j 8 merge.sh {} ::: " + " ".join(fsnames))

    # One final output
    gffnames = glob("*.all.gff")
    assert len(gffnames) == nfs

    # Again, DO NOT USE gff3_merge to merge with a smallish /tmp/ area
    gfflist = "gfflist"
    fw = open(gfflist, "w")
    print("\n".join(gffnames), file=fw)
    fw.close()

    nlines = sum(1 for x in open(gfflist))
    assert nlines == nfs  # Be extra, extra careful to include all results
    gmerge([gfflist, "-o", outputgff])
    logging.debug("Merged GFF file written to `{0}`".format(outputgff))
284449
def _merge(*args):
    """"""
    return re.compile(r'^' + r'[/-]'.join(args) + r'(?:\s+' + _dow + ')?$')
507638
def _combine_files(tsv_files, work_dir, data):
    
    header = "\t".join(["caller", "sample", "chrom", "start", "end", "svtype",
                        "lof", "annotation", "split_read_support", "paired_support_PE", "paired_support_PR"])
    sample = dd.get_sample_name(data)
    out_file = os.path.join(work_dir, "%s-prioritize.tsv" % (sample))
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            tmpdir = os.path.dirname(tx_out_file)
            input_files = " ".join(tsv_files)
            sort_cmd = bedutils.get_sort_cmd(tmpdir)
            cmd = "{{ echo '{header}'; cat {input_files} | {sort_cmd} -k3,3 -k4,4n; }} > {tx_out_file}"
            do.run(cmd.format(**locals()), "Combine prioritized from multiple callers")
    return out_file
138473
def appendData(self, bins, repnum=None):
        
        # only if the last sample was above threshold, but last-1 one wasn't
        bins[bins >= len(self._counts)] = len(self._counts) -1
        bin_totals = np.bincount(bins)
        self._counts[:len(bin_totals)] += bin_totals
        self.histo.setOpts(height=np.array(self._counts))
1058359
def combine(method, images, masks=None, dtype=None,
            region=None, zeros=None, scales=None, weights=None,
            include_variance=True, datamodel=None, method_name=""):
    """'"""

    import numina.datamodel

    nimages = len(images)

    # processing masks
    if masks is None:
        intl_masks = [None for _ in images]
    elif isinstance(masks, Extension):
        intl_masks = [img[masks.name].data for img in images]
    elif isinstance(masks, collections.Sequence):
        intl_masks = [mask[0].data for mask in masks]
    else:
        raise TypeError('mask in invalid')

    if len(intl_masks) != nimages:
        raise TypeError('len(masks) != len(images)')

    # Processing region
    if region is None:
        region = Ellipsis # meaning arr[...]

    # processing zeros, scales and weights
    num_values = dict(zeros=None, scales=None, weights=None)
    arg_values = dict(zeros=zeros, scales=scales, weights=weights)

    for key in num_values:
        arg = arg_values[key] # locals()[key]
        value_arg, arg_is_func = _inspect_method(arg)
        if arg_is_func:
            num_values[key] = [value_arg(img, mask, region) for img, mask in zip(images, intl_masks)]
        else:
            num_values[key] = value_arg
        if num_values[key] is not None:
            # Intl combine works only with np arrays
            num_values[key] = numpy.array(num_values[key])

    arrays = [hdu[0].data for hdu in images]
    if masks is None:
        fmasks = None
    else:
        fmasks = intl_masks

    out = C.generic_combine(
        method, arrays, masks=fmasks, dtype=dtype, out=None,
        zeros=num_values['zeros'], scales=num_values['scales'],
        weights=num_values['weights'])

    # Build HDUList
    headers = [img[0].header for img in images]

    base_header = headers[0].copy()
    hdu1 = fits.PrimaryHDU(data=out[0], header=base_header)
    list_of_hdu = [hdu1]

    #_logger.debug('update result header')
    prolog = ""
    if prolog:
        #_logger.debug('write prolog')
        hdu1.header['history'] = prolog

    if method_name != "":
        #_logger.info("Combined %d images using '%s'", nimages, method.__name__)
        hdu1.header['history'] = "Combined %d images using '%s'" % (nimages, method)
    else:
        hdu1.header['history'] = "Combined %d images" % (nimages,)

    hdu1.header['history'] = 'Combination time {}'.format(datetime.datetime.utcnow().isoformat())

    if datamodel is None:
        datamodel = numina.datamodel.DataModel()

    for idx, img in enumerate(images, start=1):
        hdu1.header['history'] = "Image{} {}".format(idx, datamodel.get_imgid(img))

    prevnum = base_header.get('NUM-NCOM', 1)

    hdu1.header['NUM-NCOM'] = prevnum * nimages
    hdu1.header['UUID'] = str(uuid.uuid1())

    # Headers of last image
    # hdu1.header['TSUTC2'] = headers[-1]['TSUTC2']

    if include_variance:
        varhdu = fits.ImageHDU(out[1], name='VARIANCE')
        list_of_hdu.append(varhdu)

    num = fits.ImageHDU(out[2].astype('uint8'), name='MAP')
    list_of_hdu.append(num)

    result = fits.HDUList(list_of_hdu)
    return result
88068
def delayed_bam_merge(data):
    
    if data.get("combine"):
        assert len(data["combine"].keys()) == 1
        file_key = list(data["combine"].keys())[0]
        extras = []
        for x in data["combine"][file_key].get("extras", []):
            if isinstance(x, (list, tuple)):
                extras.extend(x)
            else:
                extras.append(x)
        if file_key in data:
            extras.append(data[file_key])
        in_files = sorted(list(set(extras)))
        out_file = tz.get_in(["combine", file_key, "out"], data, _merge_out_from_infiles(in_files))
        sup_exts = data.get(file_key + "_plus", {}).keys()
        for ext in list(sup_exts) + [""]:
            merged_file = None
            if os.path.exists(utils.append_stem(out_file, "-" + ext)):
                cur_out_file, cur_in_files = out_file, []
            if ext:
                cur_in_files = list(filter(os.path.exists, (utils.append_stem(f, "-" + ext) for f in in_files)))
                cur_out_file = utils.append_stem(out_file, "-" + ext) if len(cur_in_files) > 0 else None
            else:
                cur_in_files, cur_out_file = in_files, out_file
            if cur_out_file:
                config = copy.deepcopy(data["config"])
                if len(cur_in_files) > 0:
                    merged_file = merge_bam_files(cur_in_files, os.path.dirname(cur_out_file), data,
                                                  out_file=cur_out_file)
                else:
                    assert os.path.exists(cur_out_file)
                    merged_file = cur_out_file
            if merged_file:
                if ext:
                    data[file_key + "_plus"][ext] = merged_file
                else:
                    data[file_key] = merged_file
        data.pop("region", None)
        data.pop("combine", None)
    return [[data]]
1157776
def _synthesize_multiple_python(self, text_file, output_file_path, quit_after=None, backwards=False):
        
        self.log(u"Synthesizing multiple via a Python call...")
        ret = self._synthesize_multiple_generic(
            helper_function=self._synthesize_single_python_helper,
            text_file=text_file,
            output_file_path=output_file_path,
            quit_after=quit_after,
            backwards=backwards
        )
        self.log(u"Synthesizing multiple via a Python call... done")
        return ret
389948
def combine_argtype(observations):
    
    assert len(observations) > 0
    assert is_Tuple(observations[0])
    if len(observations) > 1:
        prms = [get_Tuple_params(observations[0])]
        ln = len(prms[0])
        for obs in observations[1:]:
            assert is_Tuple(obs)
            prms.append(get_Tuple_params(obs))
            assert len(prms[-1]) == ln
        if simplify:
            prms = map(list, zip(*prms))
            if not isinstance(prms, list):
                # special care for Python 3
                prms = list(prms)
            for type_list in prms:
                simplify_for_Union(type_list)
            prms = map(tuple, prms)
        else:
            prms = map(tuple, zip(*prms))
        prms = map(Union.__getitem__, prms)
        return Tuple[tuple(prms)]
    else:
        return observations[0]
96121
def combine_pmaps(self, pmap_by_grp):
        
        grp = list(pmap_by_grp)[0]  # pmap_by_grp must be non-empty
        num_levels = pmap_by_grp[grp].shape_y
        pmaps = [probability_map.ProbabilityMap(num_levels, 1)
                 for _ in self.realizations]
        array = self.by_grp()
        for grp in pmap_by_grp:
            for gsim_idx, rlzis in enumerate(array[grp]):
                pmap = pmap_by_grp[grp].extract(gsim_idx)
                for rlzi in rlzis:
                    pmaps[rlzi] |= pmap
        return pmaps
30595
def merge_with(self, other):
        """"""
        result = ValuesAggregation()
        result.total = self.total + other.total
        result.count = self.count + other.count
        result.min = min(self.min, other.min)
        result.max = max(self.max, other.max)
        return result
288961
def merge_from(self, other):
        """"""
        if other.national_number_pattern is not None:
            self.national_number_pattern = other.national_number_pattern
        if other.example_number is not None:
            self.example_number = other.example_number
277436
def wr_row_mergeall(self, worksheet, txtstr, fmt, row_idx):
        """"""
        hdridxval = len(self.hdrs) - 1
        worksheet.merge_range(row_idx, 0, row_idx, hdridxval, txtstr, fmt)
        return row_idx + 1
183568
def _merge_theta_calls(bounds_file, result_file, cnv_file, data):
    
    out_file = "%s-merged.txt" % (result_file.replace(".BEST.results", ""))
    if not utils.file_uptodate(out_file, result_file):
        with file_transaction(data, out_file) as tx_out_file:
            updater = _update_with_calls(result_file, cnv_file)
            with open(bounds_file) as in_handle:
                with open(tx_out_file, "w") as out_handle:
                    i = 0
                    for line in in_handle:
                        if line.startswith("#"):
                            parts = line.rstrip().split("\t")
                            parts += ["cnv", "pop_cnvs", "pop_pvals"]
                            out_handle.write("\t".join(parts) + "\n")
                        else:
                            out_handle.write(updater(i, line))
                            i += 1
    return out_file