Merge extra disambiguated reads into a final BAM file.
177408
def merge_extras(items, config):
    
    final = {}
    for extra_name in items[0]["disambiguate"].keys():
        in_files = []
        for data in items:
            in_files.append(data["disambiguate"][extra_name])
        out_file = "%s-allmerged%s" % os.path.splitext(in_files[0])
        if in_files[0].endswith(".bam"):
            merged_file = merge.merge_bam_files(in_files, os.path.dirname(out_file), items[0],
                                                out_file=out_file)
        else:
            assert extra_name == "summary", extra_name
            merged_file = _merge_summary(in_files, out_file, items[0])
        final[extra_name] = merged_file
    out = []
    for data in items:
        data["disambiguate"] = final
        out.append([data])
    return out
146823
def _disassemble3(co, lasti):
    """"""
    cell_names = co.co_cellvars + co.co_freevars
    linestarts = dict(dis.findlinestarts(co))
    _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
                       co.co_consts, cell_names, linestarts)
996355
private int merge(String[] dst, String[] srcA, int lengthA, Tag[] srcB, int lengthB) {
    int i = 0;
    int ai = 0;
    int bi = 0;

    while (ai < lengthA && bi < lengthB) {
      final String ak = srcA[ai];
      final String av = srcA[ai + 1];
      Tag b = srcB[bi];
      int cmp = ak.compareTo(b.key());
      if (cmp < 0) {
        dst[i++] = ak;
        dst[i++] = av;
        ai += 2;
      } else if (cmp > 0) {
        dst[i++] = b.key();
        dst[i++] = b.value();
        ++bi;
      } else {
        // Newer tags should override, use source B if there are duplicate keys.
        // If source B has duplicates, then use the last value for the given key.
        int j = bi + 1;
        for (; j < lengthB && ak.equals(srcB[j].key()); ++j) {
          b = srcB[j];
        }
        dst[i++] = b.key();
        dst[i++] = b.value();
        bi = j;
        ai += 2; // Ignore
      }
    }

    if (ai < lengthA) {
      System.arraycopy(srcA, ai, dst, i, lengthA - ai);
      i += lengthA - ai;
    } else if (bi < lengthB) {
      i = dedup(srcB, bi, dst, i, lengthB - bi);
    }

    return i;
  }
892143
def _extract_split_and_discordants(in_bam, work_dir, data):
    
    sr_file = os.path.join(work_dir, "%s-sr.bam" % os.path.splitext(os.path.basename(in_bam))[0])
    disc_file = os.path.join(work_dir, "%s-disc.bam" % os.path.splitext(os.path.basename(in_bam))[0])
    if not utils.file_exists(sr_file) or not utils.file_exists(disc_file):
        with file_transaction(data, sr_file) as tx_sr_file:
            with file_transaction(data, disc_file) as tx_disc_file:
                cores = dd.get_num_cores(data)
                ref_file = dd.get_ref_file(data)
                cmd = ("extract-sv-reads -e --threads {cores} -T {ref_file} "
                       "-i {in_bam} -s {tx_sr_file} -d {tx_disc_file}")
                do.run(cmd.format(**locals()), "extract split and discordant reads", data)
    for fname in [sr_file, disc_file]:
        bam.index(fname, data["config"])
    return sr_file, disc_file
1277275
def _slice_bam(in_bam, region, tmp_dir, config):
    """"""
    name_file = os.path.splitext(os.path.basename(in_bam))[0]
    out_file = os.path.join(tmp_dir, os.path.join(tmp_dir, name_file + _to_str(region) + ".bam"))
    sambamba = config_utils.get_program("sambamba", config)
    region = _to_sambamba(region)
    with file_transaction(out_file) as tx_out_file:
        cmd = ("{sambamba} slice {in_bam} {region} -o {tx_out_file}")
        do.run(cmd.format(**locals()), "Slice region", {})
    return out_file
974667
static <T> void mergeFrom(byte[] data, int offset, int length, T message,
            Schema<T> schema, boolean decodeNestedMessageAsGroup)
    {
        try
        {
            final ByteArrayInput input = new ByteArrayInput(data, offset, length,
                    decodeNestedMessageAsGroup);
            schema.mergeFrom(input, message);
            input.checkLastTagWas(0);
        }
        catch (ArrayIndexOutOfBoundsException ae)
        {
            throw new RuntimeException("Truncated.", ProtobufException.truncatedMessage(ae));
        }
        catch (IOException e)
        {
            throw new RuntimeException("Reading from a byte array threw an IOException (should " +
                    "never happen).", e);
        }
    }
3524
def picard_sam_to_bam(picard, align_sam, fastq_bam, ref_file,
                      is_paired=False):
    
    to_retain = ["XS", "XG", "XM", "XN", "XO", "YT"]
    if align_sam.endswith(".sam"):
        out_bam = "%s.bam" % os.path.splitext(align_sam)[0]
    elif align_sam.endswith("-align.bam"):
        out_bam = "%s.bam" % align_sam.replace("-align.bam", "")
    else:
        raise NotImplementedError("Input format not recognized")
    if not file_exists(out_bam):
        with tx_tmpdir(picard._config) as tmp_dir:
            with file_transaction(picard._config, out_bam) as tx_out_bam:
                opts = [("UNMAPPED", fastq_bam),
                        ("ALIGNED", align_sam),
                        ("OUTPUT", tx_out_bam),
                        ("REFERENCE_SEQUENCE", ref_file),
                        ("TMP_DIR", tmp_dir),
                        ("PAIRED_RUN", ("true" if is_paired else "false")),
                        ]
                opts += [("ATTRIBUTES_TO_RETAIN", x) for x in to_retain]
                picard.run("MergeBamAlignment", opts)
    return out_bam
1256636
def merge_replacement_files(tmpdir, mergefile):
  """"""
  # The fixes suggested by clang-tidy >= 4.0.0 are given under
  # the top level key 'Diagnostics' in the output yaml files
  mergekey="Diagnostics"
  merged=[]
  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
    content = yaml.safe_load(open(replacefile, 'r'))
    if not content:
      continue # Skip empty files.
    merged.extend(content.get(mergekey, []))

  if merged:
    # MainSourceFile: The key is required by the definition inside
    # include/clang/Tooling/ReplacementsYaml.h, but the value
    # is actually never used inside clang-apply-replacements,
    # so we set it to '' here.
    output = { 'MainSourceFile': '', mergekey: merged }
    with open(mergefile, 'w') as out:
      yaml.safe_dump(output, out)
  else:
    # Empty the file:
    open(mergefile, 'w').close()
913687
def dedupFasta(reads):
    
    seen = set()
    add = seen.add
    for read in reads:
        hash_ = md5(read.sequence.encode('UTF-8')).digest()
        if hash_ not in seen:
            add(hash_)
            yield read
706977
def _prepare_inputs(ma_fn, bam_file, out_dir):
    
    fixed_fa = os.path.join(out_dir, "file_reads.fa")
    count_name =dict()
    with file_transaction(fixed_fa) as out_tx:
        with open(out_tx, 'w') as out_handle:
            with open(ma_fn) as in_handle:
                h = next(in_handle)
                for line in in_handle:
                    cols = line.split("\t")
                    name_with_counts = "%s_x%s" % (cols[0], sum(map(int, cols[2:])))
                    count_name[cols[0]] = name_with_counts
                    out_handle.write(">%s\n%s\n" % (name_with_counts, cols[1]))
    fixed_bam = os.path.join(out_dir, "align.bam")
    bam_handle = pysam.AlignmentFile(bam_file, "rb")
    with pysam.AlignmentFile(fixed_bam, "wb", template=bam_handle) as out_handle:
        for read in bam_handle.fetch():
            read.query_name = count_name[read.query_name]
            out_handle.write(read)

    return fixed_fa, fixed_bam
645328
private String merge(final String first, final String[] more) {
        assert first != null : "first must be specified";
        assert more != null : "more must be specified";

        final StringBuilder merged = new StringBuilder();
        merged.append(first);
        for (int i = 0; i < more.length; i++) {
            merged.append(ArchivePath.SEPARATOR);
            merged.append(more[i]);
        }
        return merged.toString();

    }
724786
def paired_reader_from_bamfile(args,
                               log,
                               usage_logger,
                               annotated_writer):
    ''''''
    total_aligns = pysamwrapper.total_align_count(args.input_bam)
    bamfile_generator  = _bamfile_generator(args.input_bam)
    return _paired_reader(args.umt_length,
                          bamfile_generator,
                          total_aligns,
                          log,
                          usage_logger,
                          annotated_writer)
1162886
def select_unaligned_read_pairs(in_bam, extra, out_dir, config):
    
    runner = broad.runner_from_config(config)
    base, ext = os.path.splitext(os.path.basename(in_bam))
    nomap_bam = os.path.join(out_dir, "{}-{}{}".format(base, extra, ext))
    if not utils.file_exists(nomap_bam):
        with file_transaction(nomap_bam) as tx_out:
            runner.run("FilterSamReads", [("INPUT", in_bam),
                                          ("OUTPUT", tx_out),
                                          ("EXCLUDE_ALIGNED", "true"),
                                          ("WRITE_READS_FILES", "false"),
                                          ("SORT_ORDER", "queryname")])
    has_reads = False
    with pysam.Samfile(nomap_bam, "rb") as in_pysam:
        for read in in_pysam:
            if read.is_paired:
                has_reads = True
                break
    if has_reads:
        out_fq1, out_fq2 = ["{}-{}.fq".format(os.path.splitext(nomap_bam)[0], i) for i in [1, 2]]
        runner.run_fn("picard_bam_to_fastq", nomap_bam, out_fq1, out_fq2)
        return out_fq1, out_fq2
    else:
        return None, None
117035
def setup_combine(final_file, data):
    
    if "align_split" not in data:
        return final_file, data
    align_dir = os.path.dirname(final_file)
    base, ext = os.path.splitext(os.path.basename(final_file))
    start, end = [int(x) for x in data["align_split"].split("-")]
    out_file = os.path.join(utils.safe_makedir(os.path.join(align_dir, "split")),
                            "%s-%s_%s%s" % (base, start, end, ext))
    data["combine"] = {"work_bam": {"out": final_file, "extras": []}}
    return out_file, data
326910
def _link_bam_file(in_file, new_dir, data):
    
    new_dir = utils.safe_makedir(new_dir)
    out_file = os.path.join(new_dir, os.path.basename(in_file))
    if not utils.file_exists(out_file):
        out_file = os.path.join(new_dir, "%s-prealign.bam" % dd.get_sample_name(data))
    if data.get("cwl_keys"):
        # Has indexes, we're okay to go with the original file
        if utils.file_exists(in_file + ".bai"):
            out_file = in_file
        else:
            utils.copy_plus(in_file, out_file)
    else:
        utils.symlink_plus(in_file, out_file)
    return out_file
598200
def _unassembled_reads2_out_file_name(self):
        """"""
        if self.Parameters['-2'].isOn():
            unassembled_reads2 = self._absolute(
                str(self.Parameters['-2'].Value))
        else:
            raise ValueError("No reads2 (flag -2) output path specified")
        return unassembled_reads2
523320
def tobam_cl(data, out_file, is_paired=False):
    
    do_dedup = _check_dedup(data)
    umi_consensus = dd.get_umi_consensus(data)
    with file_transaction(data, out_file) as tx_out_file:
        if not do_dedup:
            yield (sam_to_sortbam_cl(data, tx_out_file), tx_out_file)
        elif umi_consensus:
            yield (_sam_to_grouped_umi_cl(data, umi_consensus, tx_out_file), tx_out_file)
        elif is_paired and _need_sr_disc_reads(data) and not _too_many_contigs(dd.get_ref_file(data)):
            sr_file = "%s-sr.bam" % os.path.splitext(out_file)[0]
            disc_file = "%s-disc.bam" % os.path.splitext(out_file)[0]
            with file_transaction(data, sr_file) as tx_sr_file:
                with file_transaction(data, disc_file) as tx_disc_file:
                    yield (samblaster_dedup_sort(data, tx_out_file, tx_sr_file, tx_disc_file),
                           tx_out_file)
        else:
            yield (_biobambam_dedup_sort(data, tx_out_file), tx_out_file)