Associate cnvkit output with individual items.
342186
def _associate_cnvkit_out(ckouts, items, is_somatic=False):
    
    assert len(ckouts) == len(items)
    out = []
    upload_counts = collections.defaultdict(int)
    for ckout, data in zip(ckouts, items):
        ckout = copy.deepcopy(ckout)
        ckout["variantcaller"] = "cnvkit"
        if utils.file_exists(ckout["cns"]) and _cna_has_values(ckout["cns"]):
            ckout = _add_seg_to_output(ckout, data)
            ckout = _add_gainloss_to_output(ckout, data)
            ckout = _add_segmetrics_to_output(ckout, data)
            ckout = _add_variantcalls_to_output(ckout, data, items, is_somatic)
            # ckout = _add_coverage_bedgraph_to_output(ckout, data)
            ckout = _add_cnr_bedgraph_and_bed_to_output(ckout, data)
            if "svplots" in dd.get_tools_on(data):
                ckout = _add_plots_to_output(ckout, data)
            ckout["do_upload"] = upload_counts[ckout.get("vrn_file")] == 0
        if "sv" not in data:
            data["sv"] = []
        data["sv"].append(ckout)
        if ckout.get("vrn_file"):
            upload_counts[ckout["vrn_file"]] += 1
        out.append(data)
    return out
1011812
def _run_cnvkit_cancer(items, background):
    
    paired = vcfutils.get_paired_bams([x["align_bam"] for x in items], items)
    normal_data = [x for x in items if dd.get_sample_name(x) != paired.tumor_name]
    tumor_ready, normal_ready = _match_batches(paired.tumor_data, normal_data[0] if normal_data else None)
    ckouts = _run_cnvkit_shared([tumor_ready], [normal_ready] if normal_ready else [])
    if not ckouts:
        return items
    assert len(ckouts) == 1
    tumor_data = _associate_cnvkit_out(ckouts, [paired.tumor_data], is_somatic=True)
    return tumor_data + normal_data
327823
def _organize_output(output, combine_map, file_key, combine_arg_keys):
    
    out_map = collections.defaultdict(list)
    extra_args = collections.defaultdict(list)
    final_args = collections.OrderedDict()
    extras = []
    for data in output:
        cur_file = data.get(file_key)
        if not cur_file:
            extras.append([data])
        else:
            cur_out = combine_map[cur_file]
            out_map[cur_out].append(cur_file)
            extra_args[cur_out].append([data[x] for x in combine_arg_keys])
            data[file_key] = cur_out
            if cur_out not in final_args:
                final_args[cur_out] = [data]
            else:
                extras.append([data])
    combine_args = [[v, k] + _get_extra_args(extra_args[k], combine_arg_keys)
                    for (k, v) in out_map.items()]
    return combine_args, list(final_args.values()) + extras
955639
def _cnvkit_by_type(items, background):
    
    if len(items + background) == 1:
        return _run_cnvkit_single(items[0])
    elif vcfutils.get_paired_phenotype(items[0]):
        return _run_cnvkit_cancer(items, background)
    else:
        return _run_cnvkit_population(items, background)
160682
def _output_dependency_tree(self, target):
    """"""

    def make_line(dep, indent, is_dupe=False):
      indent_join, indent_chars = ('--', '  |') if self.should_tree else ('', '  ')
      dupe_char = '*' if is_dupe else ''
      return ''.join((indent * indent_chars, indent_join, dupe_char, dep))

    def output_deps(dep, indent, outputted, stack):
      dep_id, internal = self._dep_id(dep)

      if self.is_minimal and dep_id in outputted:
        return

      if self.output_candidate(internal):
        yield make_line(dep_id,
                        0 if self.is_external_only else indent,
                        is_dupe=dep_id in outputted)
        outputted.add(dep_id)

      for sub_dep in self._enumerate_visible_deps(dep, self.output_candidate):
        for item in output_deps(sub_dep, indent + 1, outputted, stack + [(dep_id, indent)]):
          yield item

    for item in output_deps(target, 0, set(), []):
      yield item
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
700497
def get_outputs_from_cm(index, cm):
    """"""
    return tuple(i for i in range(cm.shape[0]) if cm[index][i])
443201
def build_input_data(sentences, labels, vocabulary):
    """"""
    x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences])
    y = np.array(labels)
    return [x, y]
1065233
def c_array(ctype, values):
    """"""
    if isinstance(values, np.ndarray) and values.dtype.itemsize == ctypes.sizeof(ctype):
        return (ctype * len(values)).from_buffer_copy(values)
    return (ctype * len(values))(*values)
171154
def normalize_sv_coverage(*items):
    
    calcfns = {"cnvkit": _normalize_sv_coverage_cnvkit, "gatk-cnv": _normalize_sv_coverage_gatk}
    from bcbio.structural import cnvkit
    from bcbio.structural import shared as sshared
    items = [utils.to_single_data(x) for x in cwlutils.handle_combined_input(items)]
    if all(not cnvkit.use_general_sv_bins(x) for x in items):
        return [[d] for d in items]
    out_files = {}
    back_files = {}
    for group_id, gitems in itertools.groupby(items, lambda x: tz.get_in(["regions", "bins", "group"], x)):
        # No CNVkit calling for this particular set of samples
        if group_id is None:
            continue
        inputs, backgrounds = sshared.find_case_control(list(gitems))
        assert inputs, "Did not find inputs for sample batch: %s" % (" ".join(dd.get_sample_name(x) for x in items))
        work_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(inputs[0]), "structural",
                                                    dd.get_sample_name(inputs[0]), "bins"))
        back_files, out_files = calcfns[cnvkit.bin_approach(inputs[0])](group_id, inputs, backgrounds, work_dir,
                                                                        back_files, out_files)
    out = []
    for data in items:
        if dd.get_sample_name(data) in out_files:
            data["depth"]["bins"]["background"] = back_files[dd.get_sample_name(data)]
            data["depth"]["bins"]["normalized"] = out_files[dd.get_sample_name(data)]
        out.append([data])
    return out
908265
def gen_ccc(output, ascii_props=False, append=False, prefix=""):
    """"""

    obj = {}
    with codecs.open(os.path.join(HOME, 'unicodedata', UNIVERSION, 'DerivedCombiningClass.txt'), 'r', 'utf-8') as uf:
        for line in uf:
            if not line.startswith('#'):
                data = line.split('#')[0].split(';')
                if len(data) < 2:
                    continue
                span = create_span([int(i, 16) for i in data[0].strip().split('..')], is_bytes=ascii_props)
                if span is None:
                    continue
                name = format_name(data[1])

                if name not in obj:
                    obj[name] = []
                obj[name].extend(span)

    for x in range(0, 256):
        key = str(x)
        if key not in obj:
            obj[key] = []

    for name in list(obj.keys()):
        s = set(obj[name])
        obj[name] = sorted(s)

    not_explicitly_defined(obj, '0', is_bytes=ascii_props)

    # Convert characters values to ranges
    char2range(obj, is_bytes=ascii_props)

    with codecs.open(output, 'a' if append else 'w', 'utf-8') as f:
        if not append:
            f.write(HEADER)
        # Write out the Unicode properties
        f.write('%s_canonical_combining_class = {\n' % prefix)
        count = len(obj) - 1
        i = 0
        for k1, v1 in sorted(obj.items()):
            f.write('    "%s": "%s"' % (k1, v1))
            if i == count:
                f.write('\n}\n')
            else:
                f.write(',\n')
            i += 1
168703
def setup_data_split(X, y, tokenizer, proc_data_dir, **kwargs):
    
    X_train, X_val, X_test, y_train, y_val, y_test = split_data(X, y)

    # only build vocabulary on training data
    tokenizer.build_vocab(X_train)

    process_save(X_train, y_train, tokenizer, path.join(
        proc_data_dir, 'train.bin'), train=True, **kwargs)
    process_save(X_val, y_val, tokenizer, path.join(
        proc_data_dir, 'val.bin'), **kwargs)
    process_save(X_test, y_test, tokenizer, path.join(
        proc_data_dir, 'test.bin'), **kwargs)
1057803
def cnv_to_event(name, data):
    
    cur_ploidy = ploidy.get_ploidy([data])
    if name.startswith("cnv"):
        num = max([int(x) for x in name.split("_")[0].replace("cnv", "").split(";")])
        if num < cur_ploidy:
            return "DEL"
        elif num > cur_ploidy:
            return "DUP"
        else:
            return name
    else:
        return name
18462
def mnist_extract_labels(filename, num_images):
  """"""
  print('Extracting', filename)
  with gzip.open(filename) as bytestream:
    bytestream.read(8)
    buf = bytestream.read(1 * num_images)
    labels = np.frombuffer(buf, dtype=np.uint8)
  # Convert to dense 1-hot representation.
  return (np.arange(10) == labels[:, None]).astype(np.float32)
755749
def _collect_layer_outputs(mod, data, include_layer=None, max_num_examples=None, logger=None):
    """"""
    collector = _LayerOutputCollector(include_layer=include_layer, logger=logger)
    num_examples = _collect_layer_statistics(mod, data, collector, max_num_examples, logger)
    return collector.nd_dict, num_examples
846281
def chain_output_files(self):
        
        ret_list = []
        for key, val in self.file_dict.items():
            # For pure input files we only want output files that were not
            # marked as internal or temp
            if val & FileFlags.out_ch_mask == FileFlags.output_mask:
                ret_list.append(key)
        return ret_list
1156483
def _cnvkit_prioritize(sample, genes, allele_file, metrics_file):
    
    mdf = pd.read_table(metrics_file)
    mdf.columns = [x.lower() for x in mdf.columns]
    if len(genes) > 0:
        mdf = mdf[mdf["gene"].str.contains("|".join(genes))]
    mdf = mdf[["chromosome", "start", "end", "gene", "log2", "ci_hi", "ci_lo"]]
    adf = pd.read_table(allele_file)
    if len(genes) > 0:
        adf = adf[adf["gene"].str.contains("|".join(genes))]
    if "cn1" in adf.columns and "cn2" in adf.columns:
        adf = adf[["chromosome", "start", "end", "cn", "cn1", "cn2"]]
    else:
        adf = adf[["chromosome", "start", "end", "cn"]]
    df = pd.merge(mdf, adf, on=["chromosome", "start", "end"])
    df = df[df["cn"] != 2]
    if len(df) > 0:
        def passes(row):
            spread = abs(row["ci_hi"] - row["ci_lo"])
            return spread < 0.25
        df["passes"] = df.apply(passes, axis=1)
    df.insert(0, "sample", [sample] * len(df))
    return df