Group a set of items by sample names + multiple callers for prioritization
352061
def _group_by_sample(items):
    
    by_sample = collections.defaultdict(list)
    for d in items:
        by_sample[dd.get_sample_name(d)].append(d)
    out = []
    for sample_group in by_sample.values():
        cur = utils.deepish_copy(sample_group[0])
        svs = []
        for d in sample_group:
            svs.append(d["sv"])
        cur["sv"] = svs
        out.append(cur)
    return out
575373
def _array(group_idx, a, size, fill_value, dtype=None):
    """"""
    if fill_value is not None and not (np.isscalar(fill_value) or
                                       len(fill_value) == 0):
        raise ValueError("fill_value must be None, a scalar or an empty "
                         "sequence")
    order_group_idx = np.argsort(group_idx, kind='mergesort')
    counts = np.bincount(group_idx, minlength=size)
    ret = np.split(a[order_group_idx], np.cumsum(counts)[:-1])
    ret = np.asanyarray(ret)
    if fill_value is None or np.isscalar(fill_value):
        _fill_untouched(group_idx, ret, fill_value)
    return ret
253461
def _join_results(self, results, coarse, join):
    '''
    
    '''
    rval = OrderedDict()
    i_keys = set()
    for res in results:
      i_keys.update( res.keys() )
    for i_key in sorted(i_keys):
      if coarse:
        rval[i_key] = join( [res.get(i_key) for res in results] )
      else:
        rval[i_key] = OrderedDict()
        r_keys = set()
        for res in results:
          r_keys.update( res.get(i_key,{}).keys() )
        for r_key in sorted(r_keys):
          rval[i_key][r_key] = join( [res.get(i_key,{}).get(r_key) for res in results] )
    return rval
276471
def handle_multiple_callers(data, key, default=None, require_bam=True):
    
    callers = get_variantcaller(data, key, default, require_bam=require_bam)
    if isinstance(callers, six.string_types):
        return [data]
    elif not callers:
        return []
    else:
        out = []
        for caller in callers:
            base = copy.deepcopy(data)
            if not base["config"]["algorithm"].get("orig_%s" % key):
                base["config"]["algorithm"]["orig_%s" % key] = \
                  base["config"]["algorithm"][key]
            base["config"]["algorithm"][key] = caller
            # if splitting by variant caller, also split by jointcaller
            if key == "variantcaller":
                jcallers = get_variantcaller(data, "jointcaller", [])
                if isinstance(jcallers, six.string_types):
                    jcallers = [jcallers]
                if jcallers:
                    base["config"]["algorithm"]["orig_jointcaller"] = jcallers
                    jcallers = [x for x in jcallers if x.startswith(caller)]
                    if jcallers:
                        base["config"]["algorithm"]["jointcaller"] = jcallers[0]
                    else:
                        base["config"]["algorithm"]["jointcaller"] = False
            out.append(base)
        return out
794536
def _get_callers(items, stage, special_cases=False):
    
    callers = utils.deepish_copy(_CALLERS[stage])
    if special_cases and "cnvkit" in callers:
        has_lumpy = any("lumpy" in get_svcallers(d) or "lumpy" in d["config"]["algorithm"].get("svcaller_orig", [])
                        for d in items)
        if has_lumpy and any("lumpy_usecnv" in dd.get_tools_on(d) for d in items):
            if stage != "initial":
                del callers["cnvkit"]
        else:
            if stage != "standard":
                del callers["cnvkit"]
    return callers
154658
def available_composite_names(self, available_datasets=None):
        """"""
        return sorted(set(x.name for x in self.available_composite_ids(
            available_datasets=available_datasets)))
352669
def prep_db_parallel(samples, parallel_fn):
    
    batch_groups, singles, out_retrieve, extras = _group_by_batches(samples, _has_variant_calls)
    to_process = []
    has_batches = False
    for (name, caller), info in batch_groups.items():
        fnames = [x[0] for x in info]
        to_process.append([fnames, (str(name), caller, True), [x[1] for x in info], extras])
        has_batches = True
    for name, caller, data, fname in singles:
        to_process.append([[fname], (str(name), caller, False), [data], extras])
    output = parallel_fn("prep_gemini_db", to_process)
    out_fetch = {}
    for batch_id, out_file in output:
        out_fetch[tuple(batch_id)] = out_file
    out = []
    for batch_name, data in out_retrieve:
        out_variants = []
        for vrn in data["variants"]:
            use_population = vrn.pop("population", True)
            if use_population:
                vrn["population"] = out_fetch[(batch_name, vrn["variantcaller"])]
            out_variants.append(vrn)
        data["variants"] = out_variants
        out.append([data])
    for x in extras:
        out.append([x])
    return out
500390
def get_grouped(self, go_ntsets, go_all, gosubdag, **kws):
        """"""
        kws_grpd = {k:v for k, v in kws.items() if k in Grouped.kws_dict}
        kws_grpd['go2nt'] = self._init_go2ntpresent(go_ntsets, go_all, gosubdag)
        return Grouped(gosubdag, self.godag.version, **kws_grpd)
965861
def _cnv_prioritize(data):
    
    supported = {"cnvkit": {"inputs": ["call_file", "segmetrics"], "fn": _cnvkit_prioritize}}
    pcall = None
    priority_files = None
    for call in data.get("sv", []):
        if call["variantcaller"] in supported:
            priority_files = [call.get(x) for x in supported[call["variantcaller"]]["inputs"]]
            priority_files = [x for x in priority_files if x is not None and utils.file_exists(x)]
            if len(priority_files) == len(supported[call["variantcaller"]]["inputs"]):
                pcall = call
                break
    prioritize_by = tz.get_in(["config", "algorithm", "svprioritize"], data)
    if pcall and prioritize_by:
        out_file = "%s-prioritize.tsv" % utils.splitext_plus(priority_files[0])[0]
        gene_list = _find_gene_list_from_bed(prioritize_by, out_file, data)
        if gene_list:
            with open(gene_list) as in_handle:
                genes = [x.strip() for x in in_handle]
            args = [dd.get_sample_name(data), genes] + priority_files
            df = supported[pcall["variantcaller"]]["fn"](*args)
            with file_transaction(data, out_file) as tx_out_file:
                df.to_csv(tx_out_file, sep="\t", index=False)
            pcall["priority"] = out_file
    return data
1046255
def run(items, background=None):
    
    if not background: background = []
    names = [tz.get_in(["rgnames", "sample"], x) for x in items + background]
    work_bams = [x["align_bam"] for x in items + background]
    if len(items + background) < 2:
        raise ValueError("cn.mops only works on batches with multiple samples")
    data = items[0]
    work_dir = utils.safe_makedir(os.path.join(data["dirs"]["work"], "structural", names[0],
                                               "cn_mops"))
    parallel = {"type": "local", "cores": data["config"]["algorithm"].get("num_cores", 1),
                "progs": ["delly"]}
    with pysam.Samfile(work_bams[0], "rb") as pysam_work_bam:
        chroms = [None] if _get_regional_bed_file(items[0]) else pysam_work_bam.references
        out_files = run_multicore(_run_on_chrom, [(chrom, work_bams, names, work_dir, items)
                                                  for chrom in chroms],
                                  data["config"], parallel)
    out_file = _combine_out_files(out_files, work_dir, data)
    out = []
    for data in items:
        if "sv" not in data:
            data["sv"] = []
        data["sv"].append({"variantcaller": "cn_mops",
                           "vrn_file": _prep_sample_cnvs(out_file, data)})
        out.append(data)
    return out
564883
def _sample_item(self, **kwargs):
        """"""
        if self.replace:
            # Can sample from any of the items
            loc = np.random.choice(self._n_items)
        else:
            # Can only sample from items that have not been seen
            # Find ids that haven't been seen yet
            not_seen_ids = np.where(np.isnan(self.cached_labels_))[0]
            loc = np.random.choice(not_seen_ids)
        return loc, 1, {}
1120225
def _calc_sampleset(w1, w2, step, minimal):
        """"""
        if minimal:
            arr = [w1 - step, w1, w2, w2 + step]
        else:
            arr = np.arange(w1 - step, w2 + step + step, step)

        return arr
289576
def group_by(fn: Callable[[T], TR]):
    

    def inner(seq: ActualIterable[T]) -> Dict[TR, List[T]]:
        ret = defaultdict(list)
        for each in seq:
            ret[fn(each)].append(each)

        return ret

    return inner
1098301
def estimate(items, batch, config):
    
    hetcallers = {"theta": theta.run,
                  "phylowgs": phylowgs.run,
                  "bubbletree": bubbletree.run}
    paired = vcfutils.get_paired_bams([dd.get_align_bam(d) for d in items], items)
    calls = _get_calls(paired.tumor_data)
    variants = get_variants(paired.tumor_data)
    het_info = []
    for hetcaller in _get_hetcallers(items):
        try:
            hetfn = hetcallers[hetcaller]
        except KeyError:
            hetfn = None
            print("%s not yet implemented" % hetcaller)
        if hetfn:
            hetout = hetfn(variants[0], calls, paired)
            if hetout:
                het_info.append(hetout)
    out = []
    for data in items:
        if batch == _get_batches(data)[0]:
            if dd.get_sample_name(data) == paired.tumor_name:
                if het_info:
                    data["heterogeneity"] = het_info
            out.append([data])
    return out
1369990
def addCallSetFromName(self, sampleName):
        
        callSet = CallSet(self, sampleName)
        self.addCallSet(callSet)
728173
def _priority(s):
    """"""
    if type(s) in (list, tuple, set, frozenset):
        return ITERABLE
    if type(s) is dict:
        return DICT
    if issubclass(type(s), type):
        return TYPE
    if hasattr(s, "validate"):
        return VALIDATOR
    if callable(s):
        return CALLABLE
    else:
        return COMPARABLE
64015
def filter(self, table, groups, filter_string):
        """"""
        query = filter_string.lower()
        return [group for group in groups
                if query in group.name.lower()]