Combine multiple priority tsv files into a final sorted output.
507638
def _combine_files(tsv_files, work_dir, data):
    
    header = "\t".join(["caller", "sample", "chrom", "start", "end", "svtype",
                        "lof", "annotation", "split_read_support", "paired_support_PE", "paired_support_PR"])
    sample = dd.get_sample_name(data)
    out_file = os.path.join(work_dir, "%s-prioritize.tsv" % (sample))
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            tmpdir = os.path.dirname(tx_out_file)
            input_files = " ".join(tsv_files)
            sort_cmd = bedutils.get_sort_cmd(tmpdir)
            cmd = "{{ echo '{header}'; cat {input_files} | {sort_cmd} -k3,3 -k4,4n; }} > {tx_out_file}"
            do.run(cmd.format(**locals()), "Combine prioritized from multiple callers")
    return out_file
1332799
def _get_split_tasks(args, split_fn, file_key, outfile_i=-1):
    
    split_args = []
    combine_map = {}
    finished_map = collections.OrderedDict()
    extras = []
    for data in args:
        out_final, out_parts = split_fn(data)
        for parts in out_parts:
            split_args.append([utils.deepish_copy(data)] + list(parts))
        for part_file in [x[outfile_i] for x in out_parts]:
            combine_map[part_file] = out_final
        if len(out_parts) == 0:
            if out_final is not None:
                if out_final not in finished_map:
                    data[file_key] = out_final
                    finished_map[out_final] = [data]
                else:
                    extras.append([data])
            else:
                extras.append([data])
    return split_args, combine_map, list(finished_map.values()), extras
111157
def fix_multiple_files(filenames, options, output=None):
    
    filenames = find_files(filenames, options.recursive, options.exclude)
    if options.jobs > 1:
        import multiprocessing
        pool = multiprocessing.Pool(options.jobs)
        pool.map(_fix_file,
                 [(name, options) for name in filenames])
    else:
        for name in filenames:
            _fix_file((name, options, output))
519226
def sorted_source_files(self):
        """"""

        assert self.final, 'Call build() before using the graph.'
        out = []
        for node in nx.topological_sort(self.graph):
            if isinstance(node, NodeSet):
                out.append(node.nodes)
            else:
                # add a one-element list for uniformity
                out.append([node])
        return list(reversed(out))
123960
def sort(self, values):
        """"""
        for level in self:
            for wire1, wire2 in level:
                if values[wire1] > values[wire2]:
                    values[wire1], values[wire2] = values[wire2], values[wire1]
1009048
def _combine_sets(self, sets, final_set):
        
        self.cls.get_connection().sinterstore(final_set, list(sets))
        return final_set
1313811
def wr_tsv(self, fout_tsv, goea_results, **kws):
        """"""
        prt_flds = kws.get('prt_flds', self.get_prtflds_default(goea_results))
        tsv_data = MgrNtGOEAs(goea_results).get_goea_nts_prt(prt_flds, **kws)
        RPT.wr_tsv(fout_tsv, tsv_data, **kws)
160629
def makecsvdiffs(thediffs, dtls, n1, n2):
    """"""
    def ishere(val):
        if val == None:
            return "not here"
        else:
            return "is here"
    rows = []
    rows.append(['file1 = %s' % (n1, )])
    rows.append(['file2 = %s' % (n2, )])
    rows.append('')
    rows.append(theheader(n1, n2))
    keys = list(thediffs.keys()) # ensures sorting by Name
    keys.sort()
    # sort the keys in the same order as in the idd
    dtlssorter = DtlsSorter(dtls)
    keys = sorted(keys, key=dtlssorter.getkey)
    for key in keys:
        if len(key) == 2:
            rw2 = [''] + [ishere(i) for i in thediffs[key]]
        else:
            rw2 = list(thediffs[key])
        rw1 = list(key)
        rows.append(rw1 + rw2)
    return rows
848728
def postprocess(self):
        
        # Add suffix to all sub_dir/{items}
        for path in self.neb_dirs:
            for f in VASP_NEB_OUTPUT_SUB_FILES:
                f = os.path.join(path, f)
                if os.path.exists(f):
                    if self.final and self.suffix != "":
                        shutil.move(f, "{}{}".format(f, self.suffix))
                    elif self.suffix != "":
                        shutil.copy(f, "{}{}".format(f, self.suffix))

        # Add suffix to all output files
        for f in VASP_NEB_OUTPUT_FILES + [self.output_file]:
            if os.path.exists(f):
                if self.final and self.suffix != "":
                    shutil.move(f, "{}{}".format(f, self.suffix))
                elif self.suffix != "":
                    shutil.copy(f, "{}{}".format(f, self.suffix))
301148
def parseArgs():
    """"""
    parser = argparse.ArgumentParser()
    parser.add_argument("name", help="the file you want to split")
    parser.add_argument("out1", help="the name of the first file you want to output")
    parser.add_argument("out2", help="the name of the second file you want to output")
    return parser.parse_args()
904128
def fix_multiple_files(filenames, options, output=None):
    
    results = []
    filenames = find_files(filenames, options.recursive, options.exclude)
    if options.jobs > 1:
        import multiprocessing
        pool = multiprocessing.Pool(options.jobs)
        ret = pool.map(_fix_file, [(name, options) for name in filenames])
        results.extend([x for x in ret if x is not None])
    else:
        for name in filenames:
            ret = _fix_file((name, options, output))
            if ret is None:
                continue
            if options.diff:
                if ret != '':
                    results.append(ret)
            elif options.in_place:
                results.append(ret)
            else:
                original_source = readlines_from_file(name)
                if "".join(original_source).splitlines() != ret.splitlines():
                    results.append(ret)
    return results
426110
def combine_variant_files(orig_files, out_file, ref_file, config,
                          quiet_out=True, region=None):
    
    in_pipeline = False
    if isinstance(orig_files, dict):
        file_key = config["file_key"]
        in_pipeline = True
        orig_files = orig_files[file_key]
    if not utils.file_exists(out_file):
        with file_transaction(config, out_file) as tx_out_file:
            exist_files = [x for x in orig_files if os.path.exists(x)]
            ready_files = run_multicore(p_bgzip_and_index, [[x, config] for x in exist_files], config)
            dict_file = "%s.dict" % utils.splitext_plus(ref_file)[0]
            cores = dd.get_num_cores({"config": config})
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            cmd = ["picard"] + broad.get_picard_opts(config, memscale) + \
                  ["MergeVcfs", "D=%s" % dict_file, "O=%s" % tx_out_file] + \
                  ["I=%s" % f for f in ready_files]
            cmd = "%s && %s" % (utils.get_java_clprep(), " ".join(cmd))
            do.run(cmd, "Combine variant files")
    if out_file.endswith(".gz"):
        bgzip_and_index(out_file, config)
    if in_pipeline:
        return [{file_key: out_file, "region": region, "sam_ref": ref_file, "config": config}]
    else:
        return out_file
1151259
def _combine_out_files(chr_files, work_dir, data):
    
    out_file = "%s.bed" % sshared.outname_from_inputs(chr_files)
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                for chr_file in chr_files:
                    with open(chr_file) as in_handle:
                        is_empty = in_handle.readline().startswith("track name=empty")
                    if not is_empty:
                        with open(chr_file) as in_handle:
                            shutil.copyfileobj(in_handle, out_handle)
    return out_file
896169
def _organize_by_position(orig_file, cmp_file, chunk_size):
    
    with open(orig_file) as in_handle:
        reader1 = csv.reader(in_handle)
        positions = len(next(reader1)) - 1
    for positions in _chunks(range(positions), chunk_size):
        with open(orig_file) as orig_handle:
            with open(cmp_file) as cmp_handle:
                orig_reader = csv.reader(orig_handle)
                cmp_reader = csv.reader(cmp_handle)
                for item in _counts_at_position(positions,
                        orig_reader, cmp_reader):
                    yield item
629230
function consolidateFiles (files, config) {
  return Promise.resolve(files)
    .then(files => {
      var data = {};

      files.forEach(file => {
        var path = file.relative.split('.').shift().split(Path.sep);

        if (path.length >= 2 && config.flattenIndex) {
          var relPath = path.splice(-2, 2);
          path = (relPath[0] === relPath[1] || relPath[1] === 'index')
            ? path.concat(relPath[0])
            : path.concat(relPath);
        }

        data[path.join('.')] = JSON.parse(file.contents.toString());
      });

      data = sort(data);

      const tree = expand(data);
      const json = JSON.stringify(tree);

      return Promise.resolve(new Vinyl({
        base: '/',
        cwd: '/',
        path: '/' + (config.name || 'content.json'),
        contents: new Buffer(json)
      }));
    });
}
1338418
def load_results(result_files, options, run_set_id=None, columns=None,
                 columns_relevant_for_diff=set()):
    """"""
    return parallel.map(
        load_result,
        result_files,
        itertools.repeat(options),
        itertools.repeat(run_set_id),
        itertools.repeat(columns),
        itertools.repeat(columns_relevant_for_diff))
1276876
def tsplit(string, delimiters):
    """"""
    delimiters = tuple(delimiters)
    if len(delimiters) < 1:
        return [string,]
    final_delimiter = delimiters[0]
    for i in delimiters[1:]:
        string = string.replace(i, final_delimiter)
    return string.split(final_delimiter)