Create file with region sorted list of non-empty VCFs for concatenating.
355137
def _get_file_list(orig_files, out_file, regions, ref_file, config):
    
    sorted_files = _sort_by_region(orig_files, regions, ref_file, config)
    exist_files = [(c, x) for c, x in sorted_files if os.path.exists(x) and vcf_has_variants(x)]
    if len(exist_files) == 0:  # no non-empty inputs, merge the empty ones
        exist_files = [x for c, x in sorted_files if os.path.exists(x)]
    elif len(exist_files) > 1:
        exist_files = _fix_gatk_header(exist_files, out_file, config)
    else:
        exist_files = [x for c, x in exist_files]
    ready_files = run_multicore(p_bgzip_and_index, [[x, config] for x in exist_files], config)
    input_file_list = "%s-files.list" % utils.splitext_plus(out_file)[0]
    with open(input_file_list, "w") as out_handle:
        for fname in ready_files:
            out_handle.write(fname + "\n")
    return input_file_list
343156
def _survivor_merge(call_vcf, truth_vcf, stats, work_dir, data):
    
    out_file = os.path.join(work_dir, "eval-merge.vcf")
    if not utils.file_uptodate(out_file, call_vcf):
        in_call_vcf = call_vcf.replace(".vcf.gz", ".vcf")
        if not utils.file_exists(in_call_vcf):
            with file_transaction(data, in_call_vcf) as tx_in_call_vcf:
                do.run("gunzip -c {call_vcf} > {tx_in_call_vcf}".format(**locals()))
        in_truth_vcf = truth_vcf.replace(".vcf.gz", ".vcf")
        if not utils.file_exists(in_truth_vcf):
            with file_transaction(data, in_truth_vcf) as tx_in_truth_vcf:
                do.run("gunzip -c {truth_vcf} > {tx_in_truth_vcf}".format(**locals()))
        in_list_file = os.path.join(work_dir, "eval-inputs.txt")
        with open(in_list_file, "w") as out_handle:
            out_handle.write("%s\n%s\n" % (in_call_vcf, in_truth_vcf))
        with file_transaction(data, out_file) as tx_out_file:
            cmd = ("SURVIVOR merge {in_list_file} {stats[merge_size]} 1 0 0 0 {stats[min_size]} {tx_out_file}")
            do.run(cmd.format(**locals()), "Merge SV files for validation: %s" % dd.get_sample_name(data))
    return out_file
774878
def finalize_vcf(in_file, variantcaller, items):
    
    out_file = "%s-annotated%s" % utils.splitext_plus(in_file)
    if not utils.file_uptodate(out_file, in_file):
        header_cl = _add_vcf_header_sample_cl(in_file, items, out_file)
        contig_cl = _add_contig_cl(in_file, items, out_file)
        cls = [x for x in (contig_cl, header_cl) if x]
        if cls:
            post_cl = " | ".join(cls) + " | "
        else:
            post_cl = None
        dbsnp_file = tz.get_in(("genome_resources", "variation", "dbsnp"), items[0])
        if dbsnp_file:
            out_file = _add_dbsnp(in_file, dbsnp_file, items[0], out_file, post_cl)
    if utils.file_exists(out_file):
        return vcfutils.bgzip_and_index(out_file, items[0]["config"])
    else:
        return in_file
1280103
def __populate_archive_files(self):
        """"""
        self.archive_files = []
        for _ptr in _bfd.archive_list_files(self._ptr):
            try:
                self.archive_files.append(Bfd(_ptr))
            except BfdException, err:
                #print "Error populating archive file list : %s" % err
                #print_exc()
                pass
241759
def _setup_variant_regions(data, out_dir):
    
    vr_file = dd.get_variant_regions(data)
    if not vr_file:
        vr_file = regions.get_sv_bed(data, "transcripts", out_dir=out_dir)
    contigs = set([c.name for c in ref.file_contigs(dd.get_ref_file(data))])
    out_file = os.path.join(utils.safe_makedir(os.path.join(dd.get_work_dir(data), "bedprep")),
                            "%s-rnaseq_clean.bed" % utils.splitext_plus(os.path.basename(vr_file))[0])
    if not utils.file_uptodate(out_file, vr_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                with shared.bedtools_tmpdir(data):
                    for r in pybedtools.BedTool(vr_file):
                        if r.chrom in contigs:
                            if chromhacks.is_nonalt(r.chrom):
                                out_handle.write(str(r))
    data = dd.set_variant_regions(data, out_file)
    return data
915869
def create_more_container_files(sourceDir, suffix, maxElements, containers, containers2):
    """"""

    # Create files for each MPL-container with 20 to 'maxElements' elements
    # which will be used during generation.
    for container in containers:
        for i in range(20, maxElements, 10):
            # Create copy of "template"-file.
            newFile = os.path.join( sourceDir, container, container + str(i+10) + suffix )
            shutil.copyfile( os.path.join( sourceDir, container, container + "20" + suffix ), newFile ) 
            # Adjust copy of "template"-file accordingly.
            for line in fileinput.input( newFile, inplace=1, mode="rU" ):
                line = re.sub(r'20', '%TWENTY%', line.rstrip())
                line = re.sub(r'11', '%ELEVEN%', line.rstrip())
                line = re.sub(r'10(?![0-9])', '%TEN%', line.rstrip())
                line = re.sub(r'%TWENTY%', re.escape(str(i+10)), line.rstrip())
                line = re.sub(r'%ELEVEN%', re.escape(str(i + 1)), line.rstrip())
                line = re.sub(r'%TEN%', re.escape(str(i)), line.rstrip())
                print(line)
    for container in containers2:
        for i in range(20, maxElements, 10):
            # Create copy of "template"-file.
            newFile = os.path.join( sourceDir, container, container + str(i+10) + "_c" + suffix )
            shutil.copyfile( os.path.join( sourceDir, container, container + "20_c" + suffix ), newFile ) 
            # Adjust copy of "template"-file accordingly.
            for line in fileinput.input( newFile, inplace=1, mode="rU" ):
                line = re.sub(r'20', '%TWENTY%', line.rstrip())
                line = re.sub(r'11', '%ELEVEN%', line.rstrip())
                line = re.sub(r'10(?![0-9])', '%TEN%', line.rstrip())
                line = re.sub(r'%TWENTY%', re.escape(str(i+10)), line.rstrip())
                line = re.sub(r'%ELEVEN%', re.escape(str(i + 1)), line.rstrip())
                line = re.sub(r'%TEN%', re.escape(str(i)), line.rstrip())
                print(line)
405451
def combine_and_copy(f, files, group):
    
    f[group] = np.concatenate([fi[group][:] if group in fi else \
                                   np.array([], dtype=np.uint32) for fi in files])
846275
public static void exportCSVSequenceLocal(File baseDir, JavaRDD<List<List<Writable>>> sequences, long seed)
                    throws Exception {
        baseDir.mkdirs();
        if (!baseDir.isDirectory())
            throw new IllegalArgumentException("File is not a directory: " + baseDir.toString());
        String baseDirStr = baseDir.toString();

        List<String> fileContents = sequences.map(new SequenceToStringFunction(",")).collect();
        if (!(fileContents instanceof ArrayList))
            fileContents = new ArrayList<>(fileContents);
        Collections.shuffle(fileContents, new Random(seed));

        int i = 0;
        for (String s : fileContents) {
            String path = FilenameUtils.concat(baseDirStr, i + ".csv");
            File f = new File(path);
            FileUtils.writeStringToFile(f, s);
            i++;
        }
    }
635388
public static void concatenate(List<File> files, File concatenatedFile) {

        BufferedWriter writer;
        try {
            writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(concatenatedFile.getAbsoluteFile(),
                    false), DataUtilDefaults.charSet));

            FileInputStream inputStream;
            for(File input : files) {
                inputStream = new FileInputStream(input);
                BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
                String line;
                while((line = reader.readLine()) != null) {
                    writer.write(line + DataUtilDefaults.lineTerminator);
                }
                inputStream.close();
            }
            writer.flush();
            writer.close();
        } catch (UnsupportedEncodingException e) {
            throw new DataUtilException(e);
        } catch (FileNotFoundException e) {
            throw new DataUtilException(e);
        } catch (IOException e) {
            throw new DataUtilException(e);
        }
    }
1285418
def write_to_files_rec(instances, tokenizer, max_seq_length,
                       max_predictions_per_seq, output_files):
    """"""
    writers = []
    for output_file in output_files:
        writers.append(
            mx.recordio.MXIndexedRecordIO(
                os.path.splitext(output_file)[0] + '.idx', output_file, 'w'))

    writer_index = 0
    total_written = 0
    for (inst_index, instance) in enumerate(instances):
        features = transform(instance, tokenizer, max_seq_length, max_predictions_per_seq)
        example = features

        writers[writer_index].write_idx(inst_index, json.dumps(example).encode('UTF-8'))
        writer_index = (writer_index + 1) % len(writers)
        total_written += 1

    for writer in writers:
        writer.close()

    logging.info('Wrote %d total instances', total_written)
951451
def create_tarfile(files: List[str], tar_path: str) -> None:
    """"""
    with tarfile.open(tar_path, "w:gz") as tar:
        for f in files:
            tar.add(f)
573011
def create_xz(archive, compression, cmd, verbosity, interactive, filenames):
    """"""
    return _create(archive, compression, cmd, 'xz', verbosity, filenames)
844774
def subset_vcf(in_file, region, out_file, config):
    
    work_file = vcfutils.bgzip_and_index(in_file, config)
    if not file_exists(out_file):
        with file_transaction(config, out_file) as tx_out_file:
            bcftools = config_utils.get_program("bcftools", config)
            region_str = bamprep.region_to_gatk(region)
            cmd = "{bcftools} view -r {region_str} {work_file} > {tx_out_file}"
            do.run(cmd.format(**locals()), "subset %s: %s" % (os.path.basename(work_file), region_str))
    return out_file
1286527
def make_simple_gt_aware_merged_vcf_with_no_combinations(self, ref_seq):
        ''''''
        if len(self) <= 1:
            return

        merged_vcf_record = self.vcf_records[0]

        for i in range(1, len(self.vcf_records), 1):
            if self.vcf_records[i].intersects(merged_vcf_record):
                return
            else:
                merged_vcf_record = merged_vcf_record.gt_aware_merge(self.vcf_records[i], ref_seq)

        self.vcf_records = [merged_vcf_record]
679416
def concat(dfs):
    """"""
    if isstr(dfs) or not hasattr(dfs, '__iter__'):
        msg = 'Argument must be a non-string iterable (e.g., list or tuple)'
        raise TypeError(msg)

    _df = None
    for df in dfs:
        df = df if isinstance(df, IamDataFrame) else IamDataFrame(df)
        if _df is None:
            _df = copy.deepcopy(df)
        else:
            _df.append(df, inplace=True)
    return _df
1245523
def to_vcf(in_file, caller, header_fn, vcf_fn, data, sep="\t"):
    
    out_file = "%s.vcf" % utils.splitext_plus(in_file)[0]
    if not utils.file_exists(out_file + ".gz") and not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(in_file) as in_handle:
                with open(tx_out_file, "w") as out_handle:
                    out_handle.write(_vcf_header.format(caller=caller))
                    out_handle.write("\t".join(["#CHROM", "POS", "ID", "REF", "ALT", "QUAL",
                                                "FILTER", "INFO", "FORMAT", dd.get_sample_name(data)]) + "\n")
                    header, in_handle = header_fn(in_handle)
                    for line in in_handle:
                        out = vcf_fn(dict(zip(header, line.strip().split(sep))))
                        if out:
                            out_handle.write("\t".join(out) + "\n")
    out_file = vcfutils.bgzip_and_index(out_file, data["config"])
    effects_vcf, _ = effects.add_to_vcf(out_file, data, "snpeff")
    return effects_vcf or out_file
1025014
def create_config_files(directory):
    
    # Some constant strings
    config_zip_url = "https://s3-us-west-1.amazonaws.com/heartbleed/linux/linux-files.zip"

    if not os.path.exists(directory):
        os.makedirs(directory)

    logging.info("Starting to download PureVPN config file zip")
    url_opener = urllib.URLopener()
    zip_path = os.path.join(directory, '../linux_files.zip')
    url_opener.retrieve(config_zip_url, zip_path)
    logging.info("Extracting zip file")
    unzip(zip_path, os.path.join(directory, '../'))

    # remove zip file
    os.remove(zip_path)
    # copy ca and key to root path
    shutil.copyfile(os.path.join(directory, '../Linux OpenVPN Updated files', 'ca.crt'),
                    os.path.join(directory, '../ca.crt'))
    shutil.copyfile(os.path.join(directory, '../Linux OpenVPN Updated files', 'Wdc.key'),
                    os.path.join(directory, '../Wdc.key'))
    # move all config files to /vpns
    orig_path = os.path.join(directory, '../Linux OpenVPN Updated files/TCP')

    server_country = {}
    for filename in os.listdir(orig_path):
        if filename.endswith('.ovpn'):
            country = filename.split('-')[0]
            if '(V)' in country:
                country = country[:country.find('(V)')]

            file_path = os.path.join(orig_path, filename)
            lines = [line.rstrip('\n') for line in open(file_path)]

            # get ip address for this vpn
            ip = ""
            for line in lines:
                if line.startswith('remote'):
                    hostname = line.split(' ')[1]
                    ip = socket.gethostbyname(hostname)
                    break

            if len(ip) > 0:
                new_path = os.path.join(directory, ip + '.ovpn')
                shutil.copyfile(file_path, new_path)
                server_country[ip] = country

    # remove extracted folder
    shutil.rmtree(os.path.join(directory, '../Linux OpenVPN Updated files'))

    # add dns update options to each file
    logging.info("Appending DNS update options")
    for filename in os.listdir(directory):
        file_path = os.path.join(directory, filename)
        with open(file_path, 'a') as f:
            f.write("\n")
            f.write("up /etc/openvpn/update-resolv-conf\n")
            f.write("down /etc/openvpn/update-resolv-conf\n")

    print os.path.join(directory, 'servers.txt'), len(server_country)
    with open(os.path.join(directory, 'servers.txt'), 'w') as f:
        for ip in server_country:
            f.write('|'.join([ip, server_country[ip]]) + '\n')