Filter the reads into separate files based on taxonomic assignment
164656
def fastqfilter(self):
        """"""
        printtime('Creating filtered .fastqfiles', self.start)
        # Create and start threads
        for i in range(self.cpus):
            # Send the threads to the appropriate destination function
            threads = Thread(target=self.filterfastq, args=())
            # Set the daemon to true - something to do with thread management
            threads.setDaemon(True)
            # Start the threading
            threads.start()
        for sample in self.runmetadata.samples:
            self.filterqueue.put(sample)
        self.filterqueue.join()
        # Print the metadata to file
        metadataprinter.MetadataPrinter(self)
446813
def Filter(self):
    """"""
    if not self.desc:
      self.short_desc = ''
      return

    for i in range(len(self.desc)):   # replace full path with name
      if self.desc[i].find(self.executable) >= 0:
        self.desc[i] = self.desc[i].replace(self.executable, self.name)

    self.short_desc = self.desc[0]
    word_list = self.short_desc.split(' ')
    all_names = [ self.name, self.short_name, ]
    # Since the short_desc is always listed right after the name,
    #  trim it from the short_desc
    while word_list and (word_list[0] in all_names
                         or word_list[0].lower() in all_names):
      del word_list[0]
      self.short_desc = ''              # signal need to reconstruct
    if not self.short_desc and word_list:
      self.short_desc = ' '.join(word_list)
1336412
private void replaceWithFinishedReaders(List<SSTableReader> finished)
    {
        if (isOffline)
        {
            for (SSTableReader reader : discard)
            {
                if (reader.getCurrentReplacement() == reader)
                    reader.markObsolete();
                reader.selfRef().release();
            }
        }
        else
        {
            dataTracker.replaceEarlyOpenedFiles(discard, finished);
            dataTracker.unmarkCompacting(discard);
        }
        discard.clear();
    }
878115
def sample_reads(self):
        
        logging.info('Read sampling')
        for sample in self.metadata:
            # Iterate through all the desired depths of coverage
            for depth in self.read_depths:
                for read_pair in self.read_lengths:
                    # Set the name of the output directory
                    sample.sampled_reads[depth][read_pair].sampled_outputdir \
                        = os.path.join(sample.sampled_reads[depth][read_pair].outputdir, 'sampled')
                    # Set the name of the forward reads - include the depth and read length information
                    sample.sampled_reads[depth][read_pair].forward_reads.fastq = \
                        os.path.join(sample.sampled_reads[depth][read_pair].sampled_outputdir,
                                     '{name}_{depth}_{read_pair}_R1.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    # Reverse reads
                    sample.sampled_reads[depth][read_pair].reverse_reads.fastq = \
                        os.path.join(sample.sampled_reads[depth][read_pair].sampled_outputdir,
                                     '{name}_{depth}_{read_pair}_R2.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    logging.info(
                        'Sampling {num_reads} paired reads for sample {name} with the following parameters:\n'
                        'depth {dp}, forward reads {fl}bp, and reverse reads {rl}bp'
                        .format(num_reads=sample.simulated_reads[depth][read_pair].num_reads,
                                dp=depth,
                                name=sample.name,
                                fl=sample.sampled_reads[depth][read_pair].forward_reads.length,
                                rl=sample.sampled_reads[depth][read_pair].reverse_reads.length))
                    # Use the reformat method in the OLCTools bbtools wrapper
                    # Note that upsample=t is used to ensure that the target number of reads (samplereadstarget) is met
                    if not os.path.isfile(sample.sampled_reads[depth][read_pair].forward_reads.trimmed_sampled_fastq):
                        out, \
                            err, \
                            sample.sampled_reads[depth][read_pair].sample_call = bbtools \
                            .reformat_reads(forward_in=sample.sampled_reads[depth][read_pair].trimmed_forwardfastq,
                                            reverse_in=sample.sampled_reads[depth][read_pair].trimmed_reversefastq,
                                            forward_out=sample.sampled_reads[depth][read_pair].forward_reads.fastq,
                                            reverse_out=sample.sampled_reads[depth][read_pair].reverse_reads.fastq,
                                            returncmd=True,
                                            **{'samplereadstarget': sample.simulated_reads[depth][read_pair].num_reads,
                                               'upsample': 't',
                                               'minlength':
                                                   sample.sampled_reads[depth][read_pair].forward_reads.length,
                                               'ziplevel': '9',
                                               'tossbrokenreads': 't',
                                               'tossjunk': 't',
                                               'Xmx': self.mem
                                               }
                                            )
                    # # Remove the trimmed reads, as they are no longer necessary
                    # try:
                    #     os.remove(sample.sampled_reads[depth][read_pair].trimmed_forwardfastq)
                    #     os.remove(sample.sampled_reads[depth][read_pair].trimmed_reversefastq)
                    # except FileNotFoundError:
                    #     pass
            # Update the JSON file
            self.write_json(sample)
590422
def split(*items):
    
    out = []
    for data in [x[0] for x in items]:
        dis_orgs = data["config"]["algorithm"].get("disambiguate")
        if dis_orgs:
            if not data.get("disambiguate", None):
                data["disambiguate"] = {"genome_build": data["genome_build"],
                                        "base": True}
            out.append([data])
            # handle the instance where a single organism is disambiguated
            if isinstance(dis_orgs, six.string_types):
                dis_orgs = [dis_orgs]
            for dis_org in dis_orgs:
                dis_data = copy.deepcopy(data)
                dis_data["disambiguate"] = {"genome_build": dis_org}
                dis_data["genome_build"] = dis_org
                dis_data["config"]["algorithm"]["effects"] = False
                dis_data = run_info.add_reference_resources(dis_data)
                out.append([dis_data])
        else:
            out.append([data])
    return out
579653
def filter_taxa(records, taxids, unclassified=False, discard=False):
    '''
     
    '''
    taxids = set(taxids)
    kept_records = []
    ncbi = ete3.NCBITaxa()
    unclassified_taxids = {0,1}
    for r in records:
        taxid, rank, sciname, lineage = r.description.strip().partition(' ')[2].split('|')
        taxid = int(taxid)
        lineage = set(ncbi.get_lineage(taxid) or [0])  # lineage defined twice?
        intersection = lineage & taxids
        if taxid in unclassified_taxids and unclassified:
            kept_records.append(r)
        elif intersection and not discard:
            kept_records.append(r)
        elif not intersection and discard and taxid not in unclassified_taxids:
            kept_records.append(r)

    return kept_records
1132604
function filterFile(file, done) {
      // Ensure it matches the extension.
      const correctExtention = path.extname(file) === opts.extname

      // Make sure it has defined files.
      const hasFiles = files[file].files
      done(null, correctExtention && hasFiles)
    }
490749
def _parse_file_spec(self, spec):
        '''
        
        '''
        # separate wildcard specs into more specs
        if '*' in spec['file']:
            expanded_paths = _expand_paths(spec['file'])
            if not expanded_paths:
                return []
            expanded_specs = []
            for p in expanded_paths:
                _spec = copy.copy(spec)
                _spec['file'] = p
                expanded_specs.append(_spec)
            return expanded_specs

        else:
            return [spec]
130265
def process_species(self, limit):
        
        myfile = '/'.join((self.rawdir, self.files['data']['file']))
        fh = gzip.open(myfile, 'rb')
        filereader = io.TextIOWrapper(fh, newline="")
        filereader.readline()  # remove the xml declaration line
        for event, elem in ET.iterparse(filereader):  # iterparse is not deprecated
            # Species ids are == NCBITaxon ids
            self.process_xml_table(
                elem, 'Species_gb', self._process_species_table_row, limit)
        fh.close()
        return
823053
def _discarded_reads2_out_file_name(self):
        """"""
        if self.Parameters['-4'].isOn():
            discarded_reads2 = self._absolute(str(self.Parameters['-4'].Value))
        else:
            raise ValueError(
                "No discarded-reads2 (flag -4) output path specified")
        return discarded_reads2
89919
def select_files_from_pathnames(self, filenames):
        """"""
        selected_filenames = []

        for pattern in self.file_patterns:
            matching = match_filenames(filenames, pattern)
            for fname in matching:
                if fname not in selected_filenames:
                    selected_filenames.append(fname)
        if len(selected_filenames) == 0:
            logger.warning("No filenames found for reader: %s", self.name)
        return selected_filenames
547269
def main():
    """"""
    import argparse
    prs = argparse.ArgumentParser(__doc__,
                                  formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    prs.add_argument('--taxon_id', type=str,
                     help='NCBI taxon ID, must match exact species/strain used by GO Central, e.g. 4896 for S Pombe')
    prs.add_argument('--golr_url', default='http://golr.geneontology.org/solr/', type=str,
                     help='NCBI taxon ID, must match exact species/strain used by GO Central, e.g. 4896 for S Pombe')
    prs.add_argument('-o', default=None, type=str,
                     help="Specifies the name of the output file")
    prs.add_argument('--max_rows', default=100000, type=int,
                     help="maximum rows to be fetched")

    args = prs.parse_args()

    solr = pysolr.Solr(args.golr_url, timeout=30)

    sys.stderr.write("TAX:"+args.taxon_id+"\n")
    results = solr.search(q='document_category:"bioentity" AND taxon:"NCBITaxon:'+args.taxon_id+'"',
                          fl='bioentity_label,annotation_class_list', rows=args.max_rows)
    sys.stderr.write("NUM GENES:"+str(len(results))+"\n")
    if (len(results) ==0):
        sys.stderr.write("NO RESULTS")
        exit(1)
    if (len(results) == args.max_rows):
        sys.stderr.write("max_rows set too low")
        exit(1)

    file_out = sys.stdout if args.o is None else open(args.o, 'w')
    for r in results:
        gene_symbol = r['bioentity_label']
        sys.stderr.write(gene_symbol+"\n")
        if 'annotation_class_list' in r:
            file_out.write(r['bioentity_label']+"\t" + ';'.join(r['annotation_class_list'])+"\n")
        else:
            sys.stderr.write("no annotations for "+gene_symbol+"\n")
            

    if args.o is not None:
        file_out.close()
        sys.stdout.write("  WROTE: {}\n".format(args.o))
241545
def subset_to_genome(in_file, out_file, data):
    
    if not utils.file_uptodate(out_file, in_file):
        contigs = set([x.name for x in ref.file_contigs(dd.get_ref_file(data))])
        with utils.open_gzipsafe(in_file) as in_handle:
            with file_transaction(data, out_file) as tx_out_file:
                with open(tx_out_file, "w") as out_handle:
                    for line in in_handle:
                        parts = line.split()
                        if parts and parts[0] in contigs:
                            out_handle.write(line)
    return out_file
211035
function filterRepresented(modules, files, options) {
    return _.filter(modules, function(module) {
      return !_.some(files, function(file) {
        // Look for the module name somewhere in the source path
        return (
          path
            .join(sep, options.srcPrefix, file.src.replace(rmain, '$1'), sep)
            .indexOf(sep + module + sep) > -1
        )
      })
    })
  }
794300
def add_reads(self, reads):
        
        if len(reads) == 0:
            return self
        new_reads = self.reads.union(reads)
        if len(new_reads) > len(self.reads):
            return VariantSequence(
                prefix=self.prefix,
                alt=self.alt,
                suffix=self.suffix,
                reads=new_reads)
        else:
            return self
703664
function filteredFiles(folder, pattern) {
  var files = [];
    
  if (fs.existsSync(folder)){
    fs.readdirSync(folder).forEach(function(file) {
      if (file.match(pattern) != null)
        files.push(file);
    });
  }
  return files;
}
854924
def filter_to_pass_and_reject(in_file, paired, out_dir=None):
    
    from bcbio.heterogeneity import bubbletree
    out_file = "%s-prfilter.vcf.gz" % utils.splitext_plus(in_file)[0]
    if out_dir:
        out_file = os.path.join(out_dir, os.path.basename(out_file))
    if not utils.file_uptodate(out_file, in_file):
        with file_transaction(paired.tumor_data, out_file) as tx_out_file:
            max_depth = bubbletree.max_normal_germline_depth(in_file, bubbletree.PARAMS, paired)
            tx_out_plain = tx_out_file.replace(".vcf.gz", ".vcf")
            with contextlib.closing(cyvcf2.VCF(in_file)) as reader:
                reader = _add_db_to_header(reader)
                with contextlib.closing(cyvcf2.Writer(tx_out_plain, reader)) as writer:
                    for rec in reader:
                        filters = rec.FILTER.split(";") if rec.FILTER else []
                        other_filters = [x for x in filters if x not in ["PASS", ".", "REJECT"]]
                        if len(other_filters) == 0 or bubbletree.is_info_germline(rec):
                            # Germline, check if we should include based on frequencies
                            if "REJECT" in filters or bubbletree.is_info_germline(rec):
                                stats = bubbletree._is_possible_loh(rec, reader, bubbletree.PARAMS, paired,
                                                                    use_status=True, max_normal_depth=max_depth)
                                if stats:
                                    rec.FILTER = "PASS"
                                    rec.INFO["DB"] = True
                                    writer.write_record(rec)
                            # Somatic, always include
                            else:
                                writer.write_record(rec)
            vcfutils.bgzip_and_index(tx_out_plain, paired.tumor_data["config"])
    return out_file