Remove duplicates, filter for >Q, remove multiple mapping reads.
        For paired-end reads, keep only proper pairs.
518751
def filter_reads(self, input_bam, output_bam, metrics_file, paired=False, cpus=16, Q=30):
        
        nodups = re.sub("\.bam$", "", output_bam) + ".nodups.nofilter.bam"
        cmd1 = self.tools.sambamba + " markdup -t {0} -r --compression-level=0 {1} {2} 2> {3}".format(cpus, input_bam, nodups, metrics_file)
        cmd2 = self.tools.sambamba + ' view -t {0} -f bam --valid'.format(cpus)
        if paired:
            cmd2 += ' -F "not (unmapped or mate_is_unmapped) and proper_pair'
        else:
            cmd2 += ' -F "not unmapped'
        cmd2 += ' and not (secondary_alignment or supplementary) and mapping_quality >= {0}"'.format(Q)
        cmd2 += ' {0} |'.format(nodups)
        cmd2 += self.tools.sambamba + " sort -t {0} /dev/stdin -o {1}".format(cpus, output_bam)
        cmd3 = "if [[ -s {0} ]]; then rm {0}; fi".format(nodups)
        cmd4 = "if [[ -s {0} ]]; then rm {0}; fi".format(nodups + ".bai")
        return [cmd1, cmd2, cmd3, cmd4]
748647
def clean(self):
        
        # Note that we do not close the connection here -- somebody
        # may still be reading from it.
        while len(self.queue) > 0 and self._pair_stale(self.queue[0]):
            self.queue.pop(0)
691209
def _find_duplicates(seq):
    """"""
    seen = set()
    return [element for element in seq
            if seq.count(element) > 1
            and element not in seen and seen.add(element) is None]
815229
def clean_sample_data(samples):
    
    out = []
    for data in (utils.to_single_data(x) for x in samples):
        if "dirs" in data:
            data["dirs"] = {"work": data["dirs"]["work"], "galaxy": data["dirs"]["galaxy"],
                            "fastq": data["dirs"].get("fastq")}
        data["config"] = {"algorithm": data["config"]["algorithm"],
                          "resources": data["config"]["resources"]}
        for remove_attr in ["config_file", "algorithm"]:
            data.pop(remove_attr, None)
        out.append([data])
    return out
851061
public void removeMatching(MetricFilter filter) {
        metrics.entrySet().stream().filter(entry -> filter.matches(entry.getKey(), entry.getValue())).forEachOrdered(entry -> remove(entry.getKey()));
    }
324591
def remove_list_duplicates(lista, unique=False):
    
    result = []
    allready = []

    for elem in lista:
        if elem not in result:
            result.append(elem)
        else:
            allready.append(elem)

    if unique:
        for elem in allready:
            result = list(filter((elem).__ne__, result))

    return result
1033622
def check_fastq(self, input_files, output_files, paired_end):
        

        # Define a temporary function which we will return, to be called by the
        # pipeline.
        # Must define default parameters here based on the parameters passed in. This locks
        # these values in place, so that the variables will be defined when this function
        # is called without parameters as a follow function by pm.run.

        # This is AFTER merge, so if there are multiple files it means the
        # files were split into read1/read2; therefore I must divide by number
        # of files for final reads.
        def temp_func(input_files=input_files, output_files=output_files,
                      paired_end=paired_end):

            if type(input_files) != list:
                input_files = [input_files]
            if type(output_files) != list:
                output_files = [output_files]

            print(input_files)
            print(output_files)

            n_input_files = len(filter(bool, input_files))

            total_reads = sum([int(self.count_reads(input_file, paired_end))
                               for input_file in input_files])
            raw_reads = total_reads / n_input_files
            self.pm.report_result("Raw_reads", str(raw_reads))

            total_fastq_reads = sum(
                [int(self.count_reads(output_file, paired_end))
                 for output_file in output_files])
            fastq_reads = total_fastq_reads / n_input_files

            self.pm.report_result("Fastq_reads", fastq_reads)
            input_ext = self.get_input_ext(input_files[0])
            # We can only assess pass filter reads in bam files with flags.
            if input_ext == ".bam":
                num_failed_filter = sum(
                    [int(self.count_fail_reads(f, paired_end))
                     for f in input_files])
                pf_reads = int(raw_reads) - num_failed_filter
                self.pm.report_result("PF_reads", str(pf_reads))
            if fastq_reads != int(raw_reads):
                raise Exception("Fastq conversion error? Number of reads "
                                "doesn't match unaligned bam")

            return fastq_reads

        return temp_func
462568
def _clean_streams(repo, mapped_streams):
    """"""
    for stream_name in ('stdout', 'stderr'):
        stream = mapped_streams.get(stream_name)
        if not stream:
            continue

        path = os.path.relpath(stream, start=repo.working_dir)
        if (path, 0) not in repo.index.entries:
            os.remove(stream)
        else:
            blob = repo.index.entries[(path, 0)].to_blob(repo)
            with open(path, 'wb') as fp:
                fp.write(blob.data_stream.read())
593583
def remove_duplicates(lst):
    
    dset = set()
    return [l for l in lst if l not in dset and not dset.add(l)]
415390
private static void remove(AnalyzedTokenReadings readings, Pattern pattern) {
      List<AnalyzedToken> analyzedTokens = readings.getReadings();
      for (int j = analyzedTokens.size()-1; j>=0; j--) {
        AnalyzedToken analyzedToken = analyzedTokens.get(j);

        if( ! JLanguageTool.SENTENCE_END_TAGNAME.equals(analyzedToken.getPOSTag())
            && ! PosTagHelper.hasPosTag(analyzedToken, pattern) ) {
          readings.removeReading(analyzedToken);
        }
      }

  }
1244931
def __remove_duplicates(self, _other):
        """"""
        if not isinstance(_other, type(self)) \
           and not isinstance(_other, type(list)) \
           and not isinstance(_other, type([])):
            other = [_other]
        else:
            other = list(_other)

        # remove items already in self
        newother = []
        for i in range(0, len(other)):
            item = other.pop(0)
            if not list.__contains__(self, item):
                newother.append(item)

        # remove duplicate items in other
        other = []
        if newother != []:
            other.append(newother[0])
            for i in range(1, len(newother)):
                item = newother.pop()
                if not other.__contains__(item):
                    other.append(item)
        return other
931901
@Override
    public void removeMatching(MetricFilter filter) {
        for (Map.Entry<String, Metric> entry : metrics.entrySet()) {
            if (filter.matches(entry.getKey(), entry.getValue())) {
                remove(entry.getKey());
            }
        }
    }
440354
def readers_to_cleanup(reader_class)
      reader_class.
        reader_scope.
        joins(:read_marks).
        where(ReadMark.table_name => { readable_type: readable_class.name }).
        group("#{ReadMark.quoted_table_name}.reader_type, #{ReadMark.quoted_table_name}.reader_id, #{reader_class.quoted_table_name}.#{reader_class.quoted_primary_key}").
        having("COUNT(#{ReadMark.quoted_table_name}.id) > 1").
        select("#{reader_class.quoted_table_name}.#{reader_class.quoted_primary_key}")
    end
128688
def remove_duplicates(seq):
    '''
    
    '''
    last_boundary = False
    for char in seq:
        if char == '\x00':
            if not last_boundary:
                last_boundary = True
                yield char
        else:
            last_boundary = False
            yield char
1170755
def filter(self, readAlignments):
        

        # Implementation notes:
        #
        # 1. The order in which we carry out the filtering actions can make
        #    a big difference in the result of this function. The current
        #    ordering is based on what seems reasonable - it may not be the
        #    best way to do things. E.g., if maxHspsPerHit is 1 and there
        #    is a title regex, which should we perform first?
        #
        #    We perform filtering based on alignment before that based on
        #    HSPs. That's because there's no point filtering all HSPs for
        #    an alignment that we end up throwing away anyhow.
        #
        # 2. This function could be made faster if it first looked at its
        #    arguments and dynamically created an acceptance function
        #    (taking a readAlignments as an argument). The acceptance
        #    function would run without examining the desired filtering
        #    settings on each call the way the current code does.
        #
        # 3. A better approach with readIdRegex would be to allow the
        #    passing of a regex object. Then the caller would make the
        #    regex with whatever flags they liked (e.g., case insensitive).

        #
        # Alignment-only (i.e., non-HSP based) filtering.
        #
        if self.limit is not None and self.count == self.limit:
            return False

        # Does the read have too many alignments?
        if (self.maxAlignmentsPerRead is not None and
                len(readAlignments) > self.maxAlignmentsPerRead):
            return False

        # Filter on the read id.
        if (self.readIdRegex and
                self.readIdRegex.search(readAlignments.read.id) is None):
            return False

        if self.titleFilter:
            # Remove alignments against sequences whose titles are
            # unacceptable.
            wantedAlignments = []
            for alignment in readAlignments:
                if (self.titleFilter.accept(alignment.subjectTitle) !=
                        TitleFilter.REJECT):
                    wantedAlignments.append(alignment)
            if wantedAlignments:
                readAlignments[:] = wantedAlignments
            else:
                return False

        # Only return alignments that are against sequences of the
        # desired length.
        minSequenceLen = self.minSequenceLen
        maxSequenceLen = self.maxSequenceLen
        if minSequenceLen is not None or maxSequenceLen is not None:
            wantedAlignments = []
            for alignment in readAlignments:
                length = alignment.subjectLength
                if not ((minSequenceLen is not None and
                         length < minSequenceLen) or
                        (maxSequenceLen is not None and
                         length > self.maxSequenceLen)):
                    wantedAlignments.append(alignment)
            if wantedAlignments:
                readAlignments[:] = wantedAlignments
            else:
                return False

        if self.taxonomy is not None:
            wantedAlignments = []
            for alignment in readAlignments:
                lineage = self.lineageFetcher.lineage(alignment.subjectTitle)
                if lineage:
                    for taxonomyIdAndScientificName in lineage:
                        if self.taxonomy in taxonomyIdAndScientificName:
                            wantedAlignments.append(alignment)
                else:
                    # No lineage info was found. Keep the alignment
                    # since we can't rule it out.  We could add another
                    # option to control this.
                    wantedAlignments.append(alignment)
            if wantedAlignments:
                readAlignments[:] = wantedAlignments
            else:
                return False

        if self.oneAlignmentPerRead and readAlignments:
            readAlignments[:] = [bestAlignment(readAlignments)]

        #
        # From here on we do only HSP-based filtering.
        #

        # Throw out any unwanted HSPs due to maxHspsPerHit.
        if self.maxHspsPerHit is not None:
            for alignment in readAlignments:
                hsps = alignment.hsps
                if len(hsps) > self.maxHspsPerHit:
                    alignment.hsps = hsps[:self.maxHspsPerHit]

        # Throw out HSPs whose scores are not good enough.
        if self.scoreCutoff is not None:
            wantedAlignments = []
            for alignment in readAlignments:
                hsps = alignment.hsps
                wantedHsps = []
                for hsp in hsps:
                    if hsp.betterThan(self.scoreCutoff):
                        wantedHsps.append(hsp)
                if wantedHsps:
                    alignment.hsps = wantedHsps
                    wantedAlignments.append(alignment)
            if wantedAlignments:
                readAlignments[:] = wantedAlignments
            else:
                return False

        # Throw out HSPs that don't match in the desired place on the
        # matched sequence.
        minStart = self.minStart
        maxStop = self.maxStop
        if minStart is not None or maxStop is not None:
            wantedAlignments = []
            for alignment in readAlignments:
                hsps = alignment.hsps
                wantedHsps = []
                for hsp in hsps:
                    if not ((minStart is not None and
                             hsp.readStartInSubject < minStart) or
                            (maxStop is not None and
                             hsp.readEndInSubject > maxStop)):
                        wantedHsps.append(hsp)
                if wantedHsps:
                    alignment.hsps = wantedHsps
                    wantedAlignments.append(alignment)
            if wantedAlignments:
                readAlignments[:] = wantedAlignments
            else:
                return False

        self.count += 1
        return readAlignments
959938
def filter_multimappers(align_file, data):
    
    config = dd.get_config(data)
    type_flag = "" if bam.is_bam(align_file) else "S"
    base, ext = os.path.splitext(align_file)
    out_file = base + ".unique" + ext
    bed_file = dd.get_variant_regions(data) or dd.get_sample_callable(data)
    bed_cmd = '-L {0}'.format(bed_file) if bed_file else " "
    if utils.file_exists(out_file):
        return out_file
    base_filter = '-F "not unmapped {paired_filter} and not duplicate and [XA] == null and [SA] == null and not supplementary " '
    if bam.is_paired(align_file):
        paired_filter = "and paired and proper_pair"
    else:
        paired_filter = ""
    filter_string = base_filter.format(paired_filter=paired_filter)
    sambamba = config_utils.get_program("sambamba", config)
    num_cores = dd.get_num_cores(data)
    with file_transaction(out_file) as tx_out_file:
        cmd = ('{sambamba} view -h{type_flag} '
               '--nthreads {num_cores} '
               '-f bam {bed_cmd} '
               '{filter_string} '
               '{align_file} '
               '> {tx_out_file}')
        message = "Removing multimapped reads from %s." % align_file
        do.run(cmd.format(**locals()), message)
    bam.index(out_file, config)
    return out_file
833882
def filter_single_reads_by_length(in_file, quality_format, min_length=20,
                                  out_file=None):
    
    logger.info("Removing reads in %s thare are less than %d bases."
                % (in_file, min_length))
    in_iterator = SeqIO.parse(in_file, quality_format)
    out_iterator = (record for record in in_iterator if
                    len(record.seq) > min_length)
    with file_transaction(out_file) as tmp_out_file:
        with open(tmp_out_file, "w") as out_handle:
            SeqIO.write(out_iterator, out_handle, quality_format)
    return out_file