strip out columns of a MSA that represent gaps for X percent (threshold) of sequences
503756
def strip_msa_100(msa, threshold, plot = False):
	
	msa = [seq for seq in parse_fasta(msa)]
	columns = [[0, 0] for pos in msa[0][1]] # [[#bases, #gaps], [#bases, #gaps], ...]
	for seq in msa:
		for position, base in enumerate(seq[1]):
			if base == '-' or base == '.':
				columns[position][1] += 1
			else:
				columns[position][0] += 1
	columns = [float(float(g)/float(g+b)*100) for b, g in columns] # convert to percent gaps
	for seq in msa:
		stripped = []
		for position, base in enumerate(seq[1]):
			if columns[position] < threshold:
				stripped.append(base)
		yield [seq[0], ''.join(stripped)]
	if plot is not False:
		plot_gaps(plot, columns)
217784
def _remove_blank(l):
        
        ret = []
        for i, _ in enumerate(l):
            if l[i] == 0:
                break
            ret.append(l[i])
        return ret
88999
def remove_empty_cols(records):
    """"""
    # In case it's a generator, turn it into a list
    records = list(records)
    seqstrs = [str(rec.seq) for rec in records]
    clean_cols = [col
                  for col in zip(*seqstrs)
                  if not all(c == '-' for c in col)]
    clean_seqs = [''.join(row)
                  for row in zip(*clean_cols)]
    for rec, clean_seq in zip(records, clean_seqs):
        yield SeqRecord(Seq(clean_seq, rec.seq.alphabet), id=rec.id,
                        name=rec.name, description=rec.description,
                        dbxrefs=rec.dbxrefs, features=rec.features,
                        annotations=rec.annotations,
                        letter_annotations=rec.letter_annotations)
978052
def trimmed_split(s, seps=(";", ",")):
    """"""
    for sep in seps:
        if sep not in s:
            continue
        data = [item.strip() for item in s.strip().split(sep)]
        return data
    return [s]
1342490
def colformat(strings, num_sep_spaces=1, terminal_width=None):
    """"""
    if terminal_width is None:
        terminal_width = get_terminal_size().columns

    if not strings:
        return

    num_items = len(strings)
    max_len = max([len(strip_ansi_escapes(s)) for s in strings])

    num_columns = min(
        int((terminal_width + num_sep_spaces) / (max_len + num_sep_spaces)),
        num_items)
    num_columns = max(1, num_columns)

    num_lines = int(math.ceil(float(num_items) / num_columns))
    num_columns = int(math.ceil(float(num_items) / num_lines))

    num_elements_last_column = num_items % num_lines
    if num_elements_last_column == 0:
        num_elements_last_column = num_lines

    lines = []
    for i in range(num_lines):
        line_size = num_columns
        if i >= num_elements_last_column:
            line_size -= 1
        lines.append([None] * line_size)

    for i, line in enumerate(lines):
        line_size = len(line)
        for j in range(line_size):
            k = i + num_lines * j
            item = strings[k]
            if j % line_size != line_size - 1:
                item_len = len(strip_ansi_escapes(item))
                item = item + ' ' * (max_len - item_len)
            line[j] = item

    sep = ' ' * num_sep_spaces
    for line in lines:
        yield sep.join(line)
1176830
def plot_gaps(plot, columns):
	
	from plot_window import window_plot_convolve as plot_window
#	plot_window([columns], len(columns)*.01, plot)
	plot_window([[100 - i for i in columns]], len(columns)*.01, plot)
372469
def _strip(self, tokens, criteria, prop, rev=False):
        """"""
        num = len(tokens)
        res = []
        for i, token in enumerate(reversed(tokens) if rev else tokens):
            if criteria(token) and num > i + 1:
                res.insert(0, tokens.pop()) if rev else res.append(tokens.pop(0))
            else:
                break
        if res:
            self[prop] = self._clean(' '.join(res))
        return tokens
533524
def isolate_region(sequences, start, end, gap_char='-'):
    
    # Check arguments
    if end <= start:
        raise ValueError("start of slice must precede end ({0} !> {1})".format(
            end, start))

    for sequence in sequences:
        seq = sequence.seq
        start_gap = gap_char * start
        end_gap = gap_char * (len(seq) - end)
        seq = Seq(start_gap + str(seq[start:end]) + end_gap,
                alphabet=seq.alphabet)
        sequence.seq = seq
        yield sequence
372025
def trim(self, count, approximate=True):
        
        return self.database.xtrim(self.key, count, approximate)
983956
def strip_tail(sequence, values):
    """"""
    return list(reversed(list(strip_head(reversed(sequence), values))))
829299
def mask_sequence(seq, gaps):
    
    seq = [i.upper() for i in seq]
    for gap in gaps:
        for i in range(gap[0] - 1, gap[1]):
            seq[i] = seq[i].lower()
    return ''.join(seq)
101401
def process_line(self, line):
        ""
        cleaned = []
        columns = line.split(self.indel)
        # Populate indices if not defined
        if not self.indices:
            self.indices = range(len(columns))
        for i in self.indices:
            # Support turning an in col into multiple out cols
            out = self.process_column(i, columns[i])
            if isinstance(out, (list, tuple)):
                cleaned.extend(out)
            else:
                cleaned.append(out)
        return self.outdel.join(cleaned) + '\n'
54963
def remove_all_gap_columns( self ):
        
        seqs = []
        for c in self.components:
            try:
                seqs.append( list( c.text ) )
            except TypeError:
                seqs.append( None )
        i = 0
        text_size = self.text_size
        while i < text_size:
            all_gap = True
            for seq in seqs:
                if seq is None: continue
                if seq[i] != '-': all_gap = False
            if all_gap:
                for seq in seqs:
                    if seq is None: continue
                    del seq[i]
                text_size -= 1
            else:
                i += 1
        for i in range( len( self.components ) ):
            if seqs[i] is None: continue
            self.components[i].text = ''.join( seqs[i] )
        self.text_size = text_size
1173088
def _scan_smaller(self, seq, threshold=''):
        
        ll = self.ll #Shortcut for Log-likelihood matrix
        matches   = []
        endpoints = []
        scores    = []
        w         = self.width
        oseq      = seq
        seq       = seq.upper()
        for offset in range(self.width-len(seq)+1):    #Check if +/-1 needed
            maximum = 0
            for i in range(len(seq)):
                maximum = maximum + max(ll[i+offset].values())
            if not threshold: threshold = 0.8 * maximum
            total_f = 0
            total_r = 0
            for i in range(len(seq)):
                total_f = total_f + ll[i+offset      ][        seq[i] ]
                total_r = total_r + ll[w-(i+offset)-1][revcomp[seq[i]]]
            if 0:
                print "\t\t%s vs %s: F=%6.2f R=%6.2f %6.2f %4.2f"%(oseq, self.oneletter[offset:offset+len(seq)],
                                                                   total_f, total_r,  maximum,
                                                                   max([total_f,total_r])/self.maxscore)
            if total_f > threshold and total_f > total_r:
                endpoints.append( (offset,offset+self.width-1) )
                scores.append(total_f)
                matches.append(oseq[offset:offset+self.width])
            elif total_r > threshold:
                endpoints.append( (offset,offset+self.width-1) )
                scores.append(total_r)
                matches.append(oseq[offset:offset+self.width])
        return(matches,endpoints,scores)
708333
def _filter_matrix_columns(cls, matrix, phandango_header, csv_header):
        ''''''
        indexes_to_keep = set()

        for row in matrix:
            for i in range(len(row)):
                if row[i] not in {'NA', 'no'}:
                    indexes_to_keep.add(i)

        indexes_to_keep = sorted(list(indexes_to_keep))

        for i in range(len(matrix)):
            matrix[i] = [matrix[i][j] for j in indexes_to_keep]

        phandango_header = [phandango_header[i] for i in indexes_to_keep]
        csv_header = [csv_header[i] for i in indexes_to_keep]
        return phandango_header, csv_header, matrix
209883
def trim(sequences, start, end):
    
    logging.info("Trimming from %d to %d", start, end)
    return (sequence[start:end] for sequence in sequences)
1180438
def run_gap_k_selection(data, k_min=1, k_max=50, B=5,
        skip=5, **kwargs):
    
    if k_min == k_max:
        return k_min
    gap_vals = []
    sk_vals = []
    k_range = list(range(k_min, k_max, skip))
    min_k = 0
    min_i = 0
    for i, k in enumerate(k_range):
        km = KMeans(k)
        clusters = km.fit_predict(data)
        gap, sk = calculate_gap(data, clusters, km, B=B)
        if len(gap_vals) > 1:
            if gap_vals[-1] >= gap - (skip+1)*sk:
                min_i = i
                min_k = k_range[i-1]
                break
                #return k_range[-1], gap_vals, sk_vals
        gap_vals.append(gap)
        sk_vals.append(sk)
    if min_k == 0:
        min_k = k_max
    if skip == 1:
        return min_k, gap_vals, sk_vals
    gap_vals = []
    sk_vals = []
    for k in range(min_k - skip, min_k + skip):
        km = KMeans(k)
        clusters = km.fit_predict(data)
        gap, sk = calculate_gap(data, clusters, km, B=B)
        if len(gap_vals) > 1:
            if gap_vals[-1] >= gap - sk:
                min_k = k-1
                return min_k, gap_vals, sk_vals
        gap_vals.append(gap)
        sk_vals.append(sk)
    return k, gap_vals, sk_vals