Cleans the list of strings given as `pieces`.  It replaces
        multiple newlines by a maximum of 2 and returns a new list.
        It also wraps the paragraphs nicely.
168172
def clean_pieces(self, pieces):
        
        ret = []
        count = 0
        for i in pieces:
            if i == '\n':
                count = count + 1
            else:
                if i == '";':
                    if count:
                        ret.append('\n')
                elif count > 2:
                    ret.append('\n\n')
                elif count:
                    ret.append('\n'*count)
                count = 0
                ret.append(i)

        _data = "".join(ret)
        ret = []
        for i in _data.split('\n\n'):
            if i == 'Parameters:':
                ret.extend(['Parameters:\n-----------', '\n\n'])
            elif i.find('// File:') > -1: # leave comments alone.
                ret.extend([i, '\n'])
            else:
                _tmp = textwrap.fill(i.strip())
                _tmp = self.lead_spc.sub(r'\1"\2', _tmp)
                ret.extend([_tmp, '\n\n'])
        return ret
1098720
def split_pieces(piece_list, segments, num):
    
    piece_groups = []
    pieces = list(piece_list)
    while pieces:
        for i in range(segments):
            p = pieces[i::segments][:num]
            if not p:
                break
            piece_groups.append(p)
        pieces = pieces[num * segments:]

    return piece_groups
322630
def remove_blank_lines(string):
    
    return "\n".join(line
                     for line in string.split("\n")
                     if len(line.strip()))
70576
function clean(parts) {
    var p = '';
    for (var i = 0; i < parts.length; i += 3) {
        p += parts[i] || '';
    }
    return p;
}
190693
private static <L extends List<String>> L trimStringList(L strings)
    {
        for (ListIterator<String> listIt = strings.listIterator(); listIt.hasNext(); )
        {
            String string = listIt.next().trim();
            listIt.set(string);
        }
        return strings;
    }
1305314
def _clean_text(self, text):
        """"""
        output = []
        for char in text:
            cp = ord(char)
            if cp in (0, 0xfffd) or self._is_control(char):
                continue
            if self._is_whitespace(char):
                output.append(' ')
            else:
                output.append(char)
        return ''.join(output)
1022679
def clean(self, text, **kwargs):
        
        if sys.version_info < (3, 0):
            # Only in Python 2. In 3 every string is a Python 2 unicode
            if not isinstance(text, unicode):
                raise exceptions.UnicodeRequired

        clean_chunks = []
        filth = Filth()
        for next_filth in self.iter_filth(text):
            clean_chunks.append(text[filth.end:next_filth.beg])
            clean_chunks.append(next_filth.replace_with(**kwargs))
            filth = next_filth
        clean_chunks.append(text[filth.end:])
        return u''.join(clean_chunks)
992031
def clean(ctx, text):
    
    text = conversions.to_string(text, ctx)
    return ''.join([c for c in text if ord(c) >= 32])
476
function breakLines(tokens, maxWidth) {
    if (!maxWidth) {
        maxWidth = Infinity;
    }
    let i = 0;
    let lineLength = 0;
    let lastTokenWithSpace = -1;
    while (i < tokens.length) { /* take all text tokens, remove space, apply linebreaks */
        let token = tokens[i];
        if (token.type == TYPE_NEWLINE) { /* reset */
            lineLength = 0;
            lastTokenWithSpace = -1;
        }
        if (token.type != TYPE_TEXT) { /* skip non-text tokens */
            i++;
            continue;
        }
        /* remove spaces at the beginning of line */
        while (lineLength == 0 && token.value.charAt(0) == " ") {
            token.value = token.value.substring(1);
        }
        /* forced newline? insert two new tokens after this one */
        let index = token.value.indexOf("\n");
        if (index != -1) {
            token.value = breakInsideToken(tokens, i, index, true);
            /* if there are spaces at the end, we must remove them (we do not want the line too long) */
            let arr = token.value.split("");
            while (arr.length && arr[arr.length - 1] == " ") {
                arr.pop();
            }
            token.value = arr.join("");
        }
        /* token degenerated? */
        if (!token.value.length) {
            tokens.splice(i, 1);
            continue;
        }
        if (lineLength + token.value.length > maxWidth) { /* line too long, find a suitable breaking spot */
            /* is it possible to break within this token? */
            let index = -1;
            while (1) {
                let nextIndex = token.value.indexOf(" ", index + 1);
                if (nextIndex == -1) {
                    break;
                }
                if (lineLength + nextIndex > maxWidth) {
                    break;
                }
                index = nextIndex;
            }
            if (index != -1) { /* break at space within this one */
                token.value = breakInsideToken(tokens, i, index, true);
            }
            else if (lastTokenWithSpace != -1) { /* is there a previous token where a break can occur? */
                let token = tokens[lastTokenWithSpace];
                let breakIndex = token.value.lastIndexOf(" ");
                token.value = breakInsideToken(tokens, lastTokenWithSpace, breakIndex, true);
                i = lastTokenWithSpace;
            }
            else { /* force break in this token */
                token.value = breakInsideToken(tokens, i, maxWidth - lineLength, false);
            }
        }
        else { /* line not long, continue */
            lineLength += token.value.length;
            if (token.value.indexOf(" ") != -1) {
                lastTokenWithSpace = i;
            }
        }
        i++; /* advance to next token */
    }
    tokens.push({ type: TYPE_NEWLINE }); /* insert fake newline to fix the last text line */
    /* remove trailing space from text tokens before newlines */
    let lastTextToken = null;
    for (let i = 0; i < tokens.length; i++) {
        let token = tokens[i];
        switch (token.type) {
            case TYPE_TEXT:
                lastTextToken = token;
                break;
            case TYPE_NEWLINE:
                if (lastTextToken) { /* remove trailing space */
                    let arr = lastTextToken.value.split("");
                    while (arr.length && arr[arr.length - 1] == " ") {
                        arr.pop();
                    }
                    lastTextToken.value = arr.join("");
                }
                lastTextToken = null;
                break;
        }
    }
    tokens.pop(); /* remove fake token */
    return tokens;
}
671494
def clean(string):
    '''  '''
    # manual fixes in the source
    # 24/1.png
    # 9/1.png
    # 3/1.png
    #if ')' in string:  # fix in the source data
        #string = string.split(')')[0] + ')'  # remove trailing garbage
    return (string
            .replace('_', '')
            .replace('-', '')
            .replace('—', '')
            .replace('.', '')
            .replace('=', '')
            .replace('\u2018',"'")  # LEFT SINGLE QUOTATION MARK
            .replace('\u2019', "'")  # RIGHT SINGLE QUOTATION MARK
            .strip())
893676
def fix_line_breaks(s):
    
    l = s.splitlines()
    x = [i.strip() for i in l]
    x = [i for i in x if i]  # remove blank lines
    return "\n".join(x)
667032
def _clean_code(code):
    
    lines = []
    # cleanup lines, the parser is very sensitive to extra spaces,...
    for l in code.splitlines():
        # remove last .
        if l.endswith('.'):
            l = l[:-1]
        # the parser doe not like VALUE xxx.
        if "VALUE" in l:
            l = l[:l.find("VALUE")]
        # the parser does not like extra spaces between "PIC X(xxx)" and "."
        indent = len(l) - len(l.lstrip())
        tokens = l.split(" ")
        while "" in tokens:
            tokens.remove("")
        if tokens and not tokens[-1].endswith("."):
            tokens[-1] += "."
        lines.append(" " * indent + " ".join(tokens))

    return lines
840461
def normalize(string: str) -> str:
    
    string = string.replace("\t", "    ")
    lines = string.split("\n")
    while lines and (not lines[0] or lines[0].isspace()):
        lines.pop(0)
    while lines and (not lines[-1] or lines[-1].isspace()):
        lines.pop()
    for i, line in enumerate(lines):
        lines[i] = line.rstrip()
    string = "\n".join(lines)
    string = textwrap.dedent(string)
    return string
570982
def _clean_text(self, text):
        
        clean = []
        for char in text:
            if char in self.punc_stops:
                clean += '.'
            elif char not in self.punc:
                clean += char
            else:
                pass
        return (''.join(clean)).lower()
684343
def clean_line_profile_text(text):
    
    #
    profile_block_list = parse_rawprofile_blocks(text)
    #profile_block_list = fix_rawprofile_blocks(profile_block_list)
    #---
    # FIXME can be written much nicer
    prefix_list, timemap = parse_timemap_from_blocks(profile_block_list)
    # Sort the blocks by time
    sorted_lists = sorted(six.iteritems(timemap), key=operator.itemgetter(0))
    newlist = prefix_list[:]
    for key, val in sorted_lists:
        newlist.extend(val)
    # Rejoin output text
    output_text = '\n'.join(newlist)
    #---
    # Hack in a profile summary
    summary_text = get_summary(profile_block_list)
    output_text = output_text
    return output_text, summary_text
1161530
def clean_list(l0, clean_item_fn=None):
    
    clean_item_fn = clean_item_fn if clean_item_fn else clean_item
    l = list()
    for index, item in enumerate(l0):
        cleaned_item = clean_item_fn(item)
        l.append(cleaned_item)
    return l
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist