Clean text for TFIDF.
355856
def clean_text(text):
    """"""
    new_text = re.sub(ur'\p{P}+', ' ', text)

    new_text = [stem(i) for i in new_text.lower().split() if not
                re.findall(r'[0-9]', i)]

    new_text = ' '.join(new_text)

    return new_text
64330
def clean_text(self, text, **kwargs):
        """"""
        text = text.replace(" ", "")
        text = text.upper()
        return text
1160843
def _filter(self, text):
        """"""

        if self.line_endings:
            text = self.norm_nl(text)

        new_text = []
        index = 0
        last = 0
        end = len(text)
        while index < end:
            m = self.escapes.match(text, pos=index) if self.escapes else None
            if m:
                index = m.end(0)
                continue
            handled = False
            for delimiter in self.delimiters:
                m = delimiter[0].match(text, pos=index)
                if m:
                    if self.context_visible_first is True:
                        new_text.append(text[last:m.start(0)])
                    else:
                        new_text.append(m.group(delimiter[1]))
                    index = m.end(0)
                    last = index
                    handled = True
                    break
            if handled:
                continue
            index += 1
        if last < end and self.context_visible_first is True:
            new_text.append(text[last:end])

        return ' '.join(new_text)
449858
def clean_description desc
      strip_stopwords(
        desc.gsub(XML_TAG_FILTER,' ')
            .gsub(EMAIL_FILTER,'')
            .gsub(URL_FILTER,'')
            .gsub(GENDER_FILTER,'')
            .gsub(NEW_LINES,'')
            .gsub(SYMBOL_FILTER,' ')
            .gsub(WHITESPACE,' ')
            .gsub(WORDS_IN_BRACKETS, '\1')
            .gsub(CODE_TOKEN_FILTER,'')
            .downcase
            .strip
        )
    end
252151
def _clean(self, t, capitalize=None):
        """"""
        if self._from_bibtex:
            t = latex_to_unicode(t, capitalize=capitalize)
        t = ' '.join([el.rstrip('.') if el.count('.') == 1 else el for el in t.split()])
        return t
1032255
def clean_text(self, text, preserve_space):
        
        if text is None:
            return
        if preserve_space:
            text = text.replace('\r\n', ' ').replace('\n', ' ').replace('\t', ' ')
        else:
            text = text.replace('\r\n', '').replace('\n', '').replace('\t', ' ')
            text = text.strip()
            while ('  ' in text):
                text = text.replace('  ', ' ')
        return text
145328
def clean_gff(gff, cleaned, add_chr=False, chroms_to_ignore=None,
              featuretypes_to_ignore=None):
    
    logger.info("Cleaning GFF")
    chroms_to_ignore = chroms_to_ignore or []
    featuretypes_to_ignore = featuretypes_to_ignore or []
    with open(cleaned, 'w') as fout:
        for i in gffutils.iterators.DataIterator(gff):
            if add_chr:
                i.chrom = "chr" + i.chrom

            if i.chrom in chroms_to_ignore:
                continue

            if i.featuretype in featuretypes_to_ignore:
                continue
            fout.write(str(i) + '\n')
    return cleaned
552948
def normalize(self, text, cleaned=False, **kwargs):
        """"""
        if not cleaned:
            text = self.clean(text, **kwargs)
        return ensure_list(text)
345829
function(text) {
        return macros.cleanMacros(text.replace(/\n/g, '{break}').replace(/<i>/g, '{italic}').replace(/<\/i>/g, '{end-italic}'));
    }
978118
def cleanString(someText):
    
    ret = ''
    if someText is not None:
        ret = filter(unicode.isalnum, someText.lower())
    return ret
964970
function filterText(text) {
        var newText = text;
        if (text.length > preferences.getMaxFileSize()) {
            newText = "";
        }
        return newText;
    }
1087202
def clean(self):
        """"""
        return Text(self.__text_cleaner.clean(self[TEXT]), **self.__kwargs)
57692
def _normalize(sentences):
    ''''''

    tfidf = TfidfVectorizer(tokenizer=_tokenize,
                            stop_words='english',
                            decode_error='ignore',
                            ngram_range=(1,2))
    return tfidf.fit_transform(sentences)
168907
def text(self, clean=True):
        
        if clean:
            return utils.normalise_whitespace(''.join(self).replace('\n', ' '))
        else:
            return ''.join(self)
1282685
private void cleanContentLeadingSpaces()
    {
        if (getContent().length() > 0) {
            if (fPreviousInlineText.length() == 0
                || fPreviousInlineText
                .charAt(fPreviousInlineText.length() - 1) == ' ')
            {
                trimLeadingWhiteSpaces();
            }
        }
    }
763331
public static String clean(String text) {
        if (text == null)
            return "";

        text = text.trim();
        text = text.replace("\"", "\\\"");
        text = text.replace("\'", "\\\'");
        text = text.replace("\n", "");
        text = text.replace("\r", "");
        return text;
    }
620646
def trim(self, text):
        
        lines = text.splitlines()
        firstline = lines[0] or lines[1]
        indent = len(firstline) - len(firstline.lstrip())
        return '\n'.join(x[indent:] for x in lines if x.strip())