Return a list of ``(text, replaced_count, mask)``
        tuples with n_samples versions of text with some words replaced.
        By default words are replaced with '', i.e. removed.
766624
def replace_random_tokens(self,
                              n_samples,  # type: int
                              replacement='',  # type: str
                              random_state=None,
                              min_replace=1,  # type: Union[int, float]
                              max_replace=1.0,  # type: Union[int, float]
                              group_size=1  # type: int
                              ):
        # type: (...) -> List[Tuple[str, int, np.ndarray]]
        
        n_tokens = len(self.tokens)
        indices = np.arange(n_tokens)
        if not n_tokens:
            nomask = np.array([], dtype=int)
            return [('', 0, nomask)] * n_samples

        min_replace, max_replace = self._get_min_max(min_replace, max_replace,
                                                     n_tokens)
        rng = check_random_state(random_state)
        replace_sizes = rng.randint(low=min_replace, high=max_replace + 1,
                                    size=n_samples)
        res = []
        for num_to_replace in replace_sizes:
            idx_to_replace = rng.choice(indices, num_to_replace, replace=False)
            idx_to_replace = np.array([idx_to_replace] + [
                idx_to_replace + shift for shift in range(1, group_size)
            ]).ravel()
            padded_size = n_tokens + group_size - 1
            mask = indices_to_bool_mask(idx_to_replace, padded_size)[:n_tokens]
            s = self.split.masked(mask, replacement)
            res.append((s.text, num_to_replace, mask))
        return res
736705
def replace(self, re_text, replace_str, text):
        
        return re.sub(re_text, replace_str, text)
518575
def remove_each(text, items, count=None, strip=False):
    '''
    
    '''
    for item in items:
        text = remove(text, item, count=count, strip=strip)
    return text
528258
def replace_variables(sentence: List[str],
                      sentence_variables: Dict[str, str]) -> Tuple[List[str], List[str]]:
    
    tokens = []
    tags = []
    for token in sentence:
        if token not in sentence_variables:
            tokens.append(token)
            tags.append("O")
        else:
            for word in sentence_variables[token].split():
                tokens.append(word)
                tags.append(token)
    return tokens, tags
422775
def strip(self, text):
        ''''''
        tags, results = [], []
        return self.re_tag.sub(lambda m: self.clear_tag(m, tags, results), text)
478313
def sanitize_words(self, words):
    
    _words = []
    for w in words:
      if isinstance(w, string_types) and not isinstance(w, unicode):
        _words.append(unicode(w, encoding="utf-8"))
      else:
        _words.append(w)
    return _words
396771
def normalize(text: str) -> str:
    
    for data in _NORMALIZE_RULE2:
        text = re.sub(data[0].replace("t", "[่้๊๋]"), data[1], text)
    for data in list(zip(_NORMALIZE_RULE1, _NORMALIZE_RULE1)):
        text = re.sub(data[0].replace("t", "[่้๊๋]") + "+", data[1], text)
    return text
686550
def _replace(self, data, replacements):
        
        for find, repl in replacements:
            data = data.replace(find, repl)
        return data
556586
def license_cleanup(text):
    
    if not text:
        return None
    text = text.rsplit(':', 1)[-1]
    replacements = [
        'licenses',
        'license',
        'licences',
        'licence',
        'software',
        ',',
    ]
    for replacement in replacements:
        text = text.replace(replacement, '')
    text = text.strip().upper()
    text = text.replace(' ', '_')
    text = text.replace('-', '_')
    if any(trigger.upper() in text for trigger in BAD_LICENSES):
        return None
    return text
1359115
def remove_blanks(letters: List[str]):
    
    cleaned = []
    for letter in letters:
        if letter != "":
            cleaned.append(letter)
    return cleaned
1346403
def replace_suffixes_4(self, word):
        
        length = len(word)
        replacements = {'ational': 'ate', 'tional': 'tion', 'alize': 'al',
                        'icate': 'ic', 'iciti': 'ic', 'ical': 'ic',
                        'ful': '', 'ness': ''}

        for suffix in replacements.keys():
            if word.endswith(suffix):
                suffix_length = len(suffix)
                if self.r1 <= (length - suffix_length):
                    word = word[:-suffix_length] + replacements[suffix]

        if word.endswith('ative'):
            if self.r1 <= (length - 5) and self.r2 <= (length - 5):
                word = word[:-5]

        return word
163964
def clean_text(value, topic=False):
    
    for x in PROFANITY_REPLACEMENTS:
        value = value.replace(x[0], x[1])

    for bbset in BBCODE_REPLACEMENTS:
        p = re.compile(bbset[0], re.DOTALL)
        value = p.sub(bbset[1], value)

    bleached = bleach.clean(value, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, strip=True)

    # We want to retain markdown quotes and we'll be running bleach again in format_post.
    bleached = bleached.replace('&gt;', '>').replace('&amp;', '&')
    return bleached
907314
def remove_duplicates(utterances: List[Utterance]) -> List[Utterance]:
    

    filtered_utters = []
    utter_set = set() # type: Set[Tuple[int, int, str]]
    for utter in utterances:
        if (utter.start_time, utter.end_time, utter.text) in utter_set:
            continue
        filtered_utters.append(utter)
        utter_set.add((utter.start_time, utter.end_time, utter.text))

    return filtered_utters
260551
def global_matches(self, text):
        
        matches = []
        n = len(text)
        for word in self.namespace:
            if word[:n] == text and word != "__builtins__":
                matches.append(word)
        return matches
483314
def _edits2(word: str) -> Set[str]:
    
    return set(e2 for e1 in _edits1(word) for e2 in _edits1(e1))
67045
def tmpl_replchars(text, replace, chars):
        
        for char in chars:
            text = text.replace(char, replace)
        return text
299129
def word_matches(s1, s2, n=3):
    
    return __matches(s1, s2, word_ngrams, n=n)