apply some simple regexps to clean up surnames
165943
def cleanup_surname(name)
      if name.length > 4
        name.gsub!(/^Mc(\w+)/) { |s| "Mc#{$1.capitalize}" }
        name.gsub!(/^Mac(\w+)/) { |s| "Mac#{$1.capitalize}" }
        name.gsub!(/^Mac(\w+)/) { |s| "Mac#{$1.capitalize}" }
        name.gsub!(/^Osh(\w+)/) { |s| "O'sh#{$1}" }
        name.gsub!(/^Van(\w+)/) { |s| "Van#{$1.capitalize}" }
        name.gsub!(/^Von(\w+)/) { |s| "Von#{$1.capitalize}" }        
#        name.gsub!(/^Dev(\w+)/) { |s| "DeV#{$1}" }        
      end
      name
    end
474260
def normalize(name):
    """"""

    # Name should not contain some specials chars (issue #1068)
    ret = name.replace(':', '')
    ret = ret.replace('%', '')
    ret = ret.replace(' ', '_')

    return ret
622600
def CleanString(s):
    
    punc = (' ', '-', '\'', '.', '&amp;', '&', '+', '@')
    pieces = []
    for part in s.split():
        part = part.strip()
        for p in punc:
            part = part.replace(p, '_')
        part = part.strip('_')
        part = part.lower()
        pieces.append(part)
    return '_'.join(pieces)
992031
def clean(ctx, text):
    
    text = conversions.to_string(text, ctx)
    return ''.join([c for c in text if ord(c) >= 32])
171962
def __StripName(self, name):
        """"""
        if not name:
            return name
        for prefix in self.__strip_prefixes:
            if name.startswith(prefix):
                return name[len(prefix):]
        return name
163004
function toRegExp(pattern) {
  return new RegExp('^' + escape(pattern) + '$', 'i');

  function escape(x) {
    return x.replace(/(\W)/g, function(_, match) {
      return match === '*'?   '.*'
      :      /* otherwise */  '\\' + match;
    });
  }
}
1190772
def _remove_prefix(name):
    """"""
    if isinstance(name, str):
        return _do_remove_prefix(name)
    return [_do_remove_prefix(nm) for nm in name]
1363756
def _filter_name(name):
    """"""
    # Remove if length 3 or less
    if len(name) <= 3:
        return False
    # Remove if starts with IL-
    if name.startswith('IL-'):
        return False
    lowname = name.lower()
    # Remove if contains certain sequences
    if any(c in lowname for c in STOP_SUB):
        return False
    # Remove if (case-insensitive) exact match to stoplist
    if lowname in STOPLIST:
        return False
    comps = re.split('[ -]', lowname)
    # Remove if just single character + digits separated by spaces or hyphens (or the word compound)
    if all(c.isdigit() or len(c) == 1 or c == 'compound' for c in comps):
        return False
    # Remove if 3 or fewer letters with 2 or fewer digits
    if len(comps) == 2 and len(comps[0]) <= 3 and comps[0].isalpha() and len(comps[1]) <= 3 and comps[1].isdigit():
        return False
    # Remove if just greek characters and numbrs
    if re.match('^[Α-Ωα-ω0-9]+$', name):
        return False
    # Filter registry numbers? No real size benefit in DAWG.
    # if REG_RE.search(name):
    #     keep = False
    # Handle this at the token level
    # if name.endswith(' derivative') or name.endswith(' analog') or name.endswith(' solution'):
    #     keep = False
    # Filter this after matching and expanding boundaries
    # if name.startswith('-') or name.endswith('-'):
    #     keep = False
    # Filter this after matching and expanding boundaries
    # if not bracket_level(name) == 0:
    #     print(name)
    return True
598241
function (filename, types) {
    types = types.split(',');
    var pattern = '\.(';
    for (var i = 0; i < types.length; i++) {
        if (0 != i) {
            pattern += '|';
        }
        pattern += types[i].trim();
    }
    pattern += ')$';
    return new RegExp(pattern, 'i').test(filename);
}
44545
function (str) {
            return _(str).chain()
                .unescapeHTML()
                .stripTags()
                .clean()
                .value()
                .replace( matchJunk, '' )
                .toLowerCase();
        }
237370
def clean(self, text):
        """"""
        return ''.join([c for c in text if c in self.alphabet])
1363516
def clean(text):
    



    text = bold_italic.sub(r'\1', text)
    text = bold.sub(r'\1', text)
    text = italic_quote.sub(r'"\1"', text)
    text = italic.sub(r'"\1"', text)
    text = quote_quote.sub(r'"\1"', text)
    # residuals of unbalanced quotes
    text = text.replace("'''", '').replace("''", '"')
    text = newlines.sub(r'\n', text)
    text = bulletlist.sub(r'', text)

    # Collect spans

    spans = []
    # Drop HTML comments
    for m in comment.finditer(text):
            spans.append((m.start(), m.end()))

    # Drop self-closing tags
    for pattern in selfClosing_tag_patterns:
        for m in pattern.finditer(text):
            spans.append((m.start(), m.end()))

    # Drop ignored tags
    for left, right in ignored_tag_patterns:
        for m in left.finditer(text):
            spans.append((m.start(), m.end()))
        for m in right.finditer(text):
            spans.append((m.start(), m.end()))

    # Bulk remove all spans
    text = dropSpans(spans, text)

    # Drop discarded elements
    for tag in discardElements:
        text = dropNested(text, r'<\s*%s\b[^>/]*>' % tag, r'<\s*/\s*%s>' % tag)

    # Expand placeholders
    for pattern, placeholder in placeholder_tag_patterns:
        index = 1
        for match in pattern.finditer(text):
            text = text.replace(match.group(), '%s_%d' % (placeholder, index))
            index += 1

    text = text.replace('<<', u'Â«').replace('>>', u'Â»')

    #############################################

    # Cleanup text
    text = text.replace('\t', ' ')
    text = spaces.sub(' ', text)
    text = dots.sub('...', text)
    text = re.sub(u' (,:\.\)\]Â»)', r'\1', text)
    text = re.sub(u'(\[\(Â«) ', r'\1', text)
    text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations
    text = text.replace(',,', ',').replace(',.', '.')

    return text
1305314
def _clean_text(self, text):
        """"""
        output = []
        for char in text:
            cp = ord(char)
            if cp in (0, 0xfffd) or self._is_control(char):
                continue
            if self._is_whitespace(char):
                output.append(' ')
            else:
                output.append(char)
        return ''.join(output)
601686
def _name_matches(name, matches):
    '''
    
    '''
    for m in matches:
        if name.endswith(m):
            return True
        if name.lower().endswith('_' + m.lower()):
            return True
        if name.lower() == m.lower():
            return True
    return False
369190
def clean_whitespace(string, compact=False):
    """"""
    for a, b in (('\r\n', '\n'), ('\r', '\n'), ('\n\n', '\n'),
                 ('\t', ' '), ('  ', ' ')):
        string = string.replace(a, b)
    if compact:
        for a, b in (('\n', ' '), ('[ ', '['),
                     ('  ', ' '), ('  ', ' '), ('  ', ' ')):
            string = string.replace(a, b)
    return string.strip()
686937
def _clean_str(self, s):
        
        return s.translate(str.maketrans('', '', punctuation)).replace('\u200b', " ").strip().lower()
328666
def _strip(string, pattern):
    """"""
    m = re.compile(pattern).search(string)

    if m:
        return string[0:m.start()] + string[m.end():len(string)]
    else:
        return string