Word, Synonym, Regexp / ExtractionRule
354700
function augmentContext1(context, oRules, options) {
    var sKey = oRules[0].key;
    // check that rule
    if (debuglog.enabled) {
        // check consistency
        oRules.every(function (iRule) {
            if (iRule.key !== sKey) {
                throw new Error("Inhomogenous keys in rules, expected " + sKey + " was " + JSON.stringify(iRule));
            }
            return true;
        });
    }
    // look for rules which match
    var res = oRules.map(function (oRule) {
        // is this rule applicable
        switch (oRule.type) {
            case mgnlq_model_1.IFModel.EnumRuleType.WORD:
                return matchWord(oRule, context, options);
            case mgnlq_model_1.IFModel.EnumRuleType.REGEXP:
                return matchRegExp(oRule, context, options);
        }
        return undefined;
    }).filter(function (ores) {
        return !!ores;
    }).sort(sortByWeight.bind(this, sKey));
    //debuglog("hassorted" + JSON.stringify(res,undefined,2));
    return res;
    // Object.keys().forEach(function (sKey) {
    // });
}
701036
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._tokenizer.tokenize_to_spacy_doc(text)
        self._load_matcher()

        matches = [x for x in self._matcher(doc) if x[1] != x[2]]
        pos_filtered_matches = []
        neg_filtered_matches = []
        for idx, start, end in matches:
            span_doc = self._tokenizer.tokenize_to_spacy_doc(doc[start:end].text)
            this_spacy_rule = self._matcher.get(idx)
            relations = self._find_relation(span_doc, this_spacy_rule)
            rule_id, _ = self._hash_map[idx]
            this_rule = self._rule_lst[rule_id]
            if self._filter_match(doc[start:end], relations, this_rule.patterns):
                value = self._form_output(doc[start:end], this_rule.output_format, relations, this_rule.patterns)
                if this_rule.polarity:
                    pos_filtered_matches.append((start, end, value, rule_id, relations))
                else:
                    neg_filtered_matches.append((start, end, value, rule_id, relations))

        return_lst = []
        if pos_filtered_matches:
            longest_lst_pos = self._get_longest(pos_filtered_matches)
            if neg_filtered_matches:
                longest_lst_neg = self._get_longest(neg_filtered_matches)
                return_lst = self._reject_neg(longest_lst_pos, longest_lst_neg)
            else:
                return_lst = longest_lst_pos

        extractions = []
        for (start, end, value, rule_id, relation) in return_lst:
            this_extraction = Extraction(value=value,
                                         extractor_name=self.name,
                                         start_token=start,
                                         end_token=end,
                                         start_char=doc[start].idx,
                                         end_char=doc[end-1].idx+len(doc[end-1]),
                                         rule_id=rule_id.split("rule_id##")[0],
                                         match_mapping=relation)
            extractions.append(this_extraction)

        return extractions
489357
def gfm(text):
    """"""
    extractions = {}

    def extract_pre_block(matchobj):
        match = matchobj.group(0)
        hashed_match = hashlib.md5(match.encode('utf-8')).hexdigest()
        extractions[hashed_match] = match
        result = "{gfm-extraction-%s}" % hashed_match
        return result

    def escape_underscore(matchobj):
        match = matchobj.group(0)

        if match.count('_') > 1:
            return re.sub('_', '\_', match)
        else:
            return match

    def newlines_to_brs(matchobj):
        match = matchobj.group(0)
        if re.search("\n{2}", match):
            return match
        else:
            match = match.strip()
            return match + "  \n"

    def insert_pre_block(matchobj):
        string = "\n\n" + extractions[matchobj.group(1)]
        return string

    text = re.sub("(?s)<pre>.*?<\/pre>", extract_pre_block, text)
    text = re.sub("(^(?! {4}|\t)\w+_\w+_\w[\w_]*)", escape_underscore, text)
    text = re.sub("(?m)^[\w\<][^\n]*\n+", newlines_to_brs, text)
    text = re.sub("\{gfm-extraction-([0-9a-f]{32})\}", insert_pre_block, text)

    return text
787903
def _matchremove_verb_endings(self, word):
        """"""

        i_verb_endings = ['iuntur',
                            'erunt',
                            'untur',
                            'iunt',
                            'unt']

        bi_verb_endings = ['beris',
                            'bor',
                            'bo']

        eri_verb_endings = ['ero']

        verb_endings = ['mini',
                            'ntur',
                            'stis',
                            'mur',
                            'mus',
                            'ris',
                            'sti',
                            'tis',
                            'tur',
                            'ns',
                            'nt',
                            'ri',
                            'm',
                            'r',
                            's',
                            't']

        # replace i verb endings with i
        for ending in i_verb_endings:
            if word.endswith(ending):
                word = re.sub(r'{0}$'.format(ending), 'i', word)
                return word

        # replace bi verb endings with bi
        for ending in bi_verb_endings:
            if word.endswith(ending):
                word = re.sub(r'{0}$'.format(ending), 'bi', word)
                return word

        # replace eri verb endings with eri
        for ending in eri_verb_endings:
            if word.endswith(ending):
                word = re.sub(r'{0}$'.format(ending), 'eri', word)
                return word

        # otherwise, remove general verb endings
        for ending in verb_endings:
            if word.endswith(ending):
                word = re.sub(r'{0}$'.format(ending), '', word)
                break

        return word
1374103
function match(word, array, caseSensitive) {
		return $.grep(
			array,
			function(w) {
				if (caseSensitive) {
					return !w.indexOf(word);
				} else {
					return !w.toLowerCase().indexOf(word.toLowerCase());
				}
			}
		);
	}
1033156
function lex(phrase) {
        var words = phrase && typeof(phrase) === 'string' ? lexer.lex(phrase) : {status:"Invalid phrase parameter"};
        logger.log('verbose', '%s|lex|phrase=%s|words=%j', meta.module, phrase, words, meta);
        return words;
    }
1248221
function(str, lex) {
    if (lex) {
      w2.plugin({
        words: lex
      });
    }
    let doc = buildText(str, w2);
    doc.tagger();
    return doc;
  }
1012512
function _regExpOrGTFO(expr){
  var exp;

  if(typeof expr === 'string'){
    exp = new RegExp('^'+expr+'$');
  } else if (expr instanceof RegExp){
    exp = expr;
  } else {
    throw new Error('Needle must be either a string or a RegExp');
  }

  return exp;
}
604151
def lex(self, text):
        """"""
        for match in self.regex.finditer(text):
            name = match.lastgroup
            yield (name, match.group(name))
691427
function lex(source) {
    var tokens = [];
    var chunk = clean(source);
    var total = chunk.length;
    var iterations = 0;

    while (chunk.length > 0) {
        for (var regexpName in REGEXPS) {
            var regexp = REGEXPS[regexpName];
            var match = regexp.exec(chunk);

            if (match) {
                var original = match[0];
                var str = original;
                if (regexpName === 'string') str = str.replace(/\\"/g, '"');
                tokens.push({ type: regexpName, string: str, original: original });
                // Need to slice the chunk at the original match length
                chunk = chunk.slice(match[0].length, chunk.length);
                break;
            }
        }

        // We've probably failed to parse correctly if we get here
        if (iterations++ > total) {
            var parts = [
                'Runiq: Lexer ran too many iterations!',
                '--- Failed at chunk: `' + chunk + '`'
            ];
            throw new Error(parts.join('\n'));
        }
    }

    return tokens;
}
1299267
def main(*args):
    """"""
    options, args, parser = parse_options(args)

    if options.regex and options.write_changes:
        print('ERROR: --write-changes cannot be used together with '
              '--regex')
        parser.print_help()
        return 1
    word_regex = options.regex or word_regex_def
    try:
        word_regex = re.compile(word_regex)
    except re.error as err:
        print('ERROR: invalid regular expression "%s" (%s)' %
              (word_regex, err), file=sys.stderr)
        parser.print_help()
        return 1

    ignore_words_files = options.ignore_words or []
    ignore_words = set()
    for ignore_words_file in ignore_words_files:
        if not os.path.exists(ignore_words_file):
            print('ERROR: cannot find ignore-words file: %s' %
                  ignore_words_file, file=sys.stderr)
            parser.print_help()
            return 1
        build_ignore_words(ignore_words_file, ignore_words)

    ignore_words_list = options.ignore_words_list or []
    for comma_separated_words in ignore_words_list:
        for word in comma_separated_words.split(','):
            ignore_words.add(word.strip())

    dictionaries = options.dictionary or [default_dictionary]
    misspellings = dict()
    for dictionary in dictionaries:
        if dictionary == "-":
            dictionary = default_dictionary
        if not os.path.exists(dictionary):
            print('ERROR: cannot find dictionary file: %s' % dictionary,
                  file=sys.stderr)
            parser.print_help()
            return 1
        build_dict(dictionary, misspellings, ignore_words)
    colors = TermColors()
    if not options.colors or sys.platform == 'win32':
        colors.disable()

    if options.summary:
        summary = Summary()
    else:
        summary = None

    context = None
    if options.context is not None:
        if (options.before_context is not None) or \
                (options.after_context is not None):
            print('ERROR: --context/-C cannot be used together with '
                  '--context-before/-B or --context-after/-A')
            parser.print_help()
            return 1
        context_both = max(0, options.context)
        context = (context_both, context_both)
    elif (options.before_context is not None) or \
            (options.after_context is not None):
        context_before = 0
        context_after = 0
        if options.before_context is not None:
            context_before = max(0, options.before_context)
        if options.after_context is not None:
            context_after = max(0, options.after_context)
        context = (context_before, context_after)

    exclude_lines = set()
    if options.exclude_file:
        build_exclude_hashes(options.exclude_file, exclude_lines)

    file_opener = FileOpener(options.hard_encoding_detection,
                             options.quiet_level)
    glob_match = GlobMatch(options.skip)

    bad_count = 0
    for filename in args:
        # ignore hidden files
        if is_hidden(filename, options.check_hidden):
            continue

        if os.path.isdir(filename):
            for root, dirs, files in os.walk(filename):
                if glob_match.match(root):  # skip (absolute) directories
                    del dirs[:]
                    continue
                for file_ in files:
                    if glob_match.match(file_):  # skip files
                        continue
                    fname = os.path.join(root, file_)
                    if glob_match.match(fname):  # skip paths
                        continue
                    if not os.path.isfile(fname) or not os.path.getsize(fname):
                        continue
                    bad_count += parse_file(
                        fname, colors, summary, misspellings, exclude_lines,
                        file_opener, word_regex, context, options)

                # skip (relative) directories
                dirs[:] = [dir_ for dir_ in dirs if not glob_match.match(dir_)]

        else:
            bad_count += parse_file(
                filename, colors, summary, misspellings, exclude_lines,
                file_opener, word_regex, context, options)

    if summary:
        print("\n-------8<-------\nSUMMARY:")
        print(summary)
    return bad_count
245542
def _matchremove_simple_endings(self, word):
        """"""

        was_stemmed = False

        # noun, adjective, and adverb word endings sorted by charlen, then alph
        simple_endings = ['ibus',
                            'ius',
                            'ae',
                            'am',
                            'as',
                            'em',
                            'es',
                            'ia',
                            'is',
                            'nt',
                            'os',
                            'ud',
                            'um',
                            'us',
                            'a',
                            'e',
                            'i',
                            'o',
                            'u']

        for ending in simple_endings:
            if word.endswith(ending):
                word = re.sub(r'{0}$'.format(ending), '', word)
                was_stemmed = True
                break

        return word, was_stemmed
468800
def Search(self, text):
    """"""
    if isinstance(text, rdfvalue.RDFString):
      text = str(text)

    return self._regex.search(text)
621484
public static List<WordInfo> extractWords(String text, int size)
    {
        return extractWords(text, size, false);
    }
1175289
def _reExec(regex, string):
    ''''''
    m = regex.search(string)
    if not m: return None
    return [m.group()] + list(m.groups())
611149
function __match(regexp) {
  return function(ch) { 
  	return !!(ch !== DONE && (ch === undefined || ch.match(regexp)))? MATCH_TRUE: MATCH_FALSE };
}
1257141
def term_regex(term):
    
    return re.compile(r'^{0}$'.format(re.escape(term)), re.IGNORECASE)