Searches texts for presence of n-grams.
773224
def search_texts(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    store.validate(corpus, catalogue)
    ngrams = []
    for ngram_file in args.ngrams:
        ngrams.extend(utils.get_ngrams(ngram_file))
    store.search(catalogue, ngrams, sys.stdout)
707779
def check_text_similarity(a_dom, b_dom, cutoff):
    """"""
    a_words = list(tree_words(a_dom))
    b_words = list(tree_words(b_dom))

    sm = WordMatcher(a=a_words, b=b_words)
    if sm.text_ratio() >= cutoff:
        return True
    return False
670768
public static ArrayList<String> ngram(String[] strs, int[] grams) {
        ArrayList<String> list = new ArrayList<String>();
        StringBuffer buf = new StringBuffer();
        for (int j = 0; j < grams.length; j++) {
            int len = grams[j];
            if (len <= 0 || len > strs.length)
                continue;
            for (int i = 0; i < strs.length - len+1; i++) {
                buf.delete(0, buf.length());
                int k = 0;
                for(; k < len-1; ++k)   {
                    buf.append(strs[i+k]);
                    buf.append(' ');
                }
                buf.append(strs[i+k]);
                list.add(buf.toString().intern());
            }
        }
        return list;
    }
765343
def pre_check(self, data):
        """"""
        sentences = len(re.findall('[\.!?]+\W+', data)) or 1
        chars = len(data) - len(re.findall('[^a-zA-Z0-9]', data))
        num_words = len(re.findall('\s+', data))
        data = re.split('[^a-zA-Z]+', data)
        return data, sentences, chars, num_words
769124
def _get_ngrams(n, text):
  
  ngram_set = set()
  text_length = len(text)
  max_index_ngram_start = text_length - n
  for i in range(max_index_ngram_start + 1):
    ngram_set.add(tuple(text[i:i + n]))
  return ngram_set
161876
def find_type(self, txt):
        
        searchString = txt.upper()
        match = 'Unknown'
        for i in self.lst_type:
            if searchString in i:
                match = i
        return match
1253418
function( text2search, noSearchPhrase, noSearchResult ) {

  Array.call( this );

  /**
   * Gets the text to search.
   * @type {string|null}
   */
  this.text2search = text2search || null;

  /**
   * Gets a message to display when search phrase is missing.
   * @type {string}
   */
  this.noSearchPhrase = noSearchPhrase;

  /**
   * Gets a message to display when search did not found the text in the contents.
   * @type {string}
   */
  this.noSearchResult = noSearchResult;
}
567246
def prune_by_ngram(self, ngrams):
        
        self._logger.info('Pruning results by n-gram')
        self._matches = self._matches[
            ~self._matches[constants.NGRAM_FIELDNAME].isin(ngrams)]
1293883
def validateData(self, text):
        '''
        
        '''
        if text == "\n":
            return False
        for c in text:
            try:
                if str(c) != " ":
                    return True
            except:
                return False
        return False
105130
def search(self, text, lookup=None):
        ''''''
        return self.query().search(text, lookup=lookup)
551802
def check(text):
    
    err = "terms.denizen_labels.garner"
    msg = "'{}' is the preferred denizen label."

    preferences = [

        ["Afrikaner",         ["Afrikaaner"]],
        ["Afrikaner",         ["Afrikander"]],
        ["Alabamian",         ["Alabaman"]],
        ["Albuquerquean",     ["Albuquerquian"]],
        ["Anchorageite",      ["Anchoragite"]],
        ["Angeleno",          ["Los Angelean"]],
        ["Arizonan",          ["Arizonian"]],
        ["Arkansan",          ["Arkansawyer"]],
        ["Belarusian",        ["Belarusan"]],
        ["Caymanian",         ["Cayman Islander"]],
        ["Coloradan",         ["Coloradoan"]],
        ["Fairbanksan",       ["Fairbanksian"]],
        ["Fort Worthian",     ["Fort Worther"]],
        ["Grenadan",          ["Grenadian"]],
        ["Hong Konger",       ["Hong Kongite", "Hong Kongian"]],
        ["Hoosier",           ["Indianan", "Indianian"]],
        ["Illinoisan",        ["Illinoisian"]],
        ["Iowan",             ["Iowegian"]],
        ["Louisianian",       ["Louisianan"]],
        ["Michigander",       ["Michiganite", "Michiganian"]],
        ["Missourian",        ["Missouran"]],
        ["Monegasque",        ["Monacan"]],
        ["Neapolitan",        ["Neopolitan"]],
        ["New Hampshirite",   ["New Hampshireite", "New Hampshireman"]],
        ["New Jerseyan",      ["New Jerseyite"]],
        ["New Orleanian",     ["New Orleansian"]],
        ["Nutmegger",         ["Connecticuter"]],
        ["Oklahoma Cityan",   ["Oklahoma Citian"]],
        ["Oklahoman",         ["Oklahomian"]],
        ["Seattleite",        ["Seattlite"]],
        ["Surinamese",        ["Surinamer"]],
        ["Tallahasseean",     ["Tallahassean"]],
        ["Tennessean",        ["Tennesseean"]],
        ["Tusconan",          ["Tusconian", "Tusconite"]],
        ["Utahn",             ["Utahan"]],
        ["Saudi",             ["Saudi Arabian"]],
    ]

    return preferred_forms_check(text, preferences, err, msg)
134299
def analyse_text(text):
        
        if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE):
            # The code starts with REBOL header
            return 1.0
        elif re.search(r'\s*REBOL\s*[', text, re.IGNORECASE):
            # The code contains REBOL header but also some text before it
            return 0.5
49927
private double[] detectBlockLongText(List<String> ngrams) {
        assert !ngrams.isEmpty();
        double[] langprob = new double[ngramFrequencyData.getLanguageList().size()];
        Random rand = new Random(seed.or(DEFAULT_SEED));
        for (int t = 0; t < N_TRIAL; ++t) {
            double[] prob = initProbability();
            double alpha = this.alpha + (rand.nextGaussian() * ALPHA_WIDTH);

            for (int i=0; i<ITERATION_LIMIT; i++) {
                int r = rand.nextInt(ngrams.size());
                updateLangProb(prob, ngrams.get(r), 1, alpha);
                if (i % 5 == 0) {
                    if (Util.normalizeProb(prob) > CONV_THRESHOLD) break; //this break ensures that we quit the loop before all probabilities reach 0
                    if (logger.isTraceEnabled()) logger.trace("> " + sortProbability(prob));
                }
            }
            for(int j=0;j<langprob.length;++j) langprob[j] += prob[j] / N_TRIAL;
            if (logger.isDebugEnabled()) logger.debug("==> " + sortProbability(prob));
        }
        return langprob;
    }
1294475
public List<UserDictionaryMatch> findUserDictionaryMatches(String text) {
        List<UserDictionaryMatch> matchInfos = new ArrayList<>();
        int startIndex = 0;

        while (startIndex < text.length()) {
            int matchLength = 0;

            while (startIndex + matchLength < text.length()
                            && entries.containsKeyPrefix(text.substring(startIndex, startIndex + matchLength + 1))) {
                matchLength++;
            }

            if (matchLength > 0) {
                String match = text.substring(startIndex, startIndex + matchLength);
                int[] details = entries.get(match);

                if (details != null) {
                    matchInfos.addAll(makeMatchDetails(startIndex, details));
                }
            }

            startIndex++;
        }

        return matchInfos;
    }
495548
def search(self, sentence):
        
        if sentence.__class__.__name__ == "Sentence":
            pass
        elif isinstance(sentence, list) or sentence.__class__.__name__ == "Text":
            a=[]; [a.extend(self.search(s)) for s in sentence]; return a
        elif isinstance(sentence, basestring):
            sentence = Sentence(sentence)
        elif isinstance(sentence, Match) and len(sentence) > 0:
            sentence = sentence[0].sentence.slice(sentence[0].index, sentence[-1].index + 1)
        a = []
        v = self._variations()
        u = {}
        m = self.match(sentence, _v=v)
        while m:
            a.append(m)
            m = self.match(sentence, start=m.words[-1].index+1, _v=v, _u=u)
        return a
19279
def regex_search(txt, rgx, ignorecase=False, multiline=False):
    '''
    
    '''
    flag = 0
    if ignorecase:
        flag |= re.I
    if multiline:
        flag |= re.M
    obj = re.search(rgx, txt, flag)
    if not obj:
        return
    return obj.groups()
1193123
public List<String> matchAll(String text, int limit, boolean isDensityMatch, boolean isGreedMatch) {
		if(null == text){
			return null;
		}
		
		List<String> findedWords = new ArrayList<String>();
		WordTree current = this;
		int length = text.length();
		StringBuilder wordBuffer;//存放查找到的字符缓存。完整出现一个词时加到findedWords中，否则清空
		char currentChar;
		for (int i = 0; i < length; i++) {
			wordBuffer = StrUtil.builder();
			for (int j = i; j < length; j++) {
				currentChar = text.charAt(j);
//				Console.log("i: {}, j: {}, currentChar: {}", i, j, currentChar);
				if(StopChar.isStopChar(currentChar)){
					if(wordBuffer.length() > 0){
						//做为关键词中间的停顿词被当作关键词的一部分被返回
						wordBuffer.append(currentChar);
					}else{
						//停顿词做为关键词的第一个字符时需要跳过
						i++;
					}
					continue;
				}else if(false == current.containsKey(currentChar)){
					//非关键字符被整体略过，重新以下个字符开始检查
					break;
				}
				wordBuffer.append(currentChar);
				if(current.isEnd(currentChar)){
					//到达单词末尾，关键词成立，从此词的下一个位置开始查找
					findedWords.add(wordBuffer.toString());
					if(limit > 0 && findedWords.size() >= limit){
						//超过匹配限制个数，直接返回
						return findedWords;
					}
					if(false == isDensityMatch){
						//如果非密度匹配，跳过匹配到的词
						i = j;
					}
					if(false == isGreedMatch){
						//如果懒惰匹配（非贪婪匹配）。当遇到第一个结尾标记就结束本轮匹配
						break;
					}
				}
				current = current.get(currentChar);
				if(null == current){
					break;
				}
			}
			current = this;
		}
		return findedWords;
	}