compare similarity
    s1 : sentence1
    s2 : sentence2
    seg : True : The original sentences need jieba.cut
          Flase : The original sentences have been cut.
    ignore: True: ignore OOV words
            False: get vector randomly for OOV words
174901
def compare(s1, s2, seg=True, ignore=False, stopwords=False):
    '''
    
    '''
    if s1 == s2: return 1.0
    
    s1_words = []
    s2_words = []

    if seg:
        s1 = [x for x in jieba.cut(s1, cut_all=False, HMM=False)]
        s2 = [x for x in jieba.cut(s2, cut_all=False, HMM=False)]
    else:
        s1 = s1.split()
        s2 = s2.split()

    # check stopwords
    if not stopwords:
        global _stopwords
        for x in s1: 
            if not x in _stopwords:
                s1_words.append(x)
        for x in s2:
            if not x in _stopwords:
                s2_words.append(x)
    else:
        s1_words = s1 
        s2_words = s2

    assert len(s1) > 0 and len(s2) > 0, "The length of s1 and s2 should > 0."
    return _similarity_distance(s1_words, s2_words, ignore)
1375064
def segment(self, text, lower=True, use_stop_words=True, use_speech_tags_filter=False):
        
        jieba_result = cut_for_search(text)
        jieba_result = [w for w in jieba_result]

        # 去除特殊符号
        word_list = [w.strip() for w in jieba_result]
        word_list = [word for word in word_list if len(word) > 0]

        if use_speech_tags_filter == False:
            jieba_result = [w for w in jieba_result]

        if lower:
            word_list = [word.lower() for word in word_list]

        if use_stop_words:
            word_list = [word.strip() for word in word_list if word.strip() not in self.stop_words]

        return word_list
1173566
public static List<List<Term>> seg2sentence(String text, Filter... filterArrayChain)
    {
        List<List<Term>> sentenceList = SEGMENT.seg2sentence(text);
        for (List<Term> sentence : sentenceList)
        {
            ListIterator<Term> listIterator = sentence.listIterator();
            while (listIterator.hasNext())
            {
                if (filterArrayChain != null)
                {
                    Term term = listIterator.next();
                    for (Filter filter : filterArrayChain)
                    {
                        if (!filter.shouldInclude(term))
                        {
                            listIterator.remove();
                            break;
                        }
                    }
                }
            }
        }

        return sentenceList;
    }
1233566
private int findRank(SemanticSpace sspace, 
                         String target, String other) {
        Vector v1 = sspace.getVector(target);
        Vector v2 = sspace.getVector(other);
        // Compute the base similarity between the two words
        double baseSim = Similarity.cosineSimilarity(v1, v2);
        int rank = 0;
        // Next, count how many words are more similar to the target than the
        // other word is
        for (String word : sspace.getWords()) {
            Vector v = sspace.getVector(word);
            double sim = Similarity.cosineSimilarity(v1, v);
            if (sim > baseSim)
                rank++;
        }
        return rank;
    }
582600
@Override
    public double similarScore(String text1, String text2) {
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("文本1：");
            LOGGER.debug("\t" + text1);
            LOGGER.debug("文本2：");
            LOGGER.debug("\t" + text2);
        }
        if(text1 == null || text2 == null){
            //只要有一个文本为null，规定相似度分值为0，表示完全不相等
            return 0.0;
        }
        //分词
        List<Word> words1 = seg(text1);
        List<Word> words2 = seg(text2);
        //计算相似度分值
        return similarScore(words1, words2);
    }
688546
function (s) {
            var c = clean( s );
            var all = _.words( c, wordDelimiter );
            var words = _(all).chain()
                    // remove stop words
                    .filter( function (w) {
                        return (stopWords.indexOf( w ) === -1) ;
                    })
                    // apply stemmer
                    .map( function (w) {
                        return stemmer( w );
                    })
                    // collect word frequencies
                    .reduce( function (collect, w) {
                        collect[w] = collect[w] ? collect[w] + 1 : 1 ;
                        return collect;
                    }, {}).value();
            // remove a word from this sentence to reduce redundancy in results
            var remove = function (w) {
                return delete words[w];
            };
            return {
                orig: s,
                words: words,
                remove: remove
            };
        }
480609
function isPaddingBetweenTokens(first, second) {
            const comments = sourceCode.getCommentsBefore(second);
            const len = comments.length;

            // If there is no comments
            if (len === 0) {
                const linesBetweenFstAndSnd = second.loc.start.line - first.loc.end.line - 1;

                return linesBetweenFstAndSnd >= 1;
            }


            // If there are comments
            let sumOfCommentLines = 0; // the numbers of lines of comments
            let prevCommentLineNum = -1; // line number of the end of the previous comment

            for (let i = 0; i < len; i++) {
                const commentLinesOfThisComment = comments[i].loc.end.line - comments[i].loc.start.line + 1;

                sumOfCommentLines += commentLinesOfThisComment;

                /*
                 * If this comment and the previous comment are in the same line,
                 * the count of comment lines is duplicated. So decrement sumOfCommentLines.
                 */
                if (prevCommentLineNum === comments[i].loc.start.line) {
                    sumOfCommentLines -= 1;
                }

                prevCommentLineNum = comments[i].loc.end.line;
            }

            /*
             * If the first block and the first comment are in the same line,
             * the count of comment lines is duplicated. So decrement sumOfCommentLines.
             */
            if (first.loc.end.line === comments[0].loc.start.line) {
                sumOfCommentLines -= 1;
            }

            /*
             * If the last comment and the second block are in the same line,
             * the count of comment lines is duplicated. So decrement sumOfCommentLines.
             */
            if (comments[len - 1].loc.end.line === second.loc.start.line) {
                sumOfCommentLines -= 1;
            }

            const linesBetweenFstAndSnd = second.loc.start.line - first.loc.end.line - 1;

            return linesBetweenFstAndSnd - sumOfCommentLines >= 1;
        }
990975
function compareSlotSegs(seg1, seg2) {
	return seg1.start - seg2.start || // earlier start time goes first
		(seg2.end - seg2.start) - (seg1.end - seg1.start) || // tie? longer-duration goes first
		(seg1.event.title || '').localeCompare(seg2.event.title); // tie? alphabetically by title
}
55637
public AnalyzedSentence copy(AnalyzedSentence sentence) {
    AnalyzedTokenReadings[] copyTokens = new AnalyzedTokenReadings[sentence.getTokens().length];
    for (int i = 0; i < copyTokens.length; i++) {
      AnalyzedTokenReadings analyzedTokens = sentence.getTokens()[i];
      copyTokens[i] = new AnalyzedTokenReadings(analyzedTokens.getReadings(), analyzedTokens.getStartPos());
      copyTokens[i].setHistoricalAnnotations(analyzedTokens.getHistoricalAnnotations());
      copyTokens[i].setChunkTags(analyzedTokens.getChunkTags());
      if (analyzedTokens.isImmunized()) {
        copyTokens[i].immunize();
      }
      if (analyzedTokens.isIgnoredBySpeller()) {
        copyTokens[i].ignoreSpelling();
      }
      copyTokens[i].setWhitespaceBefore(sentence.getTokens()[i].isWhitespaceBefore());
    }
    return new AnalyzedSentence(copyTokens, sentence.whPositions, sentence.getTokensWithoutWhitespace(), sentence.getPreDisambigTokensWithoutWhitespace());
  }
930090
def segments(self, text):
        
        start = 0
        for (L, P, R, B, end) in Detector.candidates(text):
            # if there's already a newline there, we have nothing to do
            if B:
                continue
            if self.predict(L, P, R):
                yield text[start:end].rstrip()
                start = end
            # otherwise, there's probably not a sentence boundary here
        yield text[start:].rstrip()
1345583
@SuppressWarnings("MethodMayBeStatic")
    public NerResultSet process(final String _text) {
        // tokenization
        final List<List<Token>> tokens = TOKENIZER.process(_text);
        // extract name phrase:
        final List<List<Phrase>> phrases = new ArrayList<>();

        for (final List<Token> tokenList : tokens) {
            // for each sentence
            final List<Phrase> sentencePhrase = new ArrayList<>();
            int wordPtr = 0;
            int phraseStubStartPtr = 0;
            while (wordPtr < tokenList.size()) {

                // iterate through every words in the sentence
                final List<Token> currentPhrase = new ArrayList<>();

                // get the phrase stub
                while (true) {
                    // we merge the names combined with the word 'and'
                    // if the second phrase is much shorter than the first phrase
                    // example: Canberra Institution of Science and Technology
                    boolean mergeAndFlag = false;
                    if ("and".equalsIgnoreCase(tokenList.get(wordPtr).text)
                        && !currentPhrase.isEmpty()
                        && wordPtr + 1 < tokenList.size()
                        && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1)) {
                        if (wordPtr + currentPhrase.size() - 1 >= tokenList.size()) {
                            mergeAndFlag = true;
                        }
                        else {
                            for (int i = wordPtr + 1; i <= wordPtr + currentPhrase.size() - 1; i++) {
                                if (!detectNameWordInSentenceByPosition(tokenList, i)) {
                                    mergeAndFlag = true;
                                    break;
                                }
                            }
                        }
                    }

                    if (detectNameWordInSentenceByPosition(tokenList, wordPtr)
                        || (equalsIgnoreCase(tokenList.get(wordPtr).text, "of", "the", "'s")
                            && !currentPhrase.isEmpty()
                            && wordPtr + 1 < tokenList.size()
                            && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1))
                        || mergeAndFlag) {
                        if (currentPhrase.isEmpty()) phraseStubStartPtr = wordPtr; // record the index of the first word
                        currentPhrase.add(tokenList.get(wordPtr));
                        wordPtr++;
                        if (wordPtr == tokenList.size()) break;
                    }
                    else break;
                }

                //get the attached 'the' or 'a'
                int phrasePos = phraseStubStartPtr;
                int phraseLen = currentPhrase.size();

                // if phrasePos is zero, then it is the first word, there won't be any 'the' or 'a' attached to this word.
                if (phrasePos > 0) {
                    final Token attachedArticle = tokenList.get(phrasePos - 1);
                    if ("the".equalsIgnoreCase(attachedArticle.text) || "a".equalsIgnoreCase(attachedArticle.text)) {
                        phrasePos--;
                        phraseLen++;
                    }
                }

                //handling non-name single words such as: Monday, January, etc.
                if (currentPhrase.size() == 1 && phraseStubStartPtr == phrasePos) {
                    if (NON_NAME_WORDS.contains(currentPhrase.get(0).text)) {
                        currentPhrase.clear();
                        wordPtr--;
                    }
                }

                if (currentPhrase.isEmpty()) {
                    // if the current pointed word is not a name phrase
                    // ESTIMATE IF IT IS A TIME/DATE
                    final List<Token> currentDatePhrase = new ArrayList<>();
                    final DatePhraseModel dpm = new DatePhraseModel();
                    int tempPtr = wordPtr;
                    while (true) {
                        final Token word = tokenList.get(tempPtr);
                        final DateType type = getDateType(word.text);
                        if (type != NONE
                            || equalsIgnoreCase(word.text, "of", ",", "the")
                               && !currentDatePhrase.isEmpty()
                               && tempPtr + 1 != tokenList.size()
                               && getDateType(tokenList.get(tempPtr + 1).text) != NONE
                               && getDateType(tokenList.get(tempPtr - 1).text) != NONE) {
                            // is date word:
                            currentDatePhrase.add(word);
                            dpm.addType(type);
                            tempPtr++;
                            if (tempPtr == tokenList.size()) break;
                        }
                        else break;
                    }
                    if (dpm.isDate()) {
                        // add the phrase to phrase array
                        sentencePhrase.add(new Date(currentDatePhrase, wordPtr, currentDatePhrase.size(), wordPtr));
                        wordPtr = tempPtr;
                    }
                }
                else if (!(currentPhrase.size() == 1
                           && (isPastTense(currentPhrase.get(0).text) || isPlural(currentPhrase.get(0).text)))) {
                    sentencePhrase.add(new Name(currentPhrase, phrasePos, phraseLen, phraseStubStartPtr));
                }

                wordPtr++;
            }

            phrases.add(sentencePhrase);
        }

        // find attached words(preps):
        for (int si = 0; si < tokens.size(); si++) {
            final List<Token> sentenceToken = tokens.get(si);
            final List<Phrase> sentencePhrase = phrases.get(si);
            for (int pi = 0; pi < sentencePhrase.size(); pi++) {
                getAttachedPrep(sentenceToken, sentencePhrase, pi);
            }
        }

        return new NerResultSet(phrases, tokens);
    }
429577
private List<Word> segSentence(final String sentence){
        if(sentence.length() == 1){
            if(KEEP_WHITESPACE){
                List<Word> result = new ArrayList<>(1);
                result.add(new Word(KEEP_CASE ? sentence : sentence.toLowerCase()));
                return result;
            }else{
                if(!Character.isWhitespace(sentence.charAt(0))){
                    List<Word> result = new ArrayList<>(1);
                    result.add(new Word(KEEP_CASE ? sentence : sentence.toLowerCase()));
                    return result;
                }
            }
        }
        if(sentence.length() > 1){
            List<Word> list = segImpl(sentence);
            if(list != null){
                if(PERSON_NAME_RECOGNIZE){
                    list = PersonName.recognize(list);
                }
                return list;
            }else{
                LOGGER.error("文本 "+sentence+" 没有获得分词结果");
            }
        }
        return Collections.emptyList();
    }
1116852
def tokenize(self,
                 sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_preprocess, bool, bool, bool, bool, Callable[[str], text_type])->Union[List[text_type], TokenizedSenetence]
        
        assert isinstance(normalize, bool)
        assert isinstance(sentence, text_type)
        normalized_sentence = func_normalizer(sentence)
        result = self.call_juman_interface(normalized_sentence)

        token_objects = [
            self.__extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects
            )
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects
70024
private boolean match(Token token, Element element, int baseTokenIndex, Sentence sentence) {
		boolean match;
		boolean negated;
		// Sees if the mask must or not match.
		// Negated is optional, so it can be null, true or false.
		// If null, consider as false.
		if (element.isNegated() == null) {
			match = false;
			negated = false;
		} else {
			match = element.isNegated().booleanValue();
			negated = element.isNegated().booleanValue();
		}
		for (Mask mask : element.getMask()) {
			// If the token must match the mask.
			if (!negated) {
				// If not negated, match starts as false and just one match is needed to make it true.
				if (mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(token.getLexeme())) {
					match = true;
				} else if (mask.getPrimitiveMask() != null && matchLemma(token, mask.getPrimitiveMask())) {
					match = true;
				} else if (mask.getTagMask() != null && token.getMorphologicalTag() != null) {
					match = match | token.getMorphologicalTag().matchExact(mask.getTagMask(), false);
				} else if (mask.getTagReference() != null && token.getMorphologicalTag() != null) {
					match = match | token.getMorphologicalTag().match(RuleUtils.createTagMaskFromReference(mask.getTagReference(), sentence, baseTokenIndex), false);
				} else if (mask.getOutOfBounds() != null && (baseTokenIndex == 0 || baseTokenIndex == sentence.getTokens().size() -1)) {
					match = false;
				}
			} else { // The token must NOT match the mask.
				// If negated, match starts as true and just one match is needed to make it false.
				if (mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(token.getLexeme())) {
					match = false;
				} else if (mask.getPrimitiveMask() != null && matchLemma(token, mask.getPrimitiveMask())) {
                  match = false;
                } else if (mask.getTagMask() != null && token!=null && token.getMorphologicalTag() != null) {
					match = match & !token.getMorphologicalTag().matchExact(mask.getTagMask(),false);
				} else if (mask.getTagReference() != null && token!=null && token.getMorphologicalTag() != null) {
					match = match & !token.getMorphologicalTag().match(RuleUtils.createTagMaskFromReference(mask.getTagReference(), sentence, baseTokenIndex), false);
				} else if (mask.getOutOfBounds() != null && (baseTokenIndex == 0 || baseTokenIndex == sentence.getTokens().size() -1)) {
					match = false;
				}
			}
		}
		return match;
	}
743184
def compare(self, statement_a, statement_b):
        
        # Make both strings lowercase
        document_a = self.nlp(statement_a.text.lower())
        document_b = self.nlp(statement_b.text.lower())

        statement_a_lemmas = set([
            token.lemma_ for token in document_a if not token.is_stop
        ])
        statement_b_lemmas = set([
            token.lemma_ for token in document_b if not token.is_stop
        ])

        # Calculate Jaccard similarity
        numerator = len(statement_a_lemmas.intersection(statement_b_lemmas))
        denominator = float(len(statement_a_lemmas.union(statement_b_lemmas)))
        ratio = numerator / denominator

        return ratio
639494
public Pair<List<SuggestedReplacement>, SortedMap<String, Float>> computeFeatures(List<String> suggestions, String word, AnalyzedSentence sentence, int startPos) {
    if (suggestions.isEmpty()) {
      return Pair.of(Collections.emptyList(), Collections.emptySortedMap());
    }
    if (topN <= 0) {
      topN = suggestions.size();
    }
    List<String> topSuggestions = suggestions.subList(0, Math.min(suggestions.size(), topN));
    //EditDistance<Integer> levenshteinDistance = new LevenshteinDistance(4);
    EditDistance levenstheinDistance = new EditDistance(word, EditDistance.DistanceAlgorithm.Damerau);
    SimilarityScore<Double> jaroWrinklerDistance = new JaroWinklerDistance();
    List<Feature> features = new ArrayList<>(topSuggestions.size());

    for (String candidate : topSuggestions) {
      double prob1 = languageModel.getPseudoProbability(Collections.singletonList(candidate)).getProb();
      double prob3 = LanguageModelUtils.get3gramProbabilityFor(language, languageModel, startPos, sentence, candidate);
      //double prob4 = LanguageModelUtils.get4gramProbabilityFor(language, languageModel, startPos, sentence, candidate);
      long wordCount = ((BaseLanguageModel) languageModel).getCount(candidate);
      int levenstheinDist = levenstheinDistance.compare(candidate, 3);
      double jaroWrinklerDist = jaroWrinklerDistance.apply(word, candidate);
      DetailedDamerauLevenstheinDistance.Distance detailedDistance = DetailedDamerauLevenstheinDistance.compare(word, candidate);

      features.add(new Feature(prob1, prob3, wordCount, levenstheinDist, detailedDistance, jaroWrinklerDist, candidate));
    }
    if (!"noop".equals(score)) {
      features.sort(Feature::compareTo);
    }
    //logger.trace("Features for '%s' in '%s': %n", word, sentence.getText());
    //features.stream().map(Feature::toString).forEach(logger::trace);
    List<String> words = features.stream().map(Feature::getWord).collect(Collectors.toList());

    // compute general features, not tied to candidates
    SortedMap<String, Float> matchData = new TreeMap<>();
    matchData.put("candidateCount", (float) words.size());

    List<SuggestedReplacement> suggestionsData = features.stream().map(f -> {
      SuggestedReplacement s = new SuggestedReplacement(f.getWord());
      s.setFeatures(f.getData());
      return s;
    }).collect(Collectors.toList());
    return Pair.of(suggestionsData, matchData);
  }
847830
private List<Word> seg(String text){
        if(text == null){
            return Collections.emptyList();
        }
        if(segmentation == null){
            //延迟初始化
            segmentation = SegmentationFactory.getSegmentation(SegmentationAlgorithm.MaxNgramScore);
        }
        List<Word> words = segmentation.seg(text);
        if(filterStopWord) {
            //停用词过滤
            StopWord.filterStopWords(words);
        }
        return words;
    }