计算分词结果的二元模型分值
@param words 分词结果
@return 二元模型分值
174764
public static float bigram(List<Word> words){
        if(words.size() > 1){
            float score=0;
            for(int i=0; i<words.size()-1; i++){
                score += getScore(words.get(i).getText(), words.get(i+1).getText());
            }
            return score;
        }
        return 0;
    }
1201144
public static float trigram(List<Word> words){
        if(words.size() > 2){
            float score=0;
            for(int i=0; i<words.size()-2; i++){
                score += Trigram.getScore(words.get(i).getText(), words.get(i+1).getText(), words.get(i+2).getText());
            }
            return score;
        }
        return 0;
    }
994632
public static RoaringBitmap bitmapOf(final long[] words) {
    // split long[] into blocks.
    // each block becomes a single container, if any bit is set
    final RoaringBitmap ans = new RoaringBitmap();
    int containerIndex = 0;
    for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
      final int to = Math.min(from + BLOCK_LENGTH, words.length);
      final int blockCardinality = cardinality(from, to, words);
      if (blockCardinality > 0) {
        ans.highLowContainer.insertNewKeyValueAt(containerIndex++, Util.highbits(from * Long.SIZE),
            BitSetUtil.containerOf(from, to, blockCardinality, words));
      }
    }
    return ans;
  }
1332121
public static List<Window> windows(List<String> words, int windowSize) {

        List<Window> ret = new ArrayList<>();

        for (int i = 0; i < words.size(); i++)
            ret.add(windowForWordInPosition(windowSize, i, words));


        return ret;
    }
409641
public Map<List<Word>, Float> ngram(List<Word>... sentences){
        if("bigram".equals(NGRAM)){
            return Bigram.bigram(sentences);
      
        }
        if("trigram".equals(NGRAM)){
            return Trigram.trigram(sentences);
        }
        return null;
    }
876210
public List<List<Word>> getSimpleSentenceList()
    {
        List<List<Word>> simpleList = new LinkedList<List<Word>>();
        for (Sentence sentence : sentenceList)
        {
            List<Word> wordList = new LinkedList<Word>();
            for (IWord word : sentence.wordList)
            {
                if (word instanceof CompoundWord)
                {
                    for (Word inner : ((CompoundWord) word).innerList)
                    {
                        wordList.add(inner);
                    }
                }
                else
                {
                    wordList.add((Word) word);
                }
            }
            simpleList.add(wordList);
        }

        return simpleList;
    }
1180823
public List<Term> recognition(List<String> words, int offe) {
        List<Term> terms = new ArrayList<>(words.size());
        int tempOffe = 0;
        for (String word : words) {
            TermNatures tn = getTermNatures(word);

            terms.add(new Term(word, offe + tempOffe, tn));
            tempOffe += word.length();
        }
        new NatureRecognition().recognition(new Result(terms));
        return terms;
    }
441567
function(words) {
      for (var bytes = [], b = 0; b < words.length * 32; b += 8)
        bytes.push((words[b >>> 5] >>> (24 - b % 32)) & 0xFF);
      return bytes;
    }
1008117
public void word (Color color, String... words) {
		for (String word : words) {
			addRule(new WordHighlightRule(color, word));
		}
	}
149521
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2){
        //文本1
        StringBuilder text1 = new StringBuilder();
        words1.forEach(word -> text1.append(word.getText()));
        //文本2
        StringBuilder text2 = new StringBuilder();
        words2.forEach(word -> text2.append(word.getText()));
        int maxTextLength = Math.max(text1.length(), text2.length());
        if(maxTextLength == 0){
            //两个空字符串
            return 1.0;
        }
        //计算文本1和文本2的编辑距离
        int editDistance = editDistance(text1.toString(), text2.toString());
        double score = (1 - editDistance / (double)maxTextLength);
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("文本1："+text1.toString());
            LOGGER.debug("文本2："+text2.toString());
            LOGGER.debug("文本1和文本2的编辑距离："+editDistance);
            LOGGER.debug("文本1和文本2的最大长度："+maxTextLength);
            LOGGER.debug("文本1和文本2的相似度分值：1 - "+editDistance+" / (double)"+maxTextLength+"="+score);
        }
        return score;
    }
1208873
public List<String> getWords() {
        List<String> result = new ArrayList<>();
        List<Word> words = WordParser.parse(question.replace("?", "").replace("？", ""));
        for (Word word : words) {
            result.add(word.getText());
        }
        return result;
    }
536475
@Override
    public Collection<String> wordsNearest(INDArray words, int top) {
        return modelUtils.wordsNearest(words, top);
    }
1224118
@Override
    public double similarScore(List<Word> words1, List<Word> words2) {
        if(words1 == null || words2 == null){
            //只要有一个文本为null，规定相似度分值为0，表示完全不相等
            return 0.0;
        }
        if(words1.isEmpty() && words2.isEmpty()){
            //如果两个文本都为空，规定相似度分值为1，表示完全相等
            return 1.0;
        }
        if(words1.isEmpty() || words2.isEmpty()){
            //如果一个文本为空，另一个不为空，规定相似度分值为0，表示完全不相等
            return 0.0;
        }
        //输出词列表信息
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("词列表1：");
            LOGGER.debug("\t" + words1);
            LOGGER.debug("词列表2：");
            LOGGER.debug("\t" + words2);
        }
        double score = scoreImpl(words1, words2);
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("分值："+score);
        }
        score = (int)(score*1000000+0.5)/(double)1000000;
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("取六位小数，四舍五入，分值："+score);
        }
        return score;
    }
52569
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        //用词频来标注词的权重
        taggingWeightWithWordFrequency(words1, words2);
        //构造权重快速搜索容器
        Map<String, Float> weights1 = toFastSearchMap(words1);
        Map<String, Float> weights2 = toFastSearchMap(words2);
        //所有的不重复词
        Set<Word> words = new HashSet<>();
        words.addAll(words1);
        words.addAll(words2);
        //向量的维度为words的大小，每一个维度的权重是词频
        //manhattanDistance=|x1-x2|+|y1-y2|
        AtomicFloat manhattanDistance = new AtomicFloat();
        //计算
        words
            .parallelStream()
            .forEach(word -> {
                Float x1 = weights1.get(word.getText());
                Float x2 = weights2.get(word.getText());
                if (x1 == null) {
                    x1 = 0f;
                }
                if (x2 == null) {
                    x2 = 0f;
                }
                //|x1-x2|
                float oneOfTheDimension = Math.abs(x1 - x2);
                //+
                manhattanDistance.addAndGet(oneOfTheDimension);
            });
        double score = 0;
        if(manhattanDistance.get() == 0){
            //距离为0，表示完全相同
            score = 1.0;
        }else {
            //使用BigDecimal保证精确计算浮点数
            //score = 1 / (double)(manhattanDistance.get()+1);
            score = BigDecimal.valueOf(1).divide(BigDecimal.valueOf(manhattanDistance.get()+1), 9, BigDecimal.ROUND_HALF_UP).doubleValue();
        }
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("文本1和文本2的曼哈顿距离："+manhattanDistance.get());
            LOGGER.debug("文本1和文本2的相似度分值：1 / (double)("+manhattanDistance.get()+"+1)="+score);
        }
        return score;
    }
314084
public String[] getParams(String word) {
        for (Forest forest : forests) {
            if (forest == null) {
                continue;
            }
            SmartForest<String[]> sf = forest;
            for (int i = 0; i < word.length(); i++) {
                sf = sf.get(word.charAt(i));
                if (sf == null) {
                    return null;
                }
            }
            if (sf.getStatus() > 1) {
                return sf.getParam();
            } else {
                return null;
            }
        }
        return null;
    }
1105368
public WordBuilder<I> append(Word<? extends I> word) {
        int wLen = word.length();
        ensureAdditionalCapacity(wLen);
        word.writeToArray(0, array, length, wLen);
        length += wLen;
        return this;
    }
377736
public List<String> getSpannedWords() {
    if (isTerminal()) {
      return words;
    } else {
      List<String> words = Lists.newArrayList();
      words.addAll(left.getSpannedWords());
      words.addAll(right.getSpannedWords());
      return words;
    }
  }