编辑距离

@param wrongWord 串A，其实它们两个调换位置还是一样的
@param rightWord 串B
@return 它们之间的距离
515056
public static int compute(char[] wrongWord, char[] rightWord)
    {
        final int m = wrongWord.length;
        final int n = rightWord.length;

        int[][] d = new int[m + 1][n + 1];
        for (int j = 0; j <= n; ++j)
        {
            d[0][j] = j;
        }
        for (int i = 0; i <= m; ++i)
        {
            d[i][0] = i;
        }

        for (int i = 1; i <= m; ++i)
        {
            char ci = wrongWord[i - 1];
            for (int j = 1; j <= n; ++j)
            {
                char cj = rightWord[j - 1];
                if (ci == cj)
                {
                    d[i][j] = d[i - 1][j - 1];
                }
                else if (i > 1 && j > 1 && ci == rightWord[j - 2] && cj == wrongWord[i - 2])
                {
                    // 交错相等
                    d[i][j] = 1 + Math.min(d[i - 2][j - 2], Math.min(d[i][j - 1], d[i - 1][j]));
                }
                else
                {
                    // 等号右边的分别代表 将ci改成cj                   错串加cj         错串删ci
                    d[i][j] = Math.min(d[i - 1][j - 1] + 1, Math.min(d[i][j - 1] + 1, d[i - 1][j] + 1));
                }
            }
        }

        return d[m][n];
    }
167393
def mergewords(self, newword, *originalwords, **kwargs):
        """"""
        return self.correctwords(originalwords, [newword], **kwargs)
888984
protected void addWord(List<Word> result, String text, int start, int len){
        Word word = getWord(text, start, len);
        if(word != null){
            result.add(word);
        }
    }
1208072
def correctwords(self, originalwords, newwords, **kwargs):
        """"""
        for w in originalwords:
            if not isinstance(w, Word):
                raise Exception("Original word is not a Word instance: " + str(type(w)))
            elif w.sentence() != self:
                raise Exception("Original not found as member of sentence!")
        for w in newwords:
            if not isinstance(w, Word):
                raise Exception("New word is not a Word instance: " + str(type(w)))
        if 'suggest' in kwargs and kwargs['suggest']:
            del kwargs['suggest']
            return self.correct(suggestion=newwords,current=originalwords, **kwargs)
        else:
            return self.correct(original=originalwords, new=newwords, **kwargs)
1246448
private IWord _nextNumberWord(IWord word) throws IOException
    {
        IWord dWord = null, unit = null;
        IWord dw1 = super.next();
        if ( dw1 == null ) {
            return null;
        }

        String[] w1Entity = dw1.getEntity();
        int w1Len = dw1.getValue().length();

        //pure numeric
        if ( ArrayUtil.indexOf(Entity.E_NUMERIC_ARABIC, w1Entity) > -1 ) {
            IWord dw2 = super.next();
            if ( dw2 == null ) {
                dWord = new Word(word.getValue()+dw1.getValue(), IWord.T_CJK_WORD);
                dWord.setPosition(word.getPosition());
                dWord.setEntity(Entity.E_THE_NUMBER_A);
                return dWord;
            }

            int w2Len = dw2.getValue().length();

            //template like '个', 'xx个'
            unit = dic.get(ILexicon.NUMBER_UNIT, ""+dw2.getValue().charAt(0));
            if ( unit != null ) {
                String w2Val = dw2.getValue();
                if ( w2Len == 1 || NumericUtil.isCNNumericString(w2Val, 0, w2Len-1)
                        || StringUtil.isDigit(w2Val, 0, w2Len-1) ) {
                    dWord = new Word(word.getValue()+dw1.getValue()+w2Val, IWord.T_CJK_WORD);
                    dWord.setEntity(unit.getEntity());
                    return dWord;
                }
            }

            eWordPool.push(dw2);
            dWord = new Word(word.getValue()+dw1.getValue(), IWord.T_CJK_WORD);
            dWord.setPosition(word.getPosition());
            dWord.setEntity(Entity.E_THE_NUMBER_A);
            return dWord;
        }

        //template like '个', 'x个'
        unit = dic.get(ILexicon.NUMBER_UNIT, ""+dw1.getValue().charAt(w1Len-1));
        if ( unit != null ) {
            String w1Val = dw1.getValue();
            if ( w1Len == 1 || NumericUtil.isCNNumericString(w1Val, 0, w1Len-1)
                    || StringUtil.isDigit(w1Val, 0, w1Len-1) ) {
                dWord = new Word(word.getValue()+w1Val, IWord.T_CJK_WORD);
                dWord.setEntity(unit.getEntity());
                return dWord;
            }
        }

        eWordPool.push(dw1);
        return null;
    }
396179
private static String irregularity(String word){
        if(Character.isDigit(word.charAt(0))){
            LOGGER.debug("词以数字开头，忽略："+word);
            return null;
        }
        if(word.startsWith("0x")
                || word.startsWith("0X")){
            LOGGER.debug("词为16进制，忽略："+word);
            return null;
        }
        if(word.endsWith("l")
                && isNumeric(word.substring(0, word.length()-1))){
            LOGGER.debug("词为long类型数字，忽略："+word);
            return null;
        }
        if(UNICODE.matcher(word).find()){
            LOGGER.debug("词为UNICODE字符编码，忽略："+word);
            return null;
        }
        switch (word){
            //I’ll do it. You'll see.
            case "ll": return "will";
            //If you’re already building applications using Spring.
            case "re": return "are";
            //package com.manning.sdmia.ch04;
            case "ch": return "chapter";
            //you find you’ve made a
            case "ve": return "have";
            //but it doesn’t stop there.
            case "doesn": return "does";
            //but it isn’t enough.
            case "isn": return "is";
            //<input type="text" name="firstName" /><br/>
            case "br": return null;
        }
        return word;
    }
633114
private void foreign(Term[] terms) {
        List<Term> list = new ArrayList<>();

        Term term = null;
        Term temp = null;

        boolean mid = false;

        for (int i = 0; i < terms.length; i++) {
            term = terms[i];
            if (term == null) {
                continue;
            }

            if (DATDictionary.foreign(term)) {
                mid = "·".equals(term.getName());

                if (mid && list.size() == 0) { // mid 不能出现在名字首位
                    mid = false;
                    continue;
                }

                list.add(term);
                continue;
            }

            if (mid) {
                list = list.subList(0, list.size() - 1);
                mid = false;
            }

            if (list.size() == 0) {
                continue;
            }

            if (list.size() == 1) { //只有一个就别识别了
                list.clear();
                continue;
            }

            TermUtil.insertTerm(terms, list, TermNatures.NRF);

            list.clear();

        }

    }
826962
public static <L, R> @NonNull Pair<L, R> right(final @Nullable R right) {
    return of(null, right);
  }
1185584
public static EvaluationResult evaluation(String resultText, String standardText, String perfectResult, String wrongResult) {
        long start = System.currentTimeMillis();
        int perfectLineCount=0;
        int wrongLineCount=0;
        int perfectCharCount=0;
        int wrongCharCount=0;
        try(BufferedReader resultReader = new BufferedReader(new InputStreamReader(new FileInputStream(resultText),"utf-8"));
            BufferedReader standardReader = new BufferedReader(new InputStreamReader(new FileInputStream(standardText),"utf-8"));
            BufferedWriter perfectResultWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(perfectResult),"utf-8"));
            BufferedWriter wrongResultWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(wrongResult),"utf-8"))){
            String result;
            while( (result = resultReader.readLine()) != null ){
                result = result.trim();
                String standard = standardReader.readLine().trim();
                if(result.equals("")){
                    continue;
                }
                if(result.equals(standard)){
                    //分词结果和标准一模一样
                    perfectResultWriter.write(standard+"\n");
                    perfectLineCount++;
                    perfectCharCount+=standard.replaceAll("\\s+", "").length();
                }else{
                    //分词结果和标准不一样
                    wrongResultWriter.write("实际分词结果："+result+"\n");
                    wrongResultWriter.write("标准分词结果："+standard+"\n");
                    wrongLineCount++;
                    wrongCharCount+=standard.replaceAll("\\s+", "").length();
                }
            }
        } catch (IOException ex) {
            LOGGER.error("分词效果评估失败：", ex);
        }
        long cost = System.currentTimeMillis() - start;
        int totalLineCount = perfectLineCount+wrongLineCount;
        int totalCharCount = perfectCharCount+wrongCharCount;
        LOGGER.info("评估耗时："+cost+" 毫秒");
        EvaluationResult er = new EvaluationResult();
        er.setPerfectCharCount(perfectCharCount);
        er.setPerfectLineCount(perfectLineCount);
        er.setTotalCharCount(totalCharCount);
        er.setTotalLineCount(totalLineCount);
        er.setWrongCharCount(wrongCharCount);
        er.setWrongLineCount(wrongLineCount);     
        return er;
    }
921481
public boolean isNormalWord(@NotNull String word, @NotNull Language language) {
        Timer timer = MetricsFactory.getMetrics().getTimer(MetricsType.TEXT_UTILS_IS_NORMAL_WORD.name());

        if (StringUtils.isEmpty(word)) {
            return false;
        }

        try {
            Timer morphTimer = MetricsFactory.getMetrics().getTimer(MetricsType.TEXT_UTILS_GET_MORPH_INFO.name());
            List<String> morphInfo = MorphologyFactory.getMorphology(language).getMorphInfo(word);
            morphTimer.stop();

            return morphInfo.stream()
                            .allMatch(new MorphRulesPredicate());

        } catch (Exception e) {
            LOGGER.warn("Can't get morph info: {" + word + "} ", e);
        } finally {
            timer.stop();
        }

        return false;
    }
942890
public static Boolean and(Boolean left, Boolean right) {
        return left && Boolean.TRUE.equals(right);
    }
531340
public int indexOf(String word) {
        if (vocabulary.containsKey(word)) {
            return vocabulary.get(word).getHuffmanNode().getIdx();
        } else
            return -1;
    }
1320590
@SuppressWarnings("unchecked")
    public static <T> T ifTrue(boolean _b, T _t) {
        return _b ? _t : (_t instanceof CharSequence ? (T) "" : null);
    }
1215496
public ResponseWrapper updateSensitiveWord(String newWord, String oldWord)
            throws APIConnectionException, APIRequestException {
        return _sensitiveWordClient.updateSensitiveWord(newWord, oldWord);
    }
656306
private int findRightBrace(char chLeft, char chRight, int fromIndex) {

        int level = 0; // 记录括号重叠级别。

        // 查找匹配的右括号。
        for (int index = fromIndex; index < length; index++) {

            char ch = pattern.charAt(index);

            // 如果出现左括号，重叠级别增加。
            if (ch == chLeft) {
                level++;
            }
            // 如果出现右括号，重叠级别降低。
            else if (ch == chRight) {

                if (level == 0) {
                    // 找到右括号。
                    return index;
                }

                level--;
            }
        }

        // 没有找到匹配的括号。
        return -1;
    }
81829
def splitword(self, originalword, *newwords, **kwargs):
        """"""
        if isstring(originalword):
            originalword = self.doc[u(originalword)]
        return self.correctwords([originalword], newwords, **kwargs)
610870
public static final String[] getKeywordValues(String keyword) {
        if (!keyword.equals(KEYWORDS[0])) {
            throw new IllegalArgumentException("Invalid keyword: " + keyword);
        }
        return ICUResourceBundle.getKeywordValues(BASE, RESOURCE);
    }