将word列表转为兼容的IWord列表

@param simpleSentenceList
@return
163279
public static List<List<IWord>> convert2CompatibleList(List<List<Word>> simpleSentenceList)
    {
        List<List<IWord>> compatibleList = new LinkedList<List<IWord>>();
        for (List<Word> wordList : simpleSentenceList)
        {
            compatibleList.add(new LinkedList<IWord>(wordList));
        }
        return compatibleList;
    }
84788
public List<String> getWords() {
        List<String> result = new ArrayList<>();
        List<Word> words = WordParser.parse(title + snippet);
        for (Word word : words) {
            result.add(word.getText());
        }
        return result;
    }
275434
public String toSentenceString(ArrayList s) {
    StringBuilder sb = new StringBuilder();
    for (int wordNum = start(), end = end(); wordNum <= end; wordNum++) {
      sb.append(s.get(wordNum));
      if (wordNum != end) {
        sb.append(" ");
      }
    }
    return sb.toString();
  }
1246448
private IWord _nextNumberWord(IWord word) throws IOException
    {
        IWord dWord = null, unit = null;
        IWord dw1 = super.next();
        if ( dw1 == null ) {
            return null;
        }

        String[] w1Entity = dw1.getEntity();
        int w1Len = dw1.getValue().length();

        //pure numeric
        if ( ArrayUtil.indexOf(Entity.E_NUMERIC_ARABIC, w1Entity) > -1 ) {
            IWord dw2 = super.next();
            if ( dw2 == null ) {
                dWord = new Word(word.getValue()+dw1.getValue(), IWord.T_CJK_WORD);
                dWord.setPosition(word.getPosition());
                dWord.setEntity(Entity.E_THE_NUMBER_A);
                return dWord;
            }

            int w2Len = dw2.getValue().length();

            //template like '个', 'xx个'
            unit = dic.get(ILexicon.NUMBER_UNIT, ""+dw2.getValue().charAt(0));
            if ( unit != null ) {
                String w2Val = dw2.getValue();
                if ( w2Len == 1 || NumericUtil.isCNNumericString(w2Val, 0, w2Len-1)
                        || StringUtil.isDigit(w2Val, 0, w2Len-1) ) {
                    dWord = new Word(word.getValue()+dw1.getValue()+w2Val, IWord.T_CJK_WORD);
                    dWord.setEntity(unit.getEntity());
                    return dWord;
                }
            }

            eWordPool.push(dw2);
            dWord = new Word(word.getValue()+dw1.getValue(), IWord.T_CJK_WORD);
            dWord.setPosition(word.getPosition());
            dWord.setEntity(Entity.E_THE_NUMBER_A);
            return dWord;
        }

        //template like '个', 'x个'
        unit = dic.get(ILexicon.NUMBER_UNIT, ""+dw1.getValue().charAt(w1Len-1));
        if ( unit != null ) {
            String w1Val = dw1.getValue();
            if ( w1Len == 1 || NumericUtil.isCNNumericString(w1Val, 0, w1Len-1)
                    || StringUtil.isDigit(w1Val, 0, w1Len-1) ) {
                dWord = new Word(word.getValue()+w1Val, IWord.T_CJK_WORD);
                dWord.setEntity(unit.getEntity());
                return dWord;
            }
        }

        eWordPool.push(dw1);
        return null;
    }
315072
public ResponseWrapper addSensitiveWords(String...words) throws APIConnectionException, APIRequestException {
        JsonArray array = new JsonArray();
        for (String word: words) {
            Preconditions.checkArgument(word.length() <= 10, "one word's max length is 10");
            array.add(new JsonPrimitive(word));
        }
        return _httpClient.sendPost(_baseUrl + sensitiveWordPath, array.toString());
    }
38184
public List<List<Term>> seg2sentence(String text, boolean shortest)
    {
        List<List<Term>> resultList = new LinkedList<List<Term>>();
        {
            for (String sentence : SentencesUtil.toSentenceList(text, shortest))
            {
                resultList.add(segSentence(sentence.toCharArray()));
            }
        }

        return resultList;
    }
927205
protected String[] getRow(String masterLanguage, ITextNode textNode, IValueNode valueNode) {
    List<String> row = new ArrayList<>();

    row.add(textNode.getKey());
    row.add(valueNode.getStatus().getName());
    if (!valueNode.getLanguage().equals(masterLanguage)) {
      IValueNode masterValueNode = textNode.getValueNode(masterLanguage);
      if (masterValueNode == null) {
        row.add("");
      } else {
        row.add(masterValueNode.getValue());
      }
    }
    row.add(valueNode.getValue());
    row.add(textNode.getContext());

    return row.toArray(new String[row.size()]);
  }
1045164
public Collection<String> wordsNearest(String word, int n) {
        return modelUtils.wordsNearest(word, n);
    }
171531
@Override
    public int wordFrequency(@NonNull String word) {
        // TODO: proper wordFrequency impl should return long, instead of int
        T element = extendedVocabulary.get(word);
        if (element != null)
            return (int) element.getElementFrequency();
        return 0;
    }
882584
public static List<String> toSentenceList(String content, boolean shortest)
    {
        return toSentenceList(content.toCharArray(), shortest);
    }
224247
public IWord next() throws IOException
    {
        if ( eWordPool.size() > 0 ) {
            return eWordPool.removeFirst();
        }

        IWord word = super.next();
        if ( word == null ) {
            return null;
        }

        String[] entity = word.getEntity();

        /*
         * This is a temporary program for
         * the_number or the_episode entity recognition.
         * ignore this if the entity is not null
         * added at 2017/05/25
        */
        if ( entity == null && word.getValue().charAt(0) == '第' ) {
            IWord dWord = getNextTheWord(word);
            if ( dWord != null ) {
                dWord.setPartSpeech(IWord.QUANTIFIER);
                return dWord;
            }
        }

        if ( entity == null ) {
            return word;
        }

        int eIdx = 0;
        if ( (eIdx = ArrayUtil.startsWith("time.a", entity)) > -1
        		|| (eIdx = ArrayUtil.startsWith(
        				Entity.E_DATETIME_P, entity)) > -1 ) {
            IWord dWord = getNextTimeMergedWord(word, eIdx);
            if ( dWord != null ) {
                return dWord;
            }
        }

        if ( (eIdx = ArrayUtil.startsWith("datetime.ymd", entity)) > -1 ) {
            IWord dWord = getNextDatetimeWord(word, eIdx);
            if ( dWord != null ) {
                return dWord;
            }
        }

        return word;
    }
438846
@Nonnull
    public <T> Word<T> transform(Function<? super I, ? extends T> transformer) {
        int len = length();
        Object[] array = new Object[len];
        int i = 0;
        for (I symbol : this) {
            array[i++] = transformer.apply(symbol);
        }
        return new SharedWord<>(array);
    }
17324
public AnalyzedSentence getRawAnalyzedSentence(String sentence) throws IOException {
    List<String> tokens = language.getWordTokenizer().tokenize(sentence);
    Map<Integer, String> softHyphenTokens = replaceSoftHyphens(tokens);

    List<AnalyzedTokenReadings> aTokens = language.getTagger().tag(tokens);
    if (language.getChunker() != null) {
      language.getChunker().addChunkTags(aTokens);
    }
    int numTokens = aTokens.size();
    int posFix = 0; 
    for (int i = 1; i < numTokens; i++) {
      aTokens.get(i).setWhitespaceBefore(aTokens.get(i - 1).isWhitespace());
      aTokens.get(i).setStartPos(aTokens.get(i).getStartPos() + posFix);
      if (!softHyphenTokens.isEmpty() && softHyphenTokens.get(i) != null) {
        aTokens.get(i).addReading(language.getTagger().createToken(softHyphenTokens.get(i), null));
        posFix += softHyphenTokens.get(i).length() - aTokens.get(i).getToken().length();
      }
    }
        
    AnalyzedTokenReadings[] tokenArray = new AnalyzedTokenReadings[tokens.size() + 1];
    AnalyzedToken[] startTokenArray = new AnalyzedToken[1];
    int toArrayCount = 0;
    AnalyzedToken sentenceStartToken = new AnalyzedToken("", SENTENCE_START_TAGNAME, null);
    startTokenArray[0] = sentenceStartToken;
    tokenArray[toArrayCount++] = new AnalyzedTokenReadings(startTokenArray, 0);
    int startPos = 0;
    for (AnalyzedTokenReadings posTag : aTokens) {
      posTag.setStartPos(startPos);
      tokenArray[toArrayCount++] = posTag;
      startPos += posTag.getToken().length();
    }

    // add additional tags
    int lastToken = toArrayCount - 1;
    // make SENT_END appear at last not whitespace token
    for (int i = 0; i < toArrayCount - 1; i++) {
      if (!tokenArray[lastToken - i].isWhitespace()) {
        lastToken -= i;
        break;
      }
    }

    tokenArray[lastToken].setSentEnd();

    if (tokenArray.length == lastToken + 1 && tokenArray[lastToken].isLinebreak()) {
      tokenArray[lastToken].setParagraphEnd();
    }
    return new AnalyzedSentence(tokenArray);
  }
378572
@Override
    public IWordSerializer getSerializer(HLLType type, int wordLength, int wordCount) {
        return new BigEndianAscendingWordSerializer(wordLength, wordCount, paddingBytes(type));
    }
580262
public boolean isStopWord(String word,int minLen, int maxLen) {
		if (word.length() < minLen || word.length()>maxLen)
			return true;

		if (noise.matcher(word).matches())
			return true;

		if (sWord.contains(word))
			return true;

		return false;
	}
134179
protected String getPairPunctuationText( int c ) throws IOException 
    {
        //StringBuilder isb = new StringBuilder();
        isb.clear();
        char echar = StringUtil.getPunctuationPair( (char) c);
        boolean matched = false;
        int j, ch;
        
        //replaced with IntArrayList at 2013-09-08
        //ArrayList<Integer> chArr = new ArrayList<Integer>(config.PPT_MAX_LENGTH);
        ialist.clear();
        
        for ( j = 0; j < config.PPT_MAX_LENGTH; j++ ) {
            ch = readNext();
            if ( ch == -1 ) break;
            if ( ch == echar ) {
                matched = true;
                pushBack(ch);        //push the pair punc back.
                break;
            }
            
            isb.append( (char) ch );
            ialist.add(ch);
        }
        
        if ( matched == false ) {
            for ( int i = j - 1; i >= 0; i-- ) 
                pushBack( ialist.get(i) );
            return null;
        }
        
        return isb.toString();
    }
    
    /**
     * an abstract method to gain a CJK word from the 
     * current position. simpleSeg and ComplexSeg is different to deal this, 
     * so make it a abstract method here
     * 
     * @param  chars
     * @param  index
     * @return IChunk
     * @throws IOException
     */
    protected abstract IChunk getBestCJKChunk(char chars[], int index) throws IOException;
    
}
1105292
private List<Word> getRecognizedWords(int pageIteratorLevel) {
        List<Word> words = new ArrayList<Word>();

        try {
            TessResultIterator ri = api.TessBaseAPIGetIterator(handle);
            TessPageIterator pi = api.TessResultIteratorGetPageIterator(ri);
            api.TessPageIteratorBegin(pi);

            do {
                Pointer ptr = api.TessResultIteratorGetUTF8Text(ri, pageIteratorLevel);
                String text = ptr.getString(0);
                api.TessDeleteText(ptr);
                float confidence = api.TessResultIteratorConfidence(ri, pageIteratorLevel);
                IntBuffer leftB = IntBuffer.allocate(1);
                IntBuffer topB = IntBuffer.allocate(1);
                IntBuffer rightB = IntBuffer.allocate(1);
                IntBuffer bottomB = IntBuffer.allocate(1);
                api.TessPageIteratorBoundingBox(pi, pageIteratorLevel, leftB, topB, rightB, bottomB);
                int left = leftB.get();
                int top = topB.get();
                int right = rightB.get();
                int bottom = bottomB.get();
                Word word = new Word(text, confidence, new Rectangle(left, top, right - left, bottom - top));
                words.add(word);
            } while (api.TessPageIteratorNext(pi, pageIteratorLevel) == TRUE);
        } catch (Exception e) {
            logger.warn(e.getMessage(), e);
        }

        return words;
    }