词性标注

@param wordList
@return
359395
public String[] partOfSpeechTag(List<String> wordList)
    {
        if (posTagger == null)
        {
            throw new IllegalStateException("未提供词性标注模型");
        }
        return tag(wordList);
    }
499142
public static ImmutableList<File> loadFileList(final File fileList) throws IOException {
    return loadFileList(Files.asCharSource(fileList, Charsets.UTF_8));
  }
466789
protected void segment(final String sentence, final String normalized, final List<String> wordList, final List<CoreDictionary.Attribute> attributeList)
    {
        if (attributeList != null)
        {
            final int[] offset = new int[]{0};
            CustomDictionary.parseLongestText(sentence, new AhoCorasickDoubleArrayTrie.IHit<CoreDictionary.Attribute>()
            {
                @Override
                public void hit(int begin, int end, CoreDictionary.Attribute value)
                {
                    if (begin != offset[0])
                    {
                        segmentAfterRule(sentence.substring(offset[0], begin), normalized.substring(offset[0], begin), wordList);
                    }
                    while (attributeList.size() < wordList.size())
                        attributeList.add(null);
                    wordList.add(sentence.substring(begin, end));
                    attributeList.add(value);
                    assert wordList.size() == attributeList.size() : "词语列表与属性列表不等长";
                    offset[0] = end;
                }
            });
            if (offset[0] != sentence.length())
            {
                segmentAfterRule(sentence.substring(offset[0]), normalized.substring(offset[0]), wordList);
            }
        }
        else
        {
            segmentAfterRule(sentence, normalized, wordList);
        }
    }
352322
public static Word combine(List<Word> words){
        if(words==null || words.size() < 2){
            return null;
        }
        String key = "";
        for(Word word : words){
            key += word.getText();
            key += " ";
        }
        key=key.trim();
        String value = GENERIC_TRIE.get(key);
        if(value==null){
            return null;
        }
        return new Word(value);
    }
874367
public static void filterStopWords(List<Word> words){
        Iterator<Word> iter = words.iterator();
        while(iter.hasNext()){
            Word word = iter.next();
            if(is(word.getText())){
                //去除停用词
                if(LOGGER.isDebugEnabled()) {
                    LOGGER.debug("去除停用词：" + word.getText());
                }
                iter.remove();
            }
        }
    }
42926
public String getTitles(List<String> titleList) throws Exception {
    String titles = "";
    String delim = "";
    for (String title : titleList) {
      titles = titles + delim + normalizeTitle(title);
      delim = "%7C";
    }
    return titles;
  }
1107991
public static List<Word> getWords(String text) {
        List<Word> result = new ArrayList<>();
        List<Word> words = WordParser.parse(text);
        for (Word word : words) {
            result.add(word);
        }
        return result;
    }
451723
def getWordList(ipFile, delim):
    
    indexedWords = {}
    totWords = 0
    totLines = 0
    with codecs.open(ipFile, "r",encoding='utf-8', errors='replace') as f:
        for line in f:
            totLines = totLines + 1
            words = multi_split(line, delim)
            totWords = totWords + len(words)
            for word in words:
                cleanedWord = word.lower().strip()
                if cleanedWord not in indexedWords:
                    indexedWords[cleanedWord] =  str(totLines)
                else:
                    indexedWords[cleanedWord] = indexedWords[cleanedWord] + ' ' + str(totLines)
    return totWords, totLines, indexedWords
1197947
public static void compileWithoutNS(List<IWord> wordList)
    {
        for (IWord word : wordList)
        {
            if (word.getLabel().startsWith("ns")) continue;
            word.setValue(PosTagCompiler.compile(word.getLabel(), word.getValue()));
//            switch (word.getLabel())
//            {
//                case "nx":
//                {
//                    word.setValue(Predefine.TAG_PROPER);
//                }
//                break;
//                case "nt":
//                case "ntc":
//                case "ntcf":
//                case "ntcb":
//                case "ntch":
//                case "nto":
//                case "ntu":
//                case "nts":
//                case "nth":
//                {
//                    word.setValue(Predefine.TAG_GROUP);
//                }
//                break;
//                case "m":
//                case "mq":
//                {
//                    word.setValue(Predefine.TAG_NUMBER);
//                }
//                break;
//                case "x":
//                {
//                    word.setValue(Predefine.TAG_CLUSTER);
//                }
//                break;
//                case "xx":
//                {
//                    word.setValue(Predefine.TAG_OTHER);
//                }
//                break;
//                case "t":
//                {
//                    word.setValue(Predefine.TAG_TIME);
//                }
//                break;
//                case "nr":
//                {
//                    word.setValue(Predefine.TAG_PEOPLE);
//                }
//                break;
//            }
        }
    }
250181
public List<VocabularyWord> words() {
        List<VocabularyWord> vocab = new ArrayList<>(vocabulary.values());
        Collections.sort(vocab, new Comparator<VocabularyWord>() {
            @Override
            public int compare(VocabularyWord o1, VocabularyWord o2) {
                return Integer.compare(o2.getCount(), o1.getCount());
            }
        });

        return vocab;
    }
1183707
static List<Node> nodeList2List(NodeList nl) {
        int len = nl.getLength();
        List<Node> l = new ArrayList<Node>(len);
        for (int i = 0; i < len; i++) {
            Node n = nl.item(i);
            if (!XMLUnit.getIgnoreComments() || !(n instanceof Comment)) {
                l.add(n);
            }
        }
        return l;
    }
956345
public WordForm getWordForms(int i) {
    if (Token_Type.featOkTst && ((Token_Type)jcasType).casFeat_wordForms == null)
      jcasType.jcas.throwFeatMissing("wordForms", "com.digitalpebble.rasp.Token");
    jcasType.jcas.checkArrayBounds(jcasType.ll_cas.ll_getRefValue(addr, ((Token_Type)jcasType).casFeatCode_wordForms), i);
    return (WordForm)(jcasType.ll_cas.ll_getFSForRef(jcasType.ll_cas.ll_getRefArrayValue(jcasType.ll_cas.ll_getRefValue(addr, ((Token_Type)jcasType).casFeatCode_wordForms), i)));}
876210
public List<List<Word>> getSimpleSentenceList()
    {
        List<List<Word>> simpleList = new LinkedList<List<Word>>();
        for (Sentence sentence : sentenceList)
        {
            List<Word> wordList = new LinkedList<Word>();
            for (IWord word : sentence.wordList)
            {
                if (word instanceof CompoundWord)
                {
                    for (Word inner : ((CompoundWord) word).innerList)
                    {
                        wordList.add(inner);
                    }
                }
                else
                {
                    wordList.add((Word) word);
                }
            }
            simpleList.add(wordList);
        }

        return simpleList;
    }
1332121
public static List<Window> windows(List<String> words, int windowSize) {

        List<Window> ret = new ArrayList<>();

        for (int i = 0; i < words.size(); i++)
            ret.add(windowForWordInPosition(windowSize, i, words));


        return ret;
    }
641315
public List<String> getMetaKeywordsForModule(ModuleElement mdle) {
        if (config.keywords) {
            return Arrays.asList(mdle.getQualifiedName() + " " + "module");
        } else {
            return Collections.emptyList();
        }
    }
1310470
public static String getFeatureListOutput(Collection<String> featureList) {
        StringBuffer sb = new StringBuffer();
        for (String output : featureList)
            sb.append(output + " ");
        return sb.toString().trim();
    }
260580
public static void parsePattern(List<NT> ntList, List<Vertex> vertexList, final WordNet wordNetOptimum, final WordNet wordNetAll)
    {
//        ListIterator<Vertex> listIterator = vertexList.listIterator();
        StringBuilder sbPattern = new StringBuilder(ntList.size());
        for (NT nt : ntList)
        {
            sbPattern.append(nt.toString());
        }
        String pattern = sbPattern.toString();
        final Vertex[] wordArray = vertexList.toArray(new Vertex[0]);
        trie.parseText(pattern, new AhoCorasickDoubleArrayTrie.IHit<String>()
        {
            @Override
            public void hit(int begin, int end, String keyword)
            {
                StringBuilder sbName = new StringBuilder();
                for (int i = begin; i < end; ++i)
                {
                    sbName.append(wordArray[i].realWord);
                }
                String name = sbName.toString();
                // 对一些bad case做出调整
                if (isBadCase(name)) return;

                // 正式算它是一个名字
                if (HanLP.Config.DEBUG)
                {
                    System.out.printf("识别出机构名：%s %s\n", name, keyword);
                }
                int offset = 0;
                for (int i = 0; i < begin; ++i)
                {
                    offset += wordArray[i].realWord.length();
                }
                wordNetOptimum.insert(offset, new Vertex(Predefine.TAG_GROUP, name, ATTRIBUTE, WORD_ID), wordNetAll);
            }
        });
    }