Get lemmas from a tokenized and pos tagged sentence.

@param tokens
the tokenized sentence
@param posTags the pos tags of the sentence
@return a list of {@code Morpheme} objects containing morphological info
14355
public final List<Morpheme> getMorphemes(final String[] tokens, final String[] posTags) {
    final List<String> lemmas = lemmatize(tokens, posTags);
    final List<Morpheme> morphemes = getMorphemesFromStrings(tokens, posTags, lemmas);
    return morphemes;
  }
1275381
private boolean isPosTokenMatched(AnalyzedToken token) {
    if (posToken == null || posToken.posTag == null) {
      // if no POS set defaulting to true
      return true;
    }
    if (token.getPOSTag() == null) {
      return posToken.posUnknown && token.hasNoTag();
    }
    boolean match;
    if (posToken.regExp) {
      Matcher mPos = posToken.posPattern.matcher(token.getPOSTag());
      match = mPos.matches();
    } else {
      match = posToken.posTag.equals(token.getPOSTag());
    }
    if (!match && posToken.posUnknown) { // ignore helper tags
      match = token.hasNoTag();
    }
    return match;
  }
197020
private Collection<AnnotationData> getAnnotationSet(final Collection<TagAndLength> pTags) {
		Collection<AnnotationData> ret = null;
		if (pTags != null) {
			Map<ITag, AnnotationData> data = AnnotationUtils.getInstance().getMap().get(getClass().getName());
			ret = new ArrayList<AnnotationData>(data.size());
			for (TagAndLength tal : pTags) {
				AnnotationData ann = data.get(tal.getTag());
				if (ann != null) {
					ann.setSize(tal.getLength() * BitUtils.BYTE_SIZE);
				} else {
					ann = new AnnotationData();
					ann.setSkip(true);
					ann.setSize(tal.getLength() * BitUtils.BYTE_SIZE);
				}
				ret.add(ann);
			}
		} else {
			ret = AnnotationUtils.getInstance().getMapSet().get(getClass().getName());
		}
		return ret;
	}
1201200
public GrammaticalFeats getFeats() {
    if (Token_Type.featOkTst && ((Token_Type)jcasType).casFeat_feats == null)
      jcasType.jcas.throwFeatMissing("feats", "de.julielab.jules.types.Token");
    return (GrammaticalFeats)(jcasType.ll_cas.ll_getFSForRef(jcasType.ll_cas.ll_getRefValue(addr, ((Token_Type)jcasType).casFeatCode_feats)));}
1299636
public final String[][] getAllPosTags(final String[] tokens) {
    final String[][] allPosTags = this.posTagger.tag(13, tokens);
    return allPosTags;
  }
1108873
public static Set<String> getPosTagVocabulary(Iterable<? extends CcgExample> examples) {
    Set<String> posTagVocabulary = Sets.newHashSet();
    for (CcgExample example : examples) {
      posTagVocabulary.addAll(example.getSentence().getPosTags());
    }
    return posTagVocabulary;
  }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
422010
def lemmatize(self, tokens):
        """"""
        entries = self.entries
        forms_and_lemmas = self.forms_and_lemmas

        lemma_list = [x[0] for x in entries]
        """Provide a lemma for each token"""
        lemmatized = []
        for token in tokens:
            """check for a match between token and list of lemmas"""
            if token in lemma_list:
                lemmed = (token, token)
                lemmatized.append(lemmed)
            else:
                """if no match check for a match between token and list of lemma forms"""
                lemma = [k for k, v in forms_and_lemmas.items() if token in v]
                if lemma != []:
                    lemmed = (token, lemma)
                    lemmatized.append(lemmed)
                elif lemma == []:
                    """if no match apply regular expressions and check for a match against the list of lemmas again"""
                    regexed = regex(token)
                    if regexed in lemma_list:
                        lemmed = (token, regexed)
                        lemmatized.append(lemmed)
                    else:
                        lemmed = (token, "None")
                        lemmatized.append(lemmed)
        return lemmatized
361882
def get_lemmas(self, word, pos=None, ignore_case=False):
        
        entries = self.get_entries(word, pos, ignore_case)
        lemmas = list(set([entry["Lemma"] for entry in entries]))
        return sorted(lemmas)
283607
private List<String> getDictKeys(final String word, final String postag) {
    final List<String> keys = new ArrayList<String>();
    keys.addAll(Arrays.asList(word.toLowerCase(), postag));
    return keys;
  }
420562
public String posTagging(String text){
		String ret = text;
		if (vnPosTagger != null){
			ret = vnPosTagger.tagging(text);			
		}
		
		return ret;
	}
1262686
public void tagPOS(List<CoreLabel> tokens, Tree tree) {
        try {
            List<TaggedWord> posList = tree.getChild(0).taggedYield();
            for (int i = 0; i < tokens.size(); i++) {
                String pos = posList.get(i).tag();
                tokens.get(i).setTag(pos);
            }
        } catch (Exception e) {
            tagPOS(tokens); // At least gives you something.
            LOG.warn("POS Failed:\n" + tree.pennString());
        }
    }
535742
@Override
  public String[] synthesize(AnalyzedToken token, String posTag) throws IOException {
    List<String> wordForms = new ArrayList<>();
    lookup(token.getLemma(), posTag, wordForms);
    return wordForms.toArray(new String[0]);
  }
766981
private String getPatternCode(final List<Token> tokens) {
        final StringBuilder sb = new StringBuilder();
        sb.append(tokens.size());
        for (final Token token : tokens) {
            sb.append(token.getType().ordinal());
        }
        return sb.toString();
    }
262274
public static HMMPOSTagger learn(String[][] sentences, PennTreebankPOS[][] labels) {
        int index = 1;
        int suffixIndex = 0;
        Map<String, Integer> symbol = new HashMap<>();
        Map<String, Integer> suffix = new HashMap<>();
        for (int i = 0; i < sentences.length; i++) {
            for (int j = 0; j < sentences[i].length; j++) {
                Integer sym = symbol.get(sentences[i][j]);
                if (sym == null) {
                    symbol.put(sentences[i][j], index++);
                }
                
                if (sentences[i][j].length() > 2) {
                    String s = sentences[i][j].substring(sentences[i][j].length() - 2);
                    sym = suffix.get(s);
                    if (sym == null) {
                        suffix.put(s, suffixIndex++);
                    }
                }
            }
        }

        int numStates = PennTreebankPOS.values().length;

        double[] pi = new double[numStates];
        double[][] a = new double[numStates][numStates];
        double[][] b = new double[numStates][symbol.size() + 1];
        double[][] c = new double[numStates][suffix.size()];
        
        PennTreebankPOS[] tags = PennTreebankPOS.values();
        for (int i = 0; i < numStates; i++) {
            if (tags[i].open) {
                b[i][0] = 1.0;
            }
        }

        for (int i = 0; i < sentences.length; i++) {
            int[] tag = translate(labels[i]);
            int[][] obs = translate(symbol, suffix, sentences[i]);

            pi[tag[0]]++;
            b[tag[0]][obs[0][0]]++;            
            if (obs[0][1] >= 0) {
                c[tag[0]][obs[0][1]]++;
            }
            
            for (int j = 1; j < obs.length; j++) {
                a[tag[j-1]][tag[j]]++;
                b[tag[j]][obs[j][0]]++;
                if (obs[j][1] >= 0) {
                    c[tag[j]][obs[j][1]]++;
                }
            }                
        }

        Math.unitize1(pi);
        for (int i = 0; i < numStates; i++) {
            Math.unitize1(a[i]);
            Math.unitize1(b[i]);
            Math.unitize1(c[i]);
        }
        
        return new HMMPOSTagger(symbol, suffix, pi, a, b, c);
    }
840903
protected List<String> getTokens(String text){
		String[] tokens = text.toLowerCase().split("[\\s\\p{Punct}]");
		List<String> tokenList = new ArrayList<String>(tokens.length);
		for (int i = 0; i < tokens.length; i++){
			if (tokens[i].trim().length() < 1){
				continue;
			}
			tokenList.add(tokens[i].trim());
		}
		return tokenList;
	}
84198
public WordLemmaTag lemmatize(WordTag wT) {
    String tag = wT.tag();
    String word = wT.word();
    String lemma = lemma(word, tag);
    return new WordLemmaTag(word, lemma, tag);
  }