/*
Tag substantivated adjectives and participles, which are currently tagged not tagged correctly
(e.g., "Verletzter" in "Ein Verletzter kam ins Krankenhaus" needs to be tagged as "SUB:NOM:SIN:MAS")
@param word to be checked
358248
private List<AnalyzedToken> getSubstantivatedForms(String word, List<String> sentenceTokens, int pos) {
    if (word.endsWith("er")) {
      List<TaggedWord> lowerCaseTags = getWordTagger().tag(word.toLowerCase());
      // do not add tag words whose lower case variant is an adverb (e.g, "Früher") to avoid false negatives for DE_CASE
      if (lowerCaseTags.stream().anyMatch(t -> t.getPosTag().startsWith("ADV"))) {
        return null;
      }
      int idx = sentenceTokens.indexOf(word);
      // is followed by an uppercase word? If 'yes', the word is probably not substantivated
      while (++idx < sentenceTokens.size()) {
        String nextWord = sentenceTokens.get(idx);
        if (StringUtils.isWhitespace(nextWord)) {
          continue;
        }
        if (nextWord.length() > 0 && (Character.isUpperCase(nextWord.charAt(0)) || "als".equals(nextWord))) {
          return null;
        }
        break;
      }
      String femaleForm = word.substring(0, word.length()-1);
      List<TaggedWord> taggedFemaleForm = getWordTagger().tag(femaleForm);
      boolean isSubstantivatedForm = taggedFemaleForm.stream().anyMatch(t -> t.getPosTag().equals("SUB:NOM:SIN:FEM:ADJ"));
      if (isSubstantivatedForm) {
        List<AnalyzedToken> list = new ArrayList<>();
        list.add(new AnalyzedToken(word, "SUB:NOM:SIN:MAS:ADJ", word));
        list.add(new AnalyzedToken(word, "SUB:GEN:PLU:MAS:ADJ", word));
        //list.add(new AnalyzedToken(word, "SUB:NOM:SIN:MAS", word));
        return list;
      }
    }
    return null;
  }
617697
public void add(IWord word)
    {
        //check and extends the entity from the base word
        if ( word.getEntity() == null ) {
            word.setEntity(rootWord.getEntity());
        }
        
        //check and extends the part of speech from the base word
        if ( word.getPartSpeech() == null ) {
            word.setPartSpeech(rootWord.getPartSpeech());
        }
        
        word.setSyn(this);
        synsList.add(word);
    }
840924
@Nullable
  private String getParticipleSuggestion(String word) {
    if (word.startsWith("ge") && word.endsWith("t")) {
      // strip leading "ge" and replace trailing "t" with "en":
      String baseform = word.substring(2, word.length()-1) + "en";
      try {
        String participle = getParticipleForBaseform(baseform);
        if (participle != null) {
          return participle;
        }
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
    return null;
  }
266630
private static String mapWotanTagSetToNAF(final String postag) {
    if (postag.equalsIgnoreCase("ADV")) {
      return "A"; // adverb
    } else if (postag.equalsIgnoreCase("CONJ")
        || postag.equalsIgnoreCase("SCONJ")) {
      return "C"; // conjunction
    } else if (postag.equalsIgnoreCase("Art")) {
      return "D"; // det predeterminer
    } else if (postag.equalsIgnoreCase("ADJ")) {
      return "G"; // adjective
    } else if (postag.equalsIgnoreCase("N")) {
      return "N"; // common noun
    } else if (postag.equalsIgnoreCase("PROPN")) {
      return "R"; // proper noun
    } else if (postag.equalsIgnoreCase("Prep")) {
      return "P"; // preposition
    } else if (postag.equalsIgnoreCase("PRON")) {
      return "Q"; // pronoun
    } else if (postag.startsWith("V")) {
      return "V"; // verb
    } else {
      return "O"; // other
    }
  }
1245149
public void incrementWordCounter(String word) {
        if (vocabulary.containsKey(word)) {
            vocabulary.get(word).incrementCount();
        }
        // there's no need to throw such exception here. just do nothing if word is not found
        //else throw new IllegalStateException("No such word found");
    }
1335654
public final Morpheme createMorpheme(final String word, final String tag) {
    final Morpheme morpheme = new Morpheme();
    morpheme.setValue(word);
    morpheme.setTag(tag);
    return morpheme;
  }
499465
public void add(String word) {
		if (word.trim().length() > 0)
			m_Words.add(word.trim().toLowerCase());
	}
1166433
@Override
  protected void addProhibitedWords(List<String> words) {
    if(words.size() == 1 && words.get(0).endsWith(".*")) {
      wordStartsToBeProhibited.add(words.get(0).substring(0, words.get(0).length()-2));
    } else {
      super.addProhibitedWords(words);
    }
  }
612431
def analysis(self):
        """"""
        if not self.is_tagged(ANALYSIS):
            self.tag_analysis()
        return [word[ANALYSIS] for word in self.words]
704934
private static String lemmatize(String word, String tag, Morpha lexer, boolean lowercase) {
    boolean wordHasForbiddenChar = word.indexOf('_') >= 0 ||word.indexOf(' ') >= 0;
    String quotedWord = word;
    if (wordHasForbiddenChar) {
      // choose something unlikely. Classical Vedic!
      quotedWord = quotedWord.replaceAll("_", "\u1CF0");
      quotedWord = quotedWord.replaceAll(" ", "\u1CF1");
    }
    String wordtag = quotedWord + '_' + tag;
    if (DEBUG) System.err.println("Trying to normalize |" + wordtag + "|");
    try {
      lexer.setOption(1, lowercase);
      lexer.yyreset(new StringReader(wordtag));
      lexer.yybegin(Morpha.scan);
      String wordRes = lexer.next();
      lexer.next(); // go past tag
      if (wordHasForbiddenChar) {
        if (DEBUG) System.err.println("Restoring forbidden chars");
        wordRes = wordRes.replaceAll("\u1CF0", "_");
        wordRes = wordRes.replaceAll("\u1CF1", " ");
      }
      return wordRes;
    } catch (IOException e) {
      LOGGER.warning("Morphology.stem() had error on word " + word + "/" + tag);
      return word;
    }
  }
482678
@Override
    public void putVector(String word, INDArray vector) {
        if (word == null)
            throw new IllegalArgumentException("No null words allowed");
        if (vector == null)
            throw new IllegalArgumentException("No null vectors allowed");
        int idx = vocab.indexOf(word);
        syn0.slice(idx).assign(vector);

    }
453401
@Override
    public String[] tag(String... words)
    {
        POSInstance instance = new POSInstance(words, model.featureMap);
        return tag(instance);
    }
820206
@Override
    public int docAppearedIn(String word) {
        T element = extendedVocabulary.get(word);
        if (element != null) {
            return (int) element.getSequencesCount();
        } else
            return -1;
    }
489772
private function isReservedWord($word)
    {
        if ($word[0] === '`') {
            $word = str_replace('`', '', $word);
        }

        $keywordLists = [];
        foreach ($this->keywordLists as $keywordList) {
            if (! $keywordList->isKeyword($word)) {
                continue;
            }

            $keywordLists[] = $keywordList->getName();
        }

        return $keywordLists;
    }
828989
def add_word(self, word, freq=None, tag=None):
        
        self.check_initialized()
        word = strdecode(word)
        freq = int(freq) if freq is not None else self.suggest_freq(word, False)
        self.FREQ[word] = freq
        self.total += freq
        if tag:
            self.user_word_tag_tab[word] = tag
        for ch in xrange(len(word)):
            wfrag = word[:ch + 1]
            if wfrag not in self.FREQ:
                self.FREQ[wfrag] = 0
        if freq == 0:
            finalseg.add_force_split(word)
52072
private boolean acceptWord(String word) {
        return !word.equals(EMPTY_STRING) && 
               (semanticFilter.isEmpty() || semanticFilter.contains(word));
    }
965043
public static PennTreebankPOS tag(String word) {
        for (int i = 0; i < REGEX.length; i++) {
            if (REGEX[i].matcher(word).matches()) {
                return REGEX_POS[i];
            }
        }

        return null;
    }