Stem the words of the sentence Not used in the example, because Wordnet cannot recognise all stems
356610
function stem_sentence(sentence) {
  for (var i = 0; i < sentence.length; i++) {
    sentence[i] = natural.PorterStemmer.stem(sentence[i]);
  }
  logger.info("stem_sentence: " + sentence);
  return(sentence);
}
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
284019
def stems_for(words)
      stemmer = Lingua::Stemmer.new(language: 'en')
      words.map! { |word| stemmer.stem(word) }
    end
917337
public static String stem(String s) {
    String stem = s;

     // Handle irregular ones
     String irreg=irregular.get(s);
     if(irreg!=null) return(stem=irreg);

     // -on to -a
     if(categoryON_A.contains(s)) return(stem=cut(s,"a")+"on");

     // -um to -a
     if(categoryUM_A.contains(s)) return(stem=cut(s,"a")+"um");

     // -x to -ices
     if(categoryIX_ICES.contains(s)) return(stem=cut(s,"ices")+"ix");

     // -o to -i
     if(categoryO_I.contains(s)) return(stem=cut(s,"i")+"o");

     // -se to ses
     if(categorySE_SES.contains(s)) return(stem=cut(s,"s"));

     // -is to -es
     if(categoryIS_ES.contains(s) || s.endsWith("theses")) return(stem=cut(s,"es")+"is");

     // -us to -i
     if(categoryUS_I.contains(s)) return(stem=cut(s,"i")+"us");
     //Wrong plural
     if(s.endsWith("uses") && (categoryUS_I.contains(cut(s,"uses")+"i") ||
                               s.equals("genuses") || s.equals("corpuses"))) return(stem=cut(s,"es"));

     // -ex to -ices
     if(categoryEX_ICES.contains(s)) return(stem=cut(s,"ices")+"ex");

     // Words that do not inflect in the plural
     if(s.endsWith("ois") || s.endsWith("itis") || category00.contains(s) || categoryICS.contains(s)) return(stem=s);

     // -en to -ina
     // No other common words end in -ina
     if(s.endsWith("ina")) return(stem=cut(s,"en"));

     // -a to -ae
     // No other common words end in -ae
     if(s.endsWith("ae")) return(stem=cut(s,"e"));

     // -a to -ata
     // No other common words end in -ata
     if(s.endsWith("ata")) return(stem=cut(s,"ta"));

     // trix to -trices
     // No common word ends with -trice(s)
     if(s.endsWith("trices")) return(stem=cut(s,"trices")+"trix");

     // -us to -us
     //No other common word ends in -us, except for false plurals of French words
     //Catch words that are not latin or known to end in -u
     if(s.endsWith("us") && !s.endsWith("eaus") && !s.endsWith("ieus") && !noLatin(s)
        && !categoryU_US.contains(s)) return(stem=s);

     // -tooth to -teeth
     // -goose to -geese
     // -foot to -feet
     // -zoon to -zoa
     //No other common words end with the indicated suffixes
     if(s.endsWith("teeth")) return(stem=cut(s,"teeth")+"tooth");
     if(s.endsWith("geese")) return(stem=cut(s,"geese")+"goose");
     if(s.endsWith("feet")) return(stem=cut(s,"feet")+"foot");
     if(s.endsWith("zoa")) return(stem=cut(s,"zoa")+"zoon");

     // -eau to -eaux
     //No other common words end in eaux
     if(s.endsWith("eaux")) return(stem=cut(s,"x"));

     // -ieu to -ieux
     //No other common words end in ieux
     if(s.endsWith("ieux")) return(stem=cut(s,"x"));

     // -nx to -nges
     // Pay attention not to kill words ending in -nge with plural -nges
     // Take only Greek words (works fine, only a handfull of exceptions)
     if(s.endsWith("nges") && greek(s)) return(stem=cut(s,"nges")+"nx");

     // -[sc]h to -[sc]hes
     //No other common word ends with "shes", "ches" or "she(s)"
     //Quite a lot end with "che(s)", filter them out
     if(s.endsWith("shes") || s.endsWith("ches") && !categoryCHE_CHES.contains(s)) return(stem=cut(s,"es"));

     // -ss to -sses
     // No other common singular word ends with "sses"
     // Filter out those ending in "sse(s)"
     if(s.endsWith("sses") && !categorySSE_SSES.contains(s) && !s.endsWith("mousses")) return(stem=cut(s,"es"));

     // -x to -xes
     // No other common word ends with "xe(s)" except for "axe"
     if(s.endsWith("xes") && !s.equals("axes")) return(stem=cut(s,"es"));

     // -[nlw]ife to -[nlw]ives
     //No other common word ends with "[nlw]ive(s)" except for olive
     if(s.endsWith("nives") || s.endsWith("lives") && !s.endsWith("olives") ||
        s.endsWith("wives")) return(stem=cut(s,"ves")+"fe");

     // -[aeo]lf to -ves  exceptions: valve, solve
     // -[^d]eaf to -ves  exceptions: heave, weave
     // -arf to -ves      no exception
     if(s.endsWith("alves") && !s.endsWith("valves") ||
        s.endsWith("olves") && !s.endsWith("solves") ||
        s.endsWith("eaves") && !s.endsWith("heaves") && !s.endsWith("weaves") ||
        s.endsWith("arves") ) return(stem=cut(s,"ves")+"f");

     // -y to -ies
     // -ies is very uncommon as a singular suffix
     // but -ie is quite common, filter them out
     if(s.endsWith("ies") && !categoryIE_IES.contains(s)) return(stem=cut(s,"ies")+"y");

     // -o to -oes
     // Some words end with -oe, so don't kill the "e"
     if(s.endsWith("oes") && !categoryOE_OES.contains(s)) return(stem=cut(s,"es"));

     // -s to -ses
     // -z to -zes
     // no words end with "-ses" or "-zes" in singular
     if(s.endsWith("ses") || s.endsWith("zes") ) return(stem=cut(s,"es"));

     // - to -s
     if(s.endsWith("s") && !s.endsWith("ss") && !s.endsWith("is")) return(stem=cut(s,"s"));

     return stem;
  }
1325896
def negated(input_words, include_nt=True):
    
    input_words = [str(w).lower() for w in input_words]
    neg_words = []
    neg_words.extend(NEGATE)
    for word in neg_words:
        if word in input_words:
            return True
    if include_nt:
        for word in input_words:
            if "n't" in word:
                return True
    if "least" in input_words:
        i = input_words.index("least")
        if i > 0 and input_words[i - 1] != "at":
            return True
    return False
401666
def stem(self):

        

        def s(tokens):
            return [PorterStemmer().stem(t) for t in tokens]
        self.stems = list(map(s, self.tokens))
916127
def get_stem(self, noun, gender, mimation=True):
        """"""
        stem = ''
        if mimation and noun[-1:] == 'm':
            # noun = noun[:-1]
            pass
        # Take off ending
        if gender == 'm':
            if noun[-2:] in list(self.endings['m']['singular'].values()) + \
                    list(self.endings['m']['dual'].values()):
                stem = noun[:-2]
            elif noun[-1] in list(self.endings['m']['plural'].values()):
                stem = noun[:-1]
            else:
                print("Unknown masculine noun: {}".format(noun))
        elif gender == 'f':
            if noun[-4:] in self.endings['f']['plural']['nominative'] + \
                    self.endings['f']['plural']['oblique']:
                stem = noun[:-4] + 't'
            elif noun[-3:] in list(self.endings['f']['singular'].values()) + \
                    list(self.endings['f']['dual'].values()):
                stem = noun[:-3] + 't'
            elif noun[-2:] in list(self.endings['m']['singular'].values()) + \
                    list(self.endings['m']['dual'].values()):
                stem = noun[:-2]
            else:
                print("Unknown feminine noun: {}".format(noun))
        else:
            print("Unknown noun: {}".format(noun))
        return stem
615801
def stem_singular_word(self, word):
        """"""
        context = Context(word, self.dictionary, self.visitor_provider)
        context.execute()

        return context.result
140952
def _create_sentence_objects(self):
        """"""
        sentence_objects = []
        sentences = sent_tokenize(self.raw, tokenizer=self.tokenizer)
        char_index = 0  # Keeps track of character index within the blob
        for sent in sentences:
            # Compute the start and end indices of the sentence
            # within the blob. This only works if the sentence splitter
            # does not perform any character replacements or changes to
            # white space.
            # Working: NLTKPunktTokenizer
            # Not working: PatternTokenizer
            try:
                start_index = self.raw.index(sent, char_index)
                char_index += len(sent)
                end_index = start_index + len(sent)
            except ValueError:
                start_index = None
                end_index = None
            # Sentences share the same models as their parent blob
            s = Sentence(
                sent,
                start_index=start_index,
                end_index=end_index,
                tokenizer=self.tokenizer,
                np_extractor=self.np_extractor,
                pos_tagger=self.pos_tagger,
                analyzer=self.analyzer,
                parser=self.parser,
                classifier=self.classifier)
            sentence_objects.append(s)
        return sentence_objects
298278
def tokenize(self, sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_type, bool, bool, bool, bool, Callable[[text_type], text_type]) -> Union[TokenizedSenetence, List[text_type]]
        
        if normalize:
            normalized_sentence = func_normalizer(sentence)
        else:
            normalized_sentence = sentence

        ml_token_object = self.call_juman_interface(normalized_sentence)

        token_objects = [
            juman_utils.extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in ml_token_object]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)
            return tokenized_objects
619735
def stem(self, word):
        
        word = word.lower()

        word = word.replace("\xDF", "ss")

        # Every occurrence of 'u' and 'y'
        # between vowels is put into upper case.
        for i in range(1, len(word)-1):
            if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
                if word[i] == "u":
                    word = "".join((word[:i], "U", word[i+1:]))

                elif word[i] == "y":
                    word = "".join((word[:i], "Y", word[i+1:]))

        r1, r2 = self._r1r2_standard(word, self.__vowels)

        # R1 is adjusted so that the region before it
        # contains at least 3 letters.
        for i in range(1, len(word)):
            if word[i] not in self.__vowels and word[i-1] in self.__vowels:
                if len(word[:i+1]) < 3 and len(word[:i+1]) > 0:
                    r1 = word[3:]
                elif len(word[:i+1]) == 0:
                    return word
                break

        # STEP 1
        for suffix in self.__step1_suffixes:
            if r1.endswith(suffix):
                if (suffix in ("en", "es", "e") and
                    word[-len(suffix)-4:-len(suffix)] == "niss"):
                    word = word[:-len(suffix)-1]
                    r1 = r1[:-len(suffix)-1]
                    r2 = r2[:-len(suffix)-1]

                elif suffix == "s":
                    if word[-2] in self.__s_ending:
                        word = word[:-1]
                        r1 = r1[:-1]
                        r2 = r2[:-1]
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                    r2 = r2[:-len(suffix)]
                break

        # STEP 2
        for suffix in self.__step2_suffixes:
            if r1.endswith(suffix):
                if suffix == "st":
                    if word[-3] in self.__st_ending and len(word[:-3]) >= 3:
                        word = word[:-2]
                        r1 = r1[:-2]
                        r2 = r2[:-2]
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                    r2 = r2[:-len(suffix)]
                break

        # STEP 3: Derivational suffixes
        for suffix in self.__step3_suffixes:
            if r2.endswith(suffix):
                if suffix in ("end", "ung"):
                    if ("ig" in r2[-len(suffix)-2:-len(suffix)] and
                        "e" not in r2[-len(suffix)-3:-len(suffix)-2]):
                        word = word[:-len(suffix)-2]
                    else:
                        word = word[:-len(suffix)]

                elif (suffix in ("ig", "ik", "isch") and
                      "e" not in r2[-len(suffix)-1:-len(suffix)]):
                    word = word[:-len(suffix)]

                elif suffix in ("lich", "heit"):
                    if ("er" in r1[-len(suffix)-2:-len(suffix)] or
                        "en" in r1[-len(suffix)-2:-len(suffix)]):
                        word = word[:-len(suffix)-2]
                    else:
                        word = word[:-len(suffix)]

                elif suffix == "keit":
                    if "lich" in r2[-len(suffix)-4:-len(suffix)]:
                        word = word[:-len(suffix)-4]

                    elif "ig" in r2[-len(suffix)-2:-len(suffix)]:
                        word = word[:-len(suffix)-2]
                    else:
                        word = word[:-len(suffix)]
                break

        # Umlaut accents are removed and
        # 'u' and 'y' are put back into lower case.
        word = (word.replace("\xE4", "a").replace("\xF6", "o")
                    .replace("\xFC", "u").replace("U", "u")
                    .replace("Y", "y"))


        return word
1133185
def get_grammatically_correct_vocabulary_subset(self, text,
                                                    sent_filter='combined'):
        
        tokens = word_tokenize(text)
        sent_tokens = get_partial_sentence(tokens)

        if not sent_tokens:
            return self.vocabulary

        if sent_filter == 'combined':
            if len(sent_tokens) < 2:
                return self.get_bigram_filtered_vocab(sent_tokens)

            combined_filters = self.get_pos_filtered_vocab(sent_tokens) + \
                self.get_trigram_filtered_vocab(sent_tokens) + \
                self.get_bigram_filtered_vocab(sent_tokens)
            return combined_filters

        if sent_filter == 'pos' and len(sent_tokens) > 1:
            return self.get_pos_filtered_vocab(sent_tokens)
        elif sent_filter == 'bigram' or len(sent_tokens) < 2:
            return self.get_bigram_filtered_vocab(sent_tokens)
        elif sent_filter == 'trigram':
            return self.get_trigram_filtered_vocab(sent_tokens)
1048617
def tokenize(self, sentence,
                 normalized=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=normalize_text):
        # type: (text_type, bool, bool, bool, bool, Callable[[str], str])->Union[List[str], TokenizedSenetence]
        
        if six.PY2 and isinstance(sentence, str):
            sentence = sentence.decode(self.string_encoding)
        else:
            pass

        # decide normalization function depending on dictType
        if func_normalizer is None and self._dictType == 'neologd' and is_neologdn_valid:
            normalized_sentence = neologdn.normalize(sentence)
        elif func_normalizer is None and self._dictType == 'neologd' and is_neologdn_valid == False:
            raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
        elif func_normalizer == normalize_text:
            normalized_sentence = normalize_text(sentence, dictionary_mode=self._dictType)
        elif func_normalizer is None:
            normalized_sentence = sentence
        else:
            normalized_sentence = func_normalizer(sentence)

        # don't delete this variable. The variable "encoded_text" protects sentence from deleting
        if six.PY2:
            encoded_text = normalized_sentence.encode(self.string_encoding)
        else:
            encoded_text = normalized_sentence

        if six.PY2:
            tokenized_objects = []
            node = self.mecabObj.parseToNode(encoded_text)
            node = node.next
            while node.next is not None:
                word_surface = node.surface.decode(self.string_encoding)

                tuple_pos, word_stem = self.__feature_parser(node.feature.decode(self.string_encoding), word_surface)

                tokenized_obj = TokenizedResult(
                    node_obj=node,
                    tuple_pos=tuple_pos,
                    word_stem=word_stem,
                    word_surface=word_surface,
                    is_feature=is_feature,
                    is_surface=is_surface
                )
                tokenized_objects.append(tokenized_obj)
                node = node.next

            tokenized_sentence = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=tokenized_objects)
        else:
            parsed_result = self.mecabObj.parse(encoded_text)
            tokenized_objects = self.__postprocess_analyzed_result(
                string_mecab_parsed_result=parsed_result,
                is_feature=is_feature,
                is_surface=is_surface
            )
            tokenized_sentence = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=tokenized_objects
            )  # type: TokenizedSenetence

        if return_list:
            return tokenized_sentence.convert_list_object()
        else:
            return tokenized_sentence
215932
def noun_phrases(self):
        """"""
        return WordList([phrase.strip()
                         for phrase in self.np_extractor.extract(self.raw)
                         if len(phrase.split()) > 1])
1157707
def _literal_exclusions(self, stem: LiteralStemRange,
                            exclusions: List[ShExDocParser.LiteralExclusionContext]) -> None:
        
        for excl in exclusions:
            excl_literal_v = self.context.literal_to_ObjectLiteral(excl.literal()).value
            stem.exclusions.append(LiteralStem(excl_literal_v) if excl.STEM_MARK() else excl_literal_v)
879318
def convert_examples_to_features(examples, seq_length, tokenizer):
    """"""

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)

        if tokens_b:
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > seq_length - 2:
                tokens_a = tokens_a[0:(seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids:   0   0  0    0    0     0      0   0    1  1  1   1  1   1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids:   0   0   0   0  0     0   0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = []
        input_type_ids = []
        tokens.append("[CLS]")
        input_type_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            input_type_ids.append(0)
        tokens.append("[SEP]")
        input_type_ids.append(0)

        if tokens_b:
            for token in tokens_b:
                tokens.append(token)
                input_type_ids.append(1)
            tokens.append("[SEP]")
            input_type_ids.append(1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        while len(input_ids) < seq_length:
            input_ids.append(0)
            input_mask.append(0)
            input_type_ids.append(0)

        assert len(input_ids) == seq_length
        assert len(input_mask) == seq_length
        assert len(input_type_ids) == seq_length

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("unique_id: %s" % (example.unique_id))
            logger.info("tokens: %s" % " ".join([str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids]))

        features.append(
            InputFeatures(
                unique_id=example.unique_id,
                tokens=tokens,
                input_ids=input_ids,
                input_mask=input_mask,
                input_type_ids=input_type_ids))
    return features
787313
def gen_sentences(self, tokens, aliases=None):
        
        if aliases is None:
            aliases = {}
        for sentence in self._gen_sentences(tokens):
            try:
                alias = aliases[str(sentence[0])]
            except KeyError:
                # do nothing if no alias is found
                pass
            except IndexError:
                pass
            else:
                sentence[0:1] = list(Program(alias).gen_tokens())
            yield transform(Sentence(sentence), self.transforms)