Tag ``words`` layer with morphological analysis attributes.
778603
def tag_analysis(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        sentences = self.divide(WORDS, SENTENCES)
        for sentence in sentences:
            texts = [word[TEXT] for word in sentence]
            all_analysis = vabamorf.analyze(texts, **self.__kwargs)
            for word, analysis in zip(sentence, all_analysis):
                word[ANALYSIS] = analysis[ANALYSIS]
                word[TEXT] = analysis[TEXT]
        return self
581070
def entities(self):
    """"""
    start = 0
    end = 0
    prev_tag = u'O'
    chunks = []
    for i, (w, tag) in enumerate(self.ne_chunker.annotate(self.words)):
      if tag != prev_tag:
        if prev_tag == u'O':
          start = i
        else:
          chunks.append(Chunk(self.words[start: i], start, i, tag=prev_tag,
                              parent=self))
        prev_tag = tag
    if tag != u'O':
      chunks.append(Chunk(self.words[start: i+1], start, i+1, tag=tag,
                          parent=self))
    return chunks
407325
def extract_morphological_information(mrph_object, is_feature, is_surface):
    # type: (pyknp.Morpheme, bool, bool) -> TokenizedResult
    
    assert isinstance(mrph_object, pyknp.Morpheme)
    assert isinstance(is_feature, bool)
    assert isinstance(is_surface, bool)

    surface = mrph_object.midasi
    word_stem = mrph_object.genkei

    tuple_pos = (mrph_object.hinsi, mrph_object.bunrui)

    misc_info = {
        'katuyou1': mrph_object.katuyou1,
        'katuyou2': mrph_object.katuyou2,
        'imis': mrph_object.imis,
        'repname': mrph_object.repname
    }

    token_object = TokenizedResult(
        node_obj=None,
        tuple_pos=tuple_pos,
        word_stem=word_stem,
        word_surface=surface,
        is_feature=is_feature,
        is_surface=is_surface,
        misc_info=misc_info
    )

    return token_object
595295
def morphemes(args):
  """"""
  morfessor = load_morfessor_model(lang=args.lang)
  for l in args.input:
    words = l.strip().split()
    morphemes = [(w, u"_".join(morfessor.viterbi_segment(w)[0])) for w in words]
    line_annotations = [u"{:<16}{:<5}".format(w,p) for w, p in morphemes]
    _print(u"\n".join(line_annotations))
    _print(u"")
709145
def macronize_text(self, text):
        
        macronized_words = [entry[2] for entry in self.macronize_tags(text)]
        return " ".join(macronized_words)
1069735
def _macronize_word(self, word):
        
        head_word = word[0]
        tag = word[1]
        if tag is None:
            logger.info('Tagger {} could not tag {}.'.format(self.tagger, head_word))
            return head_word, tag, head_word
        elif tag == 'U--------':
            return (head_word, tag.lower(), head_word)
        else:
            entries = self._retrieve_morpheus_entry(head_word)
            if entries is None:
                return head_word, tag.lower(), head_word
            matched_entry = [entry for entry in entries if entry[0] == tag.lower()]
            if len(matched_entry) == 0:
                logger.info('No matching Morpheus entry found for {}.'.format(head_word))
                return head_word, tag.lower(), entries[0][2]
            elif len(matched_entry) == 1:
                return head_word, tag.lower(), matched_entry[0][2].lower()
            else:
                logger.info('Multiple matching entries found for {}.'.format(head_word))
                return head_word, tag.lower(), matched_entry[1][2].lower()
182066
def generate_additional_context(self, matching_datasets):
        """"""
        related_tags = Tag.objects.filter(
            dataset__in=matching_datasets
        ).distinct().annotate(
            Count('word')
        ).order_by('-word__count')[:5]

        return {
            'related_tags': related_tags
        }
322330
def tags(self):
        
        # See also. Sentence.__repr__().
        ch, I,O,B = self.chunk, INSIDE+"-", OUTSIDE, BEGIN+"-"
        tags = [OUTSIDE for i in range(len(self.sentence.token))]
        for i, tag in enumerate(self.sentence.token): # Default: [WORD, POS, CHUNK, PNP, RELATION, ANCHOR, LEMMA]
            if tag == WORD:
                tags[i] = encode_entities(self.string)
            elif tag == POS and self.type:
                tags[i] = self.type
            elif tag == CHUNK and ch and ch.type:
                tags[i] = (self == ch[0] and B or I) + ch.type
            elif tag == PNP and self.pnp:
                tags[i] = (self == self.pnp[0] and B or I) + "PNP"
            elif tag == REL and ch and len(ch.relations) > 0:
                tags[i] = ["-".join([str(x) for x in [ch.type]+list(reversed(r)) if x]) for r in ch.relations]
                tags[i] = "*".join(tags[i])
            elif tag == ANCHOR and ch:
                tags[i] = ch.anchor_id or OUTSIDE
            elif tag == LEMMA:
                tags[i] = encode_entities(self.lemma or "")
            elif tag in self.custom_tags:
                tags[i] = self.custom_tags.get(tag) or OUTSIDE
        return tags
1217820
def assessments(self, words=[], negation=True):
        
        a = []
        m = None # Preceding modifier (i.e., adverb or adjective).
        n = None # Preceding negation (e.g., "not beautiful").
        for w, pos in words:
            # Only assess known words, preferably by part-of-speech tag.
            # Including unknown words (polarity 0.0 and subjectivity 0.0) lowers the average.
            if w is None:
                continue
            if w in self and pos in self[w]:
                p, s, i = self[w][pos]
                # Known word not preceded by a modifier ("good").
                if m is None:
                    a.append(dict(w=[w], p=p, s=s, i=i, n=1, x=self.labeler.get(w)))
                # Known word preceded by a modifier ("really good").
                if m is not None:
                    a[-1]["w"].append(w)
                    a[-1]["p"] = max(-1.0, min(p * a[-1]["i"], +1.0))
                    a[-1]["s"] = max(-1.0, min(s * a[-1]["i"], +1.0))
                    a[-1]["i"] = i
                    a[-1]["x"] = self.labeler.get(w)
                # Known word preceded by a negation ("not really good").
                if n is not None:
                    a[-1]["w"].insert(0, n)
                    a[-1]["i"] = 1.0 / a[-1]["i"]
                    a[-1]["n"] = -1
                # Known word may be a negation.
                # Known word may be modifying the next word (i.e., it is a known adverb).
                m = None
                n = None
                if pos and pos in self.modifiers or any(map(self[w].__contains__, self.modifiers)):
                    m = (w, pos)
                if negation and w in self.negations:
                    n = w
            else:
                # Unknown word may be a negation ("not good").
                if negation and w in self.negations:
                    n = w
                # Unknown word. Retain negation across small words ("not a good").
                elif n and len(w.strip("'")) > 1:
                    n = None
                # Unknown word may be a negation preceded by a modifier ("really not good").
                if n is not None and m is not None and (pos in self.modifiers or self.modifier(m[0])):
                    a[-1]["w"].append(n)
                    a[-1]["n"] = -1
                    n = None
                # Unknown word. Retain modifier across small words ("really is a good").
                elif m and len(w) > 2:
                    m = None
                # Exclamation marks boost previous word.
                if w == "!" and len(a) > 0:
                    a[-1]["w"].append("!")
                    a[-1]["p"] = max(-1.0, min(a[-1]["p"] * 1.25, +1.0))
                # Exclamation marks in parentheses indicate sarcasm.
                if w == "(!)":
                    a.append(dict(w=[w], p=0.0, s=1.0, i=1.0, n=1, x=IRONY))
                # EMOTICONS: {("grin", +1.0): set((":-D", ":D"))}
                if w.isalpha() is False and len(w) <= 5 and w not in PUNCTUATION: # speedup
                    for (type, p), e in EMOTICONS.items():
                        if w in map(lambda e: e.lower(), e):
                            a.append(dict(w=[w], p=p, s=1.0, i=1.0, n=1, x=MOOD))
                            break
        for i in range(len(a)):
            w = a[i]["w"]
            p = a[i]["p"]
            s = a[i]["s"]
            n = a[i]["n"]
            x = a[i]["x"]
            # "not good" = slightly bad, "not bad" = slightly good.
            a[i] = (w, p * -0.5 if n < 0 else p, s, x)
        return a
357998
private GrammaticalStructure tagDependencies(List<? extends HasWord> taggedWords) {
        GrammaticalStructure gs = nndepParser.predict(taggedWords);
        return gs;
    }
154148
def v1_tag_associations(tags, tag):
    '''
    '''
    tag = tag.decode('utf-8').strip()
    return {'associations': tags.assocs_by_tag(tag)}
41896
def tag(self, text):
        
        matches = self._match(text.text)
        matches = self._resolve_conflicts(matches)

        if self.return_layer:
            return matches
        else:
            text[self.layer_name] = matches
556709
def morpheme(self,index, set=None):
        """"""
        for layer in self.select(MorphologyLayer):
            for i, m in enumerate(layer.select(Morpheme, set)):
                if index == i:
                    return m
        raise NoSuchAnnotation
1003710
def is_tagged(self, layer):
        """"""
        # we have a number of special names that are not layers but instead
        # attributes of "words" layer
        if layer == ANALYSIS:
            if WORDS in self and len(self[WORDS]) > 0:
                return ANALYSIS in self[WORDS][0]
        elif layer == LAYER_CONLL:
            if LAYER_CONLL in self and len(self[LAYER_CONLL]) > 0:
                return PARSER_OUT in self[LAYER_CONLL][0]
        elif layer == LAYER_VISLCG3:
            if LAYER_VISLCG3 in self and len(self[LAYER_VISLCG3]) > 0:
                return PARSER_OUT in self[LAYER_VISLCG3][0]
        elif layer == LABEL:
            if WORDS in self and len(self[WORDS]) > 0:
                return LABEL in self[WORDS][0]
        elif layer == CLAUSE_ANNOTATION:
            if WORDS in self and len(self[WORDS]) > 0:
                return CLAUSE_ANNOTATION in self[WORDS][0]
        elif layer == WORDNET:
            if WORDS in self and len(self[WORDS]) > 0:
                if ANALYSIS in self[WORDS][0] and len(self[WORDS][0][ANALYSIS]) > 0:
                    return WORDNET in self[WORDS][0][ANALYSIS][0]
        else:
            return layer in self
        return False
287132
def tag(self, tokens):
        """"""
        tags = []
        for token in tokens:
            normalized = self.lexicon[token].normalized
            for regex, tag in self.regexes:
                if regex.match(normalized):
                    tags.append((token, tag))
                    break
            else:
                tags.append((token, None))
        return tags
385208
def words(self, fileids=None) -> Generator[str, str, None]:
        
        for sentence in self.sents(fileids):
            words = self._word_tokenizer.tokenize(sentence)
            for word in words:
                yield word
86887
def annotate(self, word):
        ''''''
        info = []  # e.g., [ ('\'nak.su.`tus.ta', 'PUSU', 'HLHL', 'AUUA'), ]

        for syllabification, _ in syllabify(self.normalize(word), stress=True):
            stresses = ''
            weights = ''
            vowels = ''

            for syll in syllable_split(syllabification):

                try:
                    vowels += get_vowel(syll)
                    weights += get_weight(syll)
                    stresses += {'\'': 'P', '`': 'S'}.get(syll[0], 'U')

                except AttributeError:

                    # if the syllable is vowel-less...
                    if syll[-1].isalpha():
                        stresses += '*'
                        weights += '*'
                        vowels += '*'

                    else:
                        stresses += ' '
                        weights += ' '
                        vowels += ' '

            info.append((
                syllabification,
                stresses,
                weights,
                vowels,
                ))

        return info