Return array of tags for the locale specific first sentence in the text.
763974
static Tag[] firstSentenceTags(DocImpl holder, String text) {
        DocLocale doclocale = holder.env.doclocale;
        return getInlineTags(holder,
                             doclocale.localeSpecificFirstSentence(holder, text));
    }
460155
private static int[] translate(PennTreebankPOS[] tags) {
        int[] seq = new int[tags.length];
        
        for (int i = 0; i < tags.length; i++) {
            seq[i] = tags[i].ordinal();
        }

        return seq;
    }
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
1329036
def get_texts_and_labels(sentence_chunk):
    """"""
    words = sentence_chunk.split('\n')
    texts = []
    labels = []
    for word in words:
        word = word.strip()
        if len(word) > 0:
            toks = word.split('\t')
            texts.append(toks[0].strip())
            labels.append(toks[-1].strip())
    return texts, labels
1056521
protected function findCommentTagIndexes(File $file, $commentStart, $tagName)
    {
        $indexes = [];
        $tokens  = $file->getTokens();

        foreach ($tokens[$commentStart]['comment_tags'] as $index) {
            if ($tokens[$index]['content'] === $tagName) {
                $indexes[] = $index;
            }
        }

        return $indexes;
    }
706823
def find_tags(self, tokens, **kwargs):
        
        # ["The", "cat", "purs"] => [["The", "DT"], ["cat", "NN"], ["purs", "VB"]]
        return find_tags(tokens,
                    lexicon = kwargs.get(   "lexicon", self.lexicon or {}),
                      model = kwargs.get(     "model", self.model),
                 morphology = kwargs.get("morphology", self.morphology),
                    context = kwargs.get(   "context", self.context),
                   entities = kwargs.get(  "entities", self.entities),
                   language = kwargs.get(  "language", self.language),
                    default = kwargs.get(   "default", self.default),
                        map = kwargs.get(       "map", None))
1319906
def _sentence_context(match, language='latin', case_insensitive=True):
    

    language_punct = {'greek': r'\.|;',
                      'latin': r'\.|\?|!'}

    assert language in language_punct.keys(), \
        'Available punctuation schemes: {}'.format(language_punct.keys())

    start = match.start()
    end = match.end()
    window = 1000
    snippet_left = match.string[start - window:start + 1]
    snippet_right = match.string[end:end + window]
    re_match = match.string[match.start():match.end()]

    comp_sent_boundary = regex.compile(language_punct[language], flags=regex.VERSION1)
    # Left
    left_punct = []
    for punct in comp_sent_boundary.finditer(snippet_left):
        end = punct.end()
        left_punct.append(end)
    try:
        last_period = left_punct.pop() + 1
    except IndexError:
        last_period = 0

    # Right
    right_punct = []
    for punct in comp_sent_boundary.finditer(snippet_right):
        end = punct.end()
        right_punct.append(end)
    try:
        first_period = right_punct.pop(0)
    except IndexError:
        first_period = 0

    sentence = snippet_left[last_period:-1] + '*' + re_match + '*' + snippet_right[0:first_period]

    return sentence
526712
def pos_tags(self):
        
        return [(Word(word, pos_tag=t), unicode(t))
                for word, t in self.pos_tagger.tag(self.raw)
                # new keyword PatternTagger(include_punc=False)
                # if not PUNCTUATION_REGEX.match(unicode(t))
                ]
1266524
def get_unique_sentence_indices( text ):
    '''  '''
    # Add sentence annotation (if missing)
    if not text.is_tagged( SENTENCES ):
        text.tokenize_sentences()
    # Collect (unique) sent indices over the whole text
    sent_indices = []
    sent_id = 0
    for sub_text in text.split_by( SENTENCES ):
        for word in sub_text.words:
            sent_indices.append( sent_id )
        sent_id += 1
    assert len(sent_indices) == len(text.words), '(!) Number of sent indices should match nr of words!'
    return sent_indices
691556
def tag(self, corpus, tokenize=True):
        ''''''
        # Assume untokenized corpus has \n between sentences and ' ' between words
        s_split = SentenceTokenizer().tokenize if tokenize else lambda t: t.split('\n')
        w_split = WordTokenizer().tokenize if tokenize else lambda s: s.split()
        def split_sents(corpus):
            for s in s_split(corpus):
                yield w_split(s)

        prev, prev2 = self.START
        tokens = []
        for words in split_sents(corpus):
            context = self.START + [self._normalize(w) for w in words] + self.END
            for i, word in enumerate(words):
                tag = self.tagdict.get(word)
                if not tag:
                    features = self._get_features(i, word, context, prev, prev2)
                    tag = self.model.predict(features)
                tokens.append((word, tag))
                prev2 = prev
                prev = tag
        return tokens
603761
def paragraph_texts(self):
        """"""
        if not self.is_tagged(PARAGRAPHS):
            self.tokenize_paragraphs()
        return self.texts(PARAGRAPHS)
661453
def paragraph_spans(self):
        """"""
        if not self.is_tagged(PARAGRAPHS):
            self.tokenize_paragraphs()
        return self.spans(PARAGRAPHS)
752568
def word_texts(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return [word[TEXT] for word in self[WORDS]]
739466
def named_entities(self):
        """"""
        if not self.is_tagged(NAMED_ENTITIES):
            self.tag_named_entities()
        phrases = self.split_by(NAMED_ENTITIES)
        return [' '.join(phrase.lemmas) for phrase in phrases]
728655
public static function findByNameOrCreate($tags, string $group = null, string $locale = null): Collection
    {
        $locale = $locale ?? app()->getLocale();

        return collect(Taggable::parseDelimitedTags($tags))->map(function (string $tag) use ($group, $locale) {
            return static::firstByName($tag, $group, $locale) ?: static::createByName($tag, $group, $locale);
        });
    }
135687
def complete_use(self, text, *_):
        
        return [t + " " for t in REGIONS if t.startswith(text)]
1320343
def get_sentences(self, root_element, block_tags):
        """"""
        sentences = []
        for element in root_element:
            if not self.any_ends_with(block_tags, element.tag):
                # tag not in block_tags
                if element.text is not None and not re.match('^\s*$',
                                                             element.text):
                    sentences.extend(self.sentence_tokenize(element.text))
                sentences.extend(self.get_sentences(element, block_tags))

        f = open('sentence_debug.txt', 'w')
        for s in sentences:
            f.write(s.lower() + '\n')
        f.close()
        return sentences