A helper method that allows you to construct an AnnotatedSentence with just token information:
        :param (str) text -- full text of the sentence.
        :param (list[str]) toks -- tokens
18236
def from_tokens(cls, text, toks):
        
        sentence_pb = CoreNLP_pb2.Sentence()
        sentence_pb.characterOffsetBegin = 0
        sentence_pb.characterOffsetEnd = len(text)
        sentence_pb.sentenceIndex = 0
        sentence_pb.tokenOffsetBegin = 0
        sentence_pb.tokenOffsetEnd = len(toks)

        # Track progress in sentence and tokens.
        char_idx = 0
        tok_idx = 0

        buf = ""
        token_pb = None
        while char_idx < len(text):
            tok = toks[tok_idx]
            # Scan to the beginning of the token.
            if text[char_idx] != tok[0]:
                buf += text[char_idx]
                char_idx += 1
            # Aha! we have found the token. Assert that they match.
            else:
                assert text[char_idx:char_idx+len(tok)] == tok, "text did not match a token"
                # Create a new Token from this text.
                if token_pb: token_pb.after = buf
                token_pb = sentence_pb.token.add()
                token_pb.before = buf

                token_pb.beginChar = char_idx
                token_pb.endChar = char_idx + len(tok)
                # TODO(chaganty): potentially handle -LRB-?
                token_pb.value = tok
                token_pb.word = tok
                token_pb.originalText = text[char_idx:char_idx+len(tok)]

                buf = ""
                char_idx += len(tok)
                tok_idx += 1
        if token_pb: token_pb.after = buf
        assert tok_idx == len(toks), "text does not match all tokens"
        return AnnotatedSentence.from_pb(sentence_pb)
1112886
def loads(text):
    
    from prettytoml.parser import parse_tokens
    from prettytoml.lexer import tokenize as lexer
    from .file import TOMLFile

    tokens = tuple(lexer(text, is_top_level=True))
    elements = parse_tokens(tokens)
    return TOMLFile(elements)
134586
def tokenize_list(self, text):
        
        return [self.get_record_token(record) for record in self.analyze(text)]
298278
def tokenize(self, sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_type, bool, bool, bool, bool, Callable[[text_type], text_type]) -> Union[TokenizedSenetence, List[text_type]]
        
        if normalize:
            normalized_sentence = func_normalizer(sentence)
        else:
            normalized_sentence = sentence

        ml_token_object = self.call_juman_interface(normalized_sentence)

        token_objects = [
            juman_utils.extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in ml_token_object]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)
            return tokenized_objects
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char
953804
def tokenize(text):

    

    stem = PorterStemmer().stem
    tokens = re.finditer('[a-z]+', text.lower())

    for offset, match in enumerate(tokens):

        # Get the raw token.
        unstemmed = match.group(0)

        yield { # Emit the token.
            'stemmed':      stem(unstemmed),
            'unstemmed':    unstemmed,
            'offset':       offset
        }
905105
def parse(self, text):
        
        tokens = self.lex(text)
        parser = Parser(tokens)
        return parser.parse()
25308
def tokens(self):
        """"""
        spans = self.word_tokenizer.span_tokenize(self.text)
        toks = [Token(
            text=self.text[span[0]:span[1]],
            start=span[0] + self.start,
            end=span[1] + self.start,
            lexicon=self.lexicon
        ) for span in spans]
        return toks
797308
def annotate_indices(self, sentence):
        """"""
        max_index = 0
        max_depth = 1
        stack_of_indexes = [ max_index ]
        for token in sentence:
            if CLAUSE_ANNOT not in token:
                token[CLAUSE_IDX] = stack_of_indexes[-1]
            else:
                # Alustavad märgendused
                for annotation in token[CLAUSE_ANNOT]:
                    if annotation == "KIILU_ALGUS":
                        # Liigume sügavamale, alustame järgmist kiilu
                        max_index += 1
                        stack_of_indexes.append(max_index)
                        if (len(stack_of_indexes) > max_depth):
                            max_depth = len(stack_of_indexes)
                token[CLAUSE_IDX] = stack_of_indexes[-1]
                # Lõpetavad märgendused
                for annotation in token[CLAUSE_ANNOT]:
                    if annotation == "KINDEL_PIIR":
                        # Liigume edasi samal tasandil, alustame järgmist osalauset
                        max_index += 1
                        stack_of_indexes[-1] = max_index
                    elif annotation == "KIILU_LOPP":
                        # Taandume sügavusest, sulgeme ühe kiilu
                        stack_of_indexes.pop()
        return sentence
906892
def _tokenize(self, text):
        
        sentence_tokenizer = TokenizeSentence('latin')
        sentences = sentence_tokenizer.tokenize_sentences(text.lower())

        sent_words = []
        punkt = PunktLanguageVars()
        for sentence in sentences:
            words = punkt.word_tokenize(sentence)

            assert isinstance(words, list)
            words_new = []
            for word in words:
                if word not in self.punctuation or self.abbreviations or self.numbers or self.abbreviations:  # pylint: disable=line-too-long
                    words_new.append(word)

            # rm all numbers here with: re.compose(r'[09]')
            sent_words.append(words_new)

        return sent_words
1246019
def tokenize_to_spacy_doc(self, text: str) -> Doc:
        
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        doc = self.nlp(text, disable=['parser'])
        for a_token in doc:
            self.custom_token(a_token)

        return doc
342078
public static void annotateChunkText(CoreMap chunk, Class tokenTextKey)
  {
    List<CoreLabel> chunkTokens = chunk.get(CoreAnnotations.TokensAnnotation.class);
    String text = getTokenText(chunkTokens, tokenTextKey);
    chunk.set(CoreAnnotations.TextAnnotation.class, text);
  }
1267428
def tokenize(self, sentence_str):
        '''
        
        '''
        mt = MeCab.Tagger("-Owakati")
        wordlist = mt.parse(sentence_str)
        token_list = wordlist.rstrip(" \n").split(" ")
        return token_list
726734
def batch_split_sentences(self, texts: List[str]) -> List[List[str]]:
        
        return [self.split_sentences(text) for text in texts]
348574
def lex(self, text):
        
        pos = 0
        while text:
            token_class, match = self.tokens.get_token(text)
            if token_class is not None:
                matched_text = text[match.start():match.end()]
                yield token_class(matched_text)
                text = text[match.end():]
                pos += match.end()
            elif text[0] in self.blank_chars:
                text = text[1:]
                pos += 1
            else:
                raise LexerError(
                        'Invalid character %s in %s' % (text[0], text),
                        position=pos)

        yield self.end_token()
555732
def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
        ""
        for rule in self.pre_rules: t = rule(t)
        toks = tok.tokenizer(t)
        for rule in self.post_rules: toks = rule(toks)
        return toks
426064
def analyze(self, text, tokenizer=str.split):
        
        if not self.tagger:
            self.tagger = Tagger(self.model,
                                 preprocessor=self.p,
                                 tokenizer=tokenizer)

        return self.tagger.analyze(text)