Dictionary-based word segmentation, using maximal matching algorithm and Thai Character Cluster
    :param str text: text to be tokenized to words
    :return: list of words, tokenized from the text
777123
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    if not custom_dict:
        custom_dict = DEFAULT_DICT_TRIE

    return list(_onecut(text, custom_dict))
726734
def batch_split_sentences(self, texts: List[str]) -> List[List[str]]:
        
        return [self.split_sentences(text) for text in texts]
709145
def macronize_text(self, text):
        
        macronized_words = [entry[2] for entry in self.macronize_tags(text)]
        return " ".join(macronized_words)
473968
def _text_repr(text, max_len=1000):
    
    if text is None:
        text_repr = 'None'
    elif len(text) > max_len:
        text_repr = repr(text[0:max_len]) + '...'
    else:
        text_repr = repr(text)
    return text_repr
1367900
def classify(self, text):
        
        text_features = self.extract_features(text)
        return self.classifier.classify(text_features)
399299
def tokenize(self, text: str, model: object = None):
        
        sentences = re.split(self.pattern, text)
        return sentences
681094
private List<String> extractWords(@NotNull String text, Language language) {

        List<String> words = textUtils.extractWords(text, language);

        List<String> wordList = words.stream()
                                     .flatMap(word -> {
                                         return textUtils.getNormalForms(word.toLowerCase(), language)
                                                         .stream()
                                                         .filter(element -> textUtils.isNormalWord(element, language));
                                     })
                                     .collect(Collectors.toList());

        return wordList;
    }
602619
def get_spacy_entites(self, text: str) -> list:
        
        # NOTE: spaCy doc
        doc = self._language_model(text)

        entities = [{"entity": ent.label_,
                     "value": ent.text,
                     "start": ent.start_char,
                     "end": ent.end_char} for ent in doc.ents]

        return entities
18236
def from_tokens(cls, text, toks):
        
        sentence_pb = CoreNLP_pb2.Sentence()
        sentence_pb.characterOffsetBegin = 0
        sentence_pb.characterOffsetEnd = len(text)
        sentence_pb.sentenceIndex = 0
        sentence_pb.tokenOffsetBegin = 0
        sentence_pb.tokenOffsetEnd = len(toks)

        # Track progress in sentence and tokens.
        char_idx = 0
        tok_idx = 0

        buf = ""
        token_pb = None
        while char_idx < len(text):
            tok = toks[tok_idx]
            # Scan to the beginning of the token.
            if text[char_idx] != tok[0]:
                buf += text[char_idx]
                char_idx += 1
            # Aha! we have found the token. Assert that they match.
            else:
                assert text[char_idx:char_idx+len(tok)] == tok, "text did not match a token"
                # Create a new Token from this text.
                if token_pb: token_pb.after = buf
                token_pb = sentence_pb.token.add()
                token_pb.before = buf

                token_pb.beginChar = char_idx
                token_pb.endChar = char_idx + len(tok)
                # TODO(chaganty): potentially handle -LRB-?
                token_pb.value = tok
                token_pb.word = tok
                token_pb.originalText = text[char_idx:char_idx+len(tok)]

                buf = ""
                char_idx += len(tok)
                tok_idx += 1
        if token_pb: token_pb.after = buf
        assert tok_idx == len(toks), "text does not match all tokens"
        return AnnotatedSentence.from_pb(sentence_pb)
94257
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._parser(text)

        extractions = list()
        for sent in doc.sents:
            this_extraction = Extraction(value=sent.text,
                                         extractor_name=self.name,
                                         start_token=sent[0],
                                         end_token=sent[-1],
                                         start_char=sent.text[0],
                                         end_char=sent.text[-1])
            extractions.append(this_extraction)

        return extractions
586338
def extract_keywords_from_text(self, text):
        
        sentences = nltk.tokenize.sent_tokenize(text)
        self.extract_keywords_from_sentences(sentences)
1329514
public static List<List<Term>> seg2sentence(String text)
    {
        List<List<Term>> resultList = new LinkedList<List<Term>>();
        {
            for (String sentence : SentencesUtil.toSentenceList(text))
            {
                resultList.add(segment(sentence));
            }
        }

        return resultList;
    }
1335658
def mols_from_text(text, no_halt=True, assign_descriptors=True):
    
    if isinstance(text, bytes):
        t = tx.decode(text)
    else:
        t = text
    # Lazy line splitter. More efficient memory usage than str.split.
    exp = re.compile(r"[^\n]*\n|.")
    sp = (x.group(0) for x in re.finditer(exp, t))
    for c in mol_supplier(sp, no_halt, assign_descriptors):
        yield c
17585
def tokenize_words(self):
        
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        tok = self.__word_tokenizer
        text = self.text
        dicts = []
        for sentence in self[SENTENCES]:
            sent_start, sent_end = sentence[START], sentence[END]
            sent_text = text[sent_start:sent_end]
            spans = tok.span_tokenize(sent_text)
            for start, end in spans:
                dicts.append({START: start+sent_start, END: end+sent_start, TEXT: sent_text[start:end]})
        self[WORDS] = dicts
        return self
1290082
def tokenize(self, sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_type, bool, bool, bool, bool, Callable[[str],str]) -> Union[List[str], TokenizedSenetence]
        
        assert isinstance(normalize, bool)
        assert isinstance(sentence, text_type)
        normalized_sentence = func_normalizer(sentence)
        if six.PY2:
            normalized_sentence = normalized_sentence.encode('utf-8')

        result = self.__list_tags(self.kytea.getTags(normalized_sentence))

        token_objects = [
            self.__extract_morphological_information(
                kytea_tags_tuple=kytea_tags,
                is_feature=is_feature
            )
            for kytea_tags in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects
            )
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects
634973
def extract_from_text(text):
    
    return tools.remove_duplicates([i[0]
                                    for i in REGEX.findall(text) if i != ''])
940881
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}