Do sentence tokenization.

@param text to be tokenized
@return the string
352587
public String senTokenize(String text){
		String ret = text;
		
		if (isTokenization){
			ret = PennTokenizer.tokenize(text);
		}
		
		return ret.trim();
	}
842151
public String protectKeyword(String text) {
		if (!Strings.isEmpty(text) && isKeyword(text)) {
			return "^" + text;
		}
		return text;
	}
1191571
public T text(String text) throws IOException {
		if (text != null) {
			writer.write(text);
		}
		return (T) this;
	}
858350
def _tokenize(self, text):
        
        sentences = []
        tokens = []
        for word in self._clean_accents(text).split(' '):
            tokens.append(word)
            if '.' in word:
                sentences.append(tokens)
                tokens = []
        return sentences
322688
def tokenize(self, text: str, customize=True, disable=[]) -> List[Token]:
        

        """Tokenize text"""
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        # disable spacy parsing, tagging etc as it takes a long time if the text is short
        tokens = self.nlp(text, disable=disable)
        if customize:
            tokens = [self.custom_token(a_token) for a_token in tokens]

        return tokens
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
82941
def tokenize(self, text, include_punc=True, **kwargs):
        
        return self.tokenizer.word_tokenize(text, include_punc, **kwargs)
825094
public Collection<Token> tokenize(String text)
    {

        Collection<Token> tokens = new ArrayList<Token>();

        Collection<Emit> collectedEmits = parseText(text);
        // 下面是最长分词的关键
        IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits);
        intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
        // 移除结束

        int lastCollectedPosition = -1;
        for (Emit emit : collectedEmits)
        {
            if (emit.getStart() - lastCollectedPosition > 1)
            {
                tokens.add(createFragment(emit, text, lastCollectedPosition));
            }
            tokens.add(createMatch(emit, text));
            lastCollectedPosition = emit.getEnd();
        }
        if (text.length() - lastCollectedPosition > 1)
        {
            tokens.add(createFragment(null, text, lastCollectedPosition));
        }

        return tokens;
    }
1169192
public String wordSegment(String text){
		String ret = text;
		
		if (vnSegmenter == null) return ret;
		ret = vnSegmenter.segmenting(ret);
		return ret;
	}
302369
public SpannableStringBuilder append(CharSequence text) {
        int length = length();
        return replace(length, length, text, 0, text.length());
    }
1205482
public synchronized List<Sentence> sentences(String text) {
        requireValidHandle();
        Libvoikko lib = getLib();
        List<Sentence> result = new ArrayList<Sentence>();
        if (!isValidInput(text)) {
            result.add(new Sentence(text, SentenceStartType.NONE));
            return result;
        }
        byte[] textBytes = s2n(text);
        int textLen = textBytes.length - 1;
        SizeTByReference sentenceLenByRef = new SizeTByReference();
        while (textLen > 0) {
            int sentenceTypeInt = lib.voikkoNextSentenceStartCstr(handle, textBytes, new SizeT(textLen), sentenceLenByRef);
            int sentenceLen = sentenceLenByRef.getValue().intValue();
            SentenceStartType sentenceType = SentenceStartType.values()[sentenceTypeInt];
            String tokenText = text.substring(0, sentenceLen);
            result.add(new Sentence(tokenText, sentenceType));
            text = text.substring(sentenceLen);
            textBytes = s2n(text);
            textLen = textBytes.length - 1;
        }
        return result;
    }
365057
public Category classify(String text) {
		StringTokenizer itr = new StringTokenizer(text);
		Map<Category, Double> scorePerCategory = new HashMap<Category, Double>();
		double bestScore = Double.NEGATIVE_INFINITY;
		Category bestCategory = null;
		while(itr.hasMoreTokens()) {
			String token = NaiveBayesGenerate.normalizeWord(itr.nextToken());
			for(Category category : Category.values()) {
				int count = MapUtils.getInteger(wordCountPerCategory.get(category), token, 0) + 1;
				double wordScore  = Math.log(count / (double) (tokensPerCategory.get(category) + V));
				double totalScore = MapUtils.getDouble(scorePerCategory, category, 0.) + wordScore;
				if(totalScore > bestScore) {
					bestScore = totalScore;
					bestCategory = category;
				}
				scorePerCategory.put(category, totalScore);
			}
		}
		return bestCategory;
	}
912937
public <V> V convertFrom(final String text) {
        if (text == null) return null;
        return (V) convertFrom(Utility.charArray(text));
    }
1003055
public static String capitalise(String text) {
        if (empty(text)) {
            return "";
        }
        return text.substring(0, 1).toUpperCase() + text.substring(1);
    }
1329514
public static List<List<Term>> seg2sentence(String text)
    {
        List<List<Term>> resultList = new LinkedList<List<Term>>();
        {
            for (String sentence : SentencesUtil.toSentenceList(text))
            {
                resultList.add(segment(sentence));
            }
        }

        return resultList;
    }
816940
public Object parseText(String text) {
        if (text == null || "".equals(text)) {
            throw new IllegalArgumentException("Text must not be null");
        }
        return createParser().parse(text);
    }
906892
def _tokenize(self, text):
        
        sentence_tokenizer = TokenizeSentence('latin')
        sentences = sentence_tokenizer.tokenize_sentences(text.lower())

        sent_words = []
        punkt = PunktLanguageVars()
        for sentence in sentences:
            words = punkt.word_tokenize(sentence)

            assert isinstance(words, list)
            words_new = []
            for word in words:
                if word not in self.punctuation or self.abbreviations or self.numbers or self.abbreviations:  # pylint: disable=line-too-long
                    words_new.append(word)

            # rm all numbers here with: re.compose(r'[09]')
            sent_words.append(words_new)

        return sent_words