Split the given natural language text into a list of Token objects.
@param text
@return list of tokens
514366
public synchronized List<Token> tokens(String text) {
        requireValidHandle();
        List<Token> allTokens = new ArrayList<Token>();
        int lastStart = 0;
        for (int i = indexOfSpecialUnknown(text, 0); i != -1; i = indexOfSpecialUnknown(text, i + 1)) {
            allTokens.addAll(tokensNonNull(text.substring(lastStart, i), lastStart));
            allTokens.add(new Token(TokenType.UNKNOWN, Character.toString(text.charAt(i)), i));
            lastStart = i + 1;
        }
        allTokens.addAll(tokensNonNull(text.substring(lastStart), lastStart));
        return allTokens;
    }
1091115
public static String tokenizeSmileys(String text) {
        for(Map.Entry<String, String> smiley : SMILEYS_MAPPING.entrySet()) {
            text = text.replaceAll(smiley.getKey(), smiley.getValue());
        }
        return text;
    }
1168610
def split(text: str) -> List[str]:
    
    return [word for word in SEPARATOR.split(text) if word.strip(' \t')]
666334
def _split_string_to_tokens(text):
  """"""
  if not text:
    return []
  ret = []
  token_start = 0
  # Classify each character in the input string
  is_alnum = [c in _ALPHANUMERIC_CHAR_SET for c in text]
  for pos in xrange(1, len(text)):
    if is_alnum[pos] != is_alnum[pos - 1]:
      token = text[token_start:pos]
      if token != u" " or token_start == 0:
        ret.append(token)
      token_start = pos
  final_token = text[token_start:]
  ret.append(final_token)
  return ret
1205482
public synchronized List<Sentence> sentences(String text) {
        requireValidHandle();
        Libvoikko lib = getLib();
        List<Sentence> result = new ArrayList<Sentence>();
        if (!isValidInput(text)) {
            result.add(new Sentence(text, SentenceStartType.NONE));
            return result;
        }
        byte[] textBytes = s2n(text);
        int textLen = textBytes.length - 1;
        SizeTByReference sentenceLenByRef = new SizeTByReference();
        while (textLen > 0) {
            int sentenceTypeInt = lib.voikkoNextSentenceStartCstr(handle, textBytes, new SizeT(textLen), sentenceLenByRef);
            int sentenceLen = sentenceLenByRef.getValue().intValue();
            SentenceStartType sentenceType = SentenceStartType.values()[sentenceTypeInt];
            String tokenText = text.substring(0, sentenceLen);
            result.add(new Sentence(tokenText, sentenceType));
            text = text.substring(sentenceLen);
            textBytes = s2n(text);
            textLen = textBytes.length - 1;
        }
        return result;
    }
1011297
public static String[] simpleTokenize(String text) {
		if (GeneralUtils.isEmpty(text))
			return null;
		return text.split("[ ,.;:><^`�\\]\\[}{|��+_)(*&�%$#@!=\\-?�\\/\"]+", -1);
	}
1329514
public static List<List<Term>> seg2sentence(String text)
    {
        List<List<Term>> resultList = new LinkedList<List<Term>>();
        {
            for (String sentence : SentencesUtil.toSentenceList(text))
            {
                resultList.add(segment(sentence));
            }
        }

        return resultList;
    }
254820
public static List<String> splitAsList(Pattern splitPattern, String text) {
        return splitAsStream(splitPattern, text).collect(toList());
    }
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
134586
def tokenize_list(self, text):
        
        return [self.get_record_token(record) for record in self.analyze(text)]
326220
public static List<Term> segment(String text)
    {
        List<Term> termList = new LinkedList<Term>();
        Matcher matcher = WEB_URL.matcher(text);
        int begin = 0;
        int end;
        while (matcher.find())
        {
            end = matcher.start();
            termList.addAll(SEGMENT.seg(text.substring(begin, end)));
            termList.add(new Term(matcher.group(), Nature.xu));
            begin = matcher.end();
        }
        if (begin < text.length()) termList.addAll(SEGMENT.seg(text.substring(begin)));

        return termList;
    }
672325
def split(self, text):
        """"""
        text = cleanup(text)
        return self.sent_detector.tokenize(text.strip())
238990
function(text, pad) {
			var result = [];
			var lines = this.splitByLines(text);
			var nl = '\n';
				
			result.push(lines[0]);
			for (var j = 1; j < lines.length; j++) 
				result.push(nl + pad + lines[j]);
				
			return result.join('');
		}
1307659
public Collection<Text> split(char separator) {
        int pointer = pos;
        int limit = pos + len;
        int mark = pointer;

        ArrayList<Text> tokens = new ArrayList<Text>();
        while (pointer < limit) {
            if (chars[pointer] == separator) {
                tokens.add(new Text(chars, mark, pointer - mark));
                mark = pointer + 1;
            }
            pointer++;
        }
        
        tokens.add(new Text(chars, mark, limit - mark));        
        return tokens;
    }
1128743
private String[] splitInTwo(String text)	{
	  Matcher matcher = LINE_SEPARATOR.matcher(text);
	  while (matcher.find()){
		  int start = matcher.start();
		  if (start > 0 && start < text.length())	{
			  String segment = text.substring(0, start);
			  if (segment.trim().length() > 0)
				  return new String[] {text.substring(0, start), text.substring(start)};
		  }
	  }
	  return new String[] {text};
  }
1165842
private List<Word>[] fullSeg(String text){
        //文本长度
        final int textLen = text.length();
        //以每一个字作为词的开始，所能切分的词
        final List<String>[] sequence = new LinkedList[textLen];
        if(isParallelSeg()){
            //并行化
            List<Integer> list = new ArrayList<>(textLen);
            for(int i=0; i<textLen; i++){
                list.add(i);
            }
            list.parallelStream().forEach(i->sequence[i] = fullSeg(text, i));
        }else {
            //串行化
            for (int i = 0; i < textLen; i++) {
                sequence[i] = fullSeg(text, i);
            }
        }
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("全切分中间结果：");
            int i=1;
            for(List<String> list : sequence){
                LOGGER.debug("\t"+(i++)+"、"+list);
            }
        }
        //树叶
        List<Node> leaf = new LinkedList<>();
        for(String word : sequence[0]){
            //树根
            Node node = new Node(word);
            //把全切分中间结果（二维数组）转换为合理切分
            buildNode(node, sequence, word.length(), leaf);
        }
        //清理无用数据
        for(int j=0; j<sequence.length; j++){
            sequence[j].clear();
            sequence[j] = null;
        }
        //从所有树叶开始反向遍历出全切分结果
        List<Word>[] res = toWords(leaf);
        leaf.clear();
        return res;
    }
840903
protected List<String> getTokens(String text){
		String[] tokens = text.toLowerCase().split("[\\s\\p{Punct}]");
		List<String> tokenList = new ArrayList<String>(tokens.length);
		for (int i = 0; i < tokens.length; i++){
			if (tokens[i].trim().length() < 1){
				continue;
			}
			tokenList.add(tokens[i].trim());
		}
		return tokenList;
	}