默认分词算法实现：
1、把要分词的文本根据标点符号进行分割
2、对分割后的文本进行分词
3、组合分词结果
@param text 文本
@return 分词结果
15822
public List<Word> segDefault(String text) {
        List<String> sentences = Punctuation.seg(text, KEEP_PUNCTUATION);
        if(sentences.size() == 1){
            return segSentence(sentences.get(0));
        }
        if(!PARALLEL_SEG){
            //串行顺序处理，不能利用多核优势
            return sentences.stream().flatMap(sentence->segSentence(sentence).stream()).collect(Collectors.toList());
        }
        //如果是多个句子，可以利用多核提升分词速度
        Map<Integer, String> sentenceMap = new HashMap<>();
        int len = sentences.size();
        for(int i=0; i<len; i++){
            //记住句子的先后顺序，因为后面的parallelStream方法不保证顺序
            sentenceMap.put(i, sentences.get(i));
        }
        //用数组收集句子分词结果
        List<Word>[] results = new List[sentences.size()];
        sentenceMap.entrySet().stream().forEach(entry -> {
            int index = entry.getKey();
            String sentence = entry.getValue();
            results[index] = segSentence(sentence);
        });
        sentences.clear();
        sentences = null;
        sentenceMap.clear();
        sentenceMap = null;
        List<Word> resultList = new ArrayList<>();
        for(List<Word> result : results){
            if(result == null || result.isEmpty()){
                continue;
            }
            resultList.addAll(result);
        }
        return resultList;
    }
1015813
public JavaSrcTextBuffer printLinesWithIndent(final String text) {
    final String[] splitted = text.split("\n", -1);

    for (final String aSplitted : splitted) {
      this.indent().println(aSplitted);
    }

    return this;
  }
1157365
public <V> V convertFrom(final char[] text) {
        if (text == null) return null;
        return (V) convertFrom(text, 0, text.length);
    }
475713
public static List<Word> seg(String text, SegmentationAlgorithm segmentationAlgorithm){        
        List<Word> words = SegmentationFactory.getSegmentation(segmentationAlgorithm).seg(text);
        //停用词过滤
        StopWord.filterStopWords(words);
        return words;
    }
840903
protected List<String> getTokens(String text){
		String[] tokens = text.toLowerCase().split("[\\s\\p{Punct}]");
		List<String> tokenList = new ArrayList<String>(tokens.length);
		for (int i = 0; i < tokens.length; i++){
			if (tokens[i].trim().length() < 1){
				continue;
			}
			tokenList.add(tokens[i].trim());
		}
		return tokenList;
	}
13320
public String wrap(String text) {
        StringBuilder sb = new StringBuilder();
        int continuationLength = continuation.length();
        int currentPosition = 0;

        for (String word : text.split(" ")) {
            String lastWord;
            int wordLength = word.length();

            if (currentPosition + wordLength <= width) {
                if (currentPosition != 0) {
                    sb.append(" ");
                    currentPosition += 1;
                }

                sb.append(lastWord = word);
                currentPosition += wordLength;
            } else {
                if (currentPosition > 0) {
                    sb.append(LINE_SEPARATOR);
                    currentPosition = 0;
                }

                if (wordLength > width && strict) {
                    int i = 0;
                    while (i + width < wordLength) {
                        sb.append(word.substring(i, width - continuationLength)).append(continuation)
                                .append(LINE_SEPARATOR);
                        i += width - continuationLength;
                    }
                    String endOfWord = word.substring(i);
                    sb.append(lastWord = endOfWord);
                    currentPosition = endOfWord.length();
                } else {
                    sb.append(lastWord = word);
                    currentPosition += wordLength;
                }
            }

            int lastNewLine = lastWord.lastIndexOf("\n");
            if (lastNewLine != -1) {
                currentPosition = lastWord.length() - lastNewLine;
            }
        }

        return sb.toString();
    }
650693
private List<URLPatternHolder> convertTextToPatterns(String text) {
        List<URLPatternHolder> list = new ArrayList<URLPatternHolder>();
        if (StringUtils.isNotEmpty(text)) {
            BufferedReader br = new BufferedReader(new StringReader(text));
            int counter = 0;
            String line;
            while (true) {
                counter++;
                try {
                    line = br.readLine();
                } catch (IOException ioe) {
                    throw new IllegalArgumentException(ioe.getMessage());
                }
                if (line == null) {
                    break;
                }
                line = StringUtils.trim(line);
                if (StringUtils.isBlank(line)) {
                    continue;
                }
                if (logger.isDebugEnabled()) {
                    logger.debug("Line " + counter + ": " + line);
                }
                list.add(convertStringToPattern(line));
            }
        }
        return list;
    }
1369765
public CAS process(String text) {
        CAS cas = retrieve();

        cas.setDocumentText(text);
        try {
            analysisEngine.process(cas);
        } catch (AnalysisEngineProcessException e) {
            if (text != null && !text.isEmpty())
                return process(text);
            throw new RuntimeException(e);
        }

        return cas;


    }
506266
public Integer[][] textsToSequences(String[] texts) {
        Integer oovTokenIndex  = wordIndex.get(outOfVocabularyToken);
        String[] wordSequence;
        ArrayList<Integer[]> sequences = new ArrayList<>();
        for (String text: texts) {
            if (charLevel) {
                if (lower) {
                    text = text.toLowerCase();
                }
                wordSequence = text.split("");
            } else {
                wordSequence = textToWordSequence(text, filters, lower, split);
            }
            ArrayList<Integer> indexVector = new ArrayList<>();
            for (String word: wordSequence) {
                if (wordIndex.containsKey(word)) {
                    int index = wordIndex.get(word);
                    if (numWords != null && index >= numWords) {
                        if (oovTokenIndex != null)
                            indexVector.add(oovTokenIndex);
                    } else {
                        indexVector.add(index);
                    }
                } else if (oovTokenIndex != null) {
                    indexVector.add(oovTokenIndex);
                }
            }
            Integer[] indices = indexVector.toArray(new Integer[indexVector.size()]);
            sequences.add(indices);
        }
        return sequences.toArray(new Integer[sequences.size()][]);
    }
1165842
private List<Word>[] fullSeg(String text){
        //文本长度
        final int textLen = text.length();
        //以每一个字作为词的开始，所能切分的词
        final List<String>[] sequence = new LinkedList[textLen];
        if(isParallelSeg()){
            //并行化
            List<Integer> list = new ArrayList<>(textLen);
            for(int i=0; i<textLen; i++){
                list.add(i);
            }
            list.parallelStream().forEach(i->sequence[i] = fullSeg(text, i));
        }else {
            //串行化
            for (int i = 0; i < textLen; i++) {
                sequence[i] = fullSeg(text, i);
            }
        }
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("全切分中间结果：");
            int i=1;
            for(List<String> list : sequence){
                LOGGER.debug("\t"+(i++)+"、"+list);
            }
        }
        //树叶
        List<Node> leaf = new LinkedList<>();
        for(String word : sequence[0]){
            //树根
            Node node = new Node(word);
            //把全切分中间结果（二维数组）转换为合理切分
            buildNode(node, sequence, word.length(), leaf);
        }
        //清理无用数据
        for(int j=0; j<sequence.length; j++){
            sequence[j].clear();
            sequence[j] = null;
        }
        //从所有树叶开始反向遍历出全切分结果
        List<Word>[] res = toWords(leaf);
        leaf.clear();
        return res;
    }
1063836
public void fromText(String text) {
        String str;
        Occurrence occ;
        List<Term> terms = new ArrayList<Term>();
        StringBuilder sb = new StringBuilder();
        for (List<Term> sentence : seg2sentence(text)) {
            totalTerm += sentence.size();

            // 产生ngram, 总串长度不超过length
            for (int i = 0, j, len = sentence.size(); i < len; ++i) {
                for (j = i; j < len; ++j) {
                    str = sentence.get(j).getRealName();
                    if (termMap.containsKey(str)) {
                        termMap.put(str, termMap.get(str) + 1);
                    } else {
                        addTerm(str);
                    }

                    sb.append(str);
                    if (length < sb.length()) {
                        break;
                    }

                    terms.add(sentence.get(j));

                    str = sb.toString();
                    occ = occurrenceMap.containsKey(str) ? occurrenceMap.get(str) : new Occurrence(new ArrayList<Term>(terms));

                    // 添加左邻项
                    if (0 < i) {
                        occ.addLeftTerm(sentence.get(i - 1).getRealName());
                    }

                    // 添加右临项
                    if (j < len - 1) {
                        occ.addRightTerm(sentence.get(j + 1).getRealName());
                    }

                    // 递增出现频次
                    occ.increaseFrequency();

                    //
                    if (occ.getFrequency() == 1) {
                        addOccurrence(str, occ);
                    }
                }

                //
                terms.clear();
                sb.delete(0, sb.length());
            }
        }
    }
916002
public List<String> splitMatches(String text, int minSize) {
    TrieNode node = inner.root();
    List<String> matches = new ArrayList<>();
    String accumulator = "";
    for (int i = 0; i < text.length(); i++) {
      short prevDepth = node.getDepth();
      TrieNode prevNode = node;
      node = node.getContinuation(text.charAt(i));
      if (null == node) node = inner.root();
      if (!accumulator.isEmpty() && (node.getDepth() < prevDepth || (prevNode.hasChildren() && node.getDepth() == prevDepth))) {
        if (accumulator.length() > minSize) {
          matches.add(accumulator);
          node = ((Optional<TrieNode>) inner.root().getChild(text.charAt(i))).orElse(inner.root());
        }
        accumulator = "";
      }
      else if (!accumulator.isEmpty()) {
        accumulator += text.charAt(i);
      }
      else if (accumulator.isEmpty() && node.getDepth() > prevDepth) {
        accumulator = node.getString();
      }
    }
    List<String> tokenization = new ArrayList<>();
    for (String match : matches) {
      int index = text.indexOf(match);
      assert (index >= 0);
      if (index > 0) tokenization.add(text.substring(0, index));
      tokenization.add(text.substring(index, index + match.length()));
      text = text.substring(index + match.length());
    }
    tokenization.add(text);
    return tokenization;
  }
477347
public long text(int group, String text) {
		return send(new Filter(false, String.valueOf(group)), null, "text", text);
	}
1343442
public Span trim(final CharSequence text) {

    int newStartOffset = getStart();

    for (int i = getStart(); i < getEnd()
        && StringUtil.isWhitespace(text.charAt(i)); i++) {
      newStartOffset++;
    }

    int newEndOffset = getEnd();
    for (int i = getEnd(); i > getStart()
        && StringUtil.isWhitespace(text.charAt(i - 1)); i--) {
      newEndOffset--;
    }

    if (newStartOffset == getStart() && newEndOffset == getEnd()) {
      return this;
    } else if (newStartOffset > newEndOffset) {
      return new Span(getStart(), getStart(), getType());
    } else {
      return new Span(newStartOffset, newEndOffset, getType());
    }
  }
1128743
private String[] splitInTwo(String text)	{
	  Matcher matcher = LINE_SEPARATOR.matcher(text);
	  while (matcher.find()){
		  int start = matcher.start();
		  if (start > 0 && start < text.length())	{
			  String segment = text.substring(0, start);
			  if (segment.trim().length() > 0)
				  return new String[] {text.substring(0, start), text.substring(start)};
		  }
	  }
	  return new String[] {text};
  }
776552
public static List<String> splitAndTrimAsList(String text, String sep) {
        ArrayList<String> answer = new ArrayList<>();
        if (text != null && text.length() > 0) {
            for (String v : text.split(sep)) {
                String trim = v.trim();
                if (trim.length() > 0) {
                    answer.add(trim);
                }
            }
        }
        return answer;
    }
825094
public Collection<Token> tokenize(String text)
    {

        Collection<Token> tokens = new ArrayList<Token>();

        Collection<Emit> collectedEmits = parseText(text);
        // 下面是最长分词的关键
        IntervalTree intervalTree = new IntervalTree((List<Intervalable>) (List<?>) collectedEmits);
        intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
        // 移除结束

        int lastCollectedPosition = -1;
        for (Emit emit : collectedEmits)
        {
            if (emit.getStart() - lastCollectedPosition > 1)
            {
                tokens.add(createFragment(emit, text, lastCollectedPosition));
            }
            tokens.add(createMatch(emit, text));
            lastCollectedPosition = emit.getEnd();
        }
        if (text.length() - lastCollectedPosition > 1)
        {
            tokens.add(createFragment(null, text, lastCollectedPosition));
        }

        return tokens;
    }