Segment the rest of the text taking into account some exceptions for
periods as sentence breakers. It decides when a period marks an end of
sentence.

@param lines
the segmented sentences so far
@return all the segmented sentences
1981
public String[] segmenterExceptions(final String[] lines) {
    final List<String> sentences = new ArrayList<>();
    for (final String line : lines) {
      final String segmentedLine = segmenterNonBreaker(line);
      final String[] lineSentences = segmentedLine.split("\n");
      for (final String lineSentence : lineSentences) {
        sentences.add(lineSentence);
      }
    }
    return sentences.toArray(new String[sentences.size()]);
  }
101456
private String getTextLine(String[] textLines, int line)
	{
		return (isIgnoreTrailingWhiteSpaces()) ? textLines[line].trim() : textLines[line];
	}
137194
def _read_parsed(self, lines):
        
        self.log(u"Parsing fragments from parsed text format")
        pairs = []
        for line in lines:
            pieces = line.split(gc.PARSED_TEXT_SEPARATOR)
            if len(pieces) == 2:
                identifier = pieces[0].strip()
                text = pieces[1].strip()
                if len(identifier) > 0:
                    pairs.append((identifier, [text]))
        self._create_text_fragments(pairs)
984511
def read_from_list_with_ids(self, lines):
        
        self.log(u"Reading text fragments from list with ids")
        self._create_text_fragments([(line[0], [line[1]]) for line in lines])
1368778
private String getLines(SlotsLines lines) {
    StringBuilder builder = new StringBuilder();
    for (MiniCapability cap : lines.getLinesType()) {
      String icon = cap.getIcon();
      String version = cap.getVersion();
      builder.append("<p>");
      if (version != null) {
        builder.append("v:").append(version);
      }
      for (TestSlot s : lines.getLine(cap)) {
        builder.append(getSingleSlotHtml(s, icon));
      }
      builder.append("</p>");
    }
    return builder.toString();
  }
1374357
private String trim(String line) {
        char[] chars = line.toCharArray();
        int len = chars.length;

        while (len > 0) {
            if (!Character.isWhitespace(chars[len - 1])) {
                break;
            }
            len--;
        }

        return line.substring(0, len);
    }
1048293
private List<String> searchClasses(List<String> lines) {
        List<String> result = new LinkedList<String>();
        for (String line : lines) {
            if (isCSharpClass(line)) {
                result.add(getClassName(line));
            }
        }
        return result;
    }
1058391
def lines
      @lines ||= begin
        all_lines = @buffer.source_lines
        last_token_line = tokens.any? ? tokens.last.line : all_lines.size
        result = []
        all_lines.each_with_index do |line, ix|
          break if ix >= last_token_line && line == '__END__'

          result << line
        end
        result
      end
    end
1344003
public void seg(String text){
        segmentation.seg(text).parallelStream().forEach(word -> {
            //停用词过滤
            if(isRemoveStopWord() && StopWord.is(word.getText())){
                return;
            }
            statistics(word, 1, statisticsMap);
        });
    }
1024967
public List<Tokenizer.Token> tokenize(String sentence) {
      Sequence seq = Sequence.create(Chars.asList(sentence.toCharArray())
                                          .stream()
                                          .map(Object::toString));
      Labeling labels = model.label(seq);

      int n = sentence.length();
      int[][] spans = new int[n + 1][2];
      double[] best = new double[n + 1];
      best[0] = 1d;

      for (int i = 1; i <= n; i++) {
         for (int j = Math.max(0, i - maxSpanSize); j < i; j++) {
            int w = i - j;
            String span = sentence.substring(j, j + w);
            double score = scoreWord(j, j + w, span, labels);
            double segmentScore = multi(best[i - w], score);
            logFiner("'{0}' > '{1}' : {2} : {3}",
                     sentence.substring(spans[i - w][0], spans[i - w][1]),
                     span,
                     segmentScore,
                     best[i]);
            if (segmentScore >= best[i]) {
               best[i] = segmentScore;
               spans[i][0] = j;
               spans[i][1] = j + w;
            }
         }
      }
      int i = n;
      LinkedList<Tokenizer.Token> tokens = new LinkedList<>();
      while (i > 0) {
         String token = StringUtils.trim(sentence.substring(spans[i][0], spans[i][1]));
         if (!StringUtils.isNullOrBlank(token)) {
            tokens.addFirst(
               new Tokenizer.Token(token, TokenType.UNKNOWN, spans[i][0], spans[i][0] + token.length(), 0));
         }
         i = i - (spans[i][1] - spans[i][0]);
      }
      i = 0;
      for (Tokenizer.Token token : tokens) {
         token.index = i;
         i++;
      }

      return tokens;
   }
1226430
@Override
    public final MatchResult matches(final List<String> lines) {
        for (int i = 0; i < lines.size(); i++) {
            if (matches(i + 1, lines.get(i))) return new MatchResult(true, this, i);
        }
        return NO_MATCH;
    }
842523
public List<String> parse(String line) throws ParseException {
    List<String> cols = new ArrayList<>();
    boolean afterQuote = false;
    boolean afterEscape = false;
    boolean afterSeparator = false;
    boolean outerQuote = false;
    boolean afterEndField = false;
    StringBuilder sb = new StringBuilder();

    for (int i = 0; ; ++i) {
      char c = (i == line.length()) ? EOT : line.charAt(i);
      if (afterEndField && c != separator && c != EOT) {
        if (!trim || c != ' ') {
          throw new ParseException("Extraneous character after end of quoted field", i);
        }
      } else if (afterEscape) {
        if (c == EOT) {
          // [abc\^]
          throw new ParseException("Escape not followed by a character", i);
        } else {
          // [abc\x...] => abcx...
          afterEscape = false;
          sb.append(c);
        }
      } else if (c == separator) {
        if (outerQuote && !afterQuote) {
          // ["abc,...] => abc,...
          sb.append(c);
          afterSeparator = false;
        } else {
          // [abc,...] => {abc, ...}
          endCol(cols, sb, i, outerQuote, afterQuote);
          afterQuote = afterEscape = afterEndField = outerQuote = false;
          afterSeparator = true;
        }
      } else if (c == EOT) {
        if (sb.length() != 0 || afterSeparator || outerQuote) {
          // [...,abc^] => {..., abc}
          // [...,^] => {..., ""}
          endCol(cols, sb, i, outerQuote, afterQuote);
        }
        return cols;
      } else if (c == escape) {
        // [...\...]
        afterEscape = true;
        afterSeparator = false;
      } else if (c == quote) {
        if (afterQuote && outerQuote) {
          // Two consecutive quotes inside quoted string, so the pair has to be internal
          // (the second quote cannot be terminating the field).
          sb.append(quote);
          afterQuote = false;
        } else if (sb.length() == 0 && !outerQuote) {
          outerQuote = true;
        } else if (sb.length() != 0 && !outerQuote) {
          // Fields that are not quote-delimited cannot have any internal quotes,
          // unless they are escaped which was already handled.
          throw new ParseException(escape == NUL
              ? "Unescaped quote inside non-quote-delimited field"
              : "Quote inside non-quote-delimited field",
              i);
        } else {
          afterQuote = true;
        }
        afterSeparator = false;
      } else if (c == ' ' && trim
          && ((!outerQuote && sb.length() == 0) // Trimmed space before field
              || afterEndField)) {              // Trimmed space after field
      } else if (c == ' ' && trim && afterQuote) {
        afterEndField = true;
      } else if (outerQuote && afterQuote) {
        // ["abc"x]
        throw new ParseException("Extraneous character after end of quoted field", i);
      } else {
        // Common character.
        sb.append(c);
        afterSeparator = false;
      }
    }
  }
1169192
public String wordSegment(String text){
		String ret = text;
		
		if (vnSegmenter == null) return ret;
		ret = vnSegmenter.segmenting(ret);
		return ret;
	}
1183327
public List<Term> seg(char[] text)
    {
        assert text != null;
        if (HanLP.Config.Normalization)
        {
            CharTable.normalization(text);
        }
        return segSentence(text);
    }
1010139
private String[] getTokens(String line) {

    // these are fine because they do not affect offsets
    line = line.trim();
    line = doubleSpaces.matcher(line).replaceAll(" ");
    // remove non printable stuff
    line = asciiHex.matcher(line).replaceAll(" ");
    line = generalBlankPunctuation.matcher(line).replaceAll(" ");

    // separate question and exclamation marks
    line = qexc.matcher(line).replaceAll(" $1 ");
    // separate dash if before or after space
    line = spaceDashSpace.matcher(line).replaceAll(" $1 ");
    // tokenize everything but these characters [^\p{Alnum}s.'`,-?!/]
    line = specials.matcher(line).replaceAll(" $1 ");

    // do not separate multidots
    line = generateMultidots(line);

    // separate "," except if within numbers (1,200)
    line = noDigitComma.matcher(line).replaceAll("$1 $2");
    line = commaNoDigit.matcher(line).replaceAll("$1 $2");
    // separate pre and post digit
    line = digitCommaNoDigit.matcher(line).replaceAll("$1 $2 $3");
    line = noDigitCommaDigit.matcher(line).replaceAll("$1 $2 $3");

    // contractions it's, l'agila, c'est, don't
    line = treatContractions(line);
    // exceptions for period tokenization
    line = this.nonBreaker.TokenizerNonBreaker(line);

    // restore multidots
    line = restoreMultidots(line);
    // urls
    // TODO normalize URLs after tokenization for offsets
    line = detokenizeURLs(line);
    line = beginLink.matcher(line).replaceAll("$1://");
    line = endLink.matcher(line).replaceAll("$1$2");

    // these are fine because they do not affect offsets
    line = line.trim();
    line = doubleSpaces.matcher(line).replaceAll(" ");
    line = detokenParagraphs.matcher(line).replaceAll("$1$2");

    if (DEBUG) {
      System.out.println("->Tokens:" + line);
    }
    final String[] tokens = line.split(" ");
    return tokens;
  }
809464
public static String[] getDataFromLine(final String line, final String seperator)
	{
		final String[] splittedLine = line.split(seperator);
		return splittedLine;
	}
1097154
public synchronized String getRecentLines() {
        StringBuilder sb = new StringBuilder();
        for (String line : ringBuffer) {
            if (sb.length() > 0) {
                sb.append('\n');
            }
            sb.append(line);
        }
        return sb.toString();
    }