Returns a list of sentences from the given string.
            Punctuation marks are separated from each word by a space.
8470
def find_tokens(self, string, **kwargs):
        
        # "The cat purs." => ["The cat purs ."]
        return find_tokens(string,
                punctuation = kwargs.get(  "punctuation", PUNCTUATION),
              abbreviations = kwargs.get("abbreviations", ABBREVIATIONS),
                    replace = kwargs.get(      "replace", replacements),
                  linebreak = r"\n{2,}")
1144300
static private String[] getStringList( String str )
    {
        if ( str == null )
        {
            return null;
        }
        List<String> list = new ArrayList<>();
        int i = 0;
        int length = str.length();
        StringBuffer sb = null;
        while ( i < length )
        {
            char ch = str.charAt( i );
            if ( ch == ' ' )
            {
                // A space was hit. Add string to list
                if ( sb != null )
                {
                    list.add( sb.toString() );
                    sb = null;
                }
            }
            else if ( ch == '\\' )
            {
                // It is a delimiter. Add next character
                if ( i + 1 < length )
                {
                    ch = str.charAt( ++i );
                    if ( sb == null )
                    {
                        sb = new StringBuffer();
                    }
                    sb.append( ch );
                }
            }
            else
            {
                if ( sb == null )
                {
                    sb = new StringBuffer();
                }
                sb.append( ch );
            }
            i++; // Next character
        }
        // Make sure to add the last part to the list too
        if ( sb != null )
        {
            list.add( sb.toString() );
        }
        if ( list.size() == 0 )
        {
            return null;
        }
        String[] results = new String[list.size()];
        return list.toArray( results );
    }
1110641
public static List<String> split(String input, String punctuationChars, String quoteChars) {
		return split(input, punctuationChars, quoteChars, false, false, false);
	}
312471
public static List<Sentence> splitSentences(CharSequence text) {
    return JavaConversions.seqAsJavaList(
        TwitterKoreanProcessor.splitSentences(text)
    );
  }
469306
def auto_undent(string):
    
    lines = string.splitlines()
    while lines[0].strip() == "":
        lines = lines[1:]
        if not lines:
            return []
    spaces = len(lines[0]) - len(lines[0].lstrip(" "))
    out = []
    for line in lines:
        num_spaces = len(line) - len(line.lstrip(" "))
        out.append(line[min(spaces, num_spaces) :])
    return out
1206439
public static List<List<Term>> seg2sentence(String text)
    {
        List<List<Term>> sentenceList = SEGMENT.seg2sentence(text);
        for (List<Term> sentence : sentenceList)
        {
            ListIterator<Term> listIterator = sentence.listIterator();
            while (listIterator.hasNext())
            {
                if (!CoreStopWordDictionary.shouldInclude(listIterator.next()))
                {
                    listIterator.remove();
                }
            }
        }

        return sentenceList;
    }
1001039
def sentence_spans(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.spans(SENTENCES)
376152
def sentences(self, index = None):
        """"""
        if index is None:
            return self.select(Sentence,None,True,[Quote])
        else:
            if index < 0:
                index = sum(t.count(Sentence,None,True,[Quote]) for t in self.data) + index
            for t in self.data:
                for i,e in enumerate(t.select(Sentence,None,True,[Quote])) :
                    if i == index:
                        return e
            raise IndexError
775621
def split_sentences(s, pad=0):
    
    sentences = []
    for index, sentence in enumerate(s.split('. ')):
        padding = ''
        if index > 0:
            padding = ' ' * (pad + 1)
        if sentence.endswith('.'):
            sentence = sentence[:-1]
        sentences.append('%s %s.' % (padding, sentence.strip()))
    return "\n".join(sentences)
384157
def get_reply_states(self, string, dataset):
        
        words = get_words(string)
        if not words:
            return []
        long_word = 4
        long_words = [word for word in words if len(word) >= long_word]
        short_words = [word for word in words if len(word) < long_word]
        for words in (long_words, short_words):
            ret = [
                states
                for states in (
                    self.storage.get_states(dataset, word)
                    for word in words
                )
                if states
            ]
            if ret:
                return ret
        return []
1010683
private String[] segment(final String builtText) {

    // these are fine because they do not affect offsets
    String line = builtText.trim();
    line = RuleBasedTokenizer.doubleSpaces.matcher(line).replaceAll(" ");

    if (this.isHardParagraph) {
      // convert every (spurious) paragraph in newlines and keep them
      line = paragraph.matcher(line).replaceAll("\n$1");
    } else {
      // end of sentence markers, paragraph mark and beginning of link
      line = endPunctLinkPara.matcher(line).replaceAll("$1\n$2$3");
      line = conventionalPara.matcher(line).replaceAll("$1\n$2$3");
      line = endInsideQuotesPara.matcher(line).replaceAll("$1\n$3$4");
      line = multiDotsParaStarters.matcher(line).replaceAll("$1\n$2$3");
      // remove spurious paragraphs
      line = alphaNumParaLowerNum.matcher(line).replaceAll("$1 $3");
      line = spuriousParagraph.matcher(line).replaceAll(" $2");
    }
    // non-period end of sentence markers (?!) followed by sentence starters.
    line = noPeriodSpaceEnd.matcher(line).replaceAll("$1\n$2");
    // multi-dots followed by sentence starters
    line = multiDotsSpaceStarters.matcher(line).replaceAll("$1\n$2");
    // end of sentence inside quotes or brackets
    line = endInsideQuotesSpace.matcher(line).replaceAll("$1\n$2");
    // end of sentence marker, sentence starter punctuation and upper case.
    line = punctSpaceUpper.matcher(line).replaceAll("$1\n$2");
    // end of sentence markers, maybe space or paragraph mark and beginning of
    // link
    line = endPunctLinkSpace.matcher(line).replaceAll("$1\n$2");

    // special case of multi-punctuation
    line = punctSpaceMultiPunct.matcher(line).replaceAll("$1\n$2");

    // non breaker segments everything else with some exceptions
    final String[] lines = line.split("\n");
    final String[] sentences = this.nonBreaker.segmenterExceptions(lines);
    return sentences;
  }
840903
protected List<String> getTokens(String text){
		String[] tokens = text.toLowerCase().split("[\\s\\p{Punct}]");
		List<String> tokenList = new ArrayList<String>(tokens.length);
		for (int i = 0; i < tokens.length; i++){
			if (tokens[i].trim().length() < 1){
				continue;
			}
			tokenList.add(tokens[i].trim());
		}
		return tokenList;
	}
1048963
def tokenize_paragraphs(cls, text):
        """"""
        paragraphs = []
        paragraphs_first_pass = text.split('\n')
        for p in paragraphs_first_pass:
            paragraphs_second_pass = re.split('\s{4,}', p)
            paragraphs += paragraphs_second_pass

        # Remove empty strings from list
        paragraphs = [p for p in paragraphs if p]
        return paragraphs
487910
def sentences(total)
      list = []
      total.times do
        word_count = rand(word_count_range_in_a_sentence)
        sentence_len = word_count.times.inject(0){ |sum| sum + rand(char_count_range_in_a_word) }
        sentence_key = bsearch(sentence_map_keys, sentence_len)
        unless sentence_key
          sentence_key = sentence_map.keys.min
        end
        list << sentence_map[sentence_key].sample
      end
      return list.join(@line_break)
    end
1240575
def from_string(cls, string):
        
        items = shlex.split(string, **cls.split_args)
        return cls(items)
1185163
def split_strings_in_list_retain_spaces(orig_list):
    
    temp_list = list()
    for line in orig_list:
        line_split = __re.split(r'(\s+)', line)
        temp_list.append(line_split)

    return temp_list
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist