Get the word separator to split words in mplain format.

        :rtype: string
350445
def _mplain_word_separator(self):
        
        word_separator = gf.safe_get(self.parameters, gc.PPN_TASK_IS_TEXT_MPLAIN_WORD_SEPARATOR, u" ")
        if (word_separator is None) or (word_separator == "space"):
            return u" "
        elif word_separator == "equal":
            return u"="
        elif word_separator == "pipe":
            return u"|"
        elif word_separator == "tab":
            return u"\u0009"
        return word_separator
338920
def _run_split_on_punc(self, text):
        """"""
        chars = list(text)
        i = 0
        start_new_word = True
        output = []
        while i < len(chars):
            char = chars[i]
            if self._is_punctuation(char):
                output.append([char])
                start_new_word = True
            else:
                if start_new_word:
                    output.append([])
                start_new_word = False
                output[-1].append(char)
            i += 1

        return [''.join(x) for x in output]
753515
def rejoin(string, separator_regexp=None, spaced=False):
    
    strings = split_and_strip(string)
    if separator_regexp is None:
        separator_regexp = _default_separator()
    joiner = spaced and '%s ' % separator_regexp or separator_regexp
    return joiner.join(strings)
902807
def word_spans(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return self.spans(WORDS)
1205378
function split(str, separator) {
    separator = separator || '';
    if (_.isString(str)) {
        return str.split(separator);
    } else {
        return str;
    }
}
429091
def _split_basic(string):
    
    tuples = []
    for word in string.split(','):
        # Attempt to split on colon
        parts = word.split(':', 2)
        key, modifier_fn, explode = parts[0], _identity, False
        if len(parts) > 1:
            modifier_fn = functools.partial(
                _truncate, num_chars=int(parts[1]))
        if word[len(word) - 1] == '*':
            key = word[:len(word) - 1]
            explode = True
        tuples.append((key, modifier_fn, explode))
    return tuples
822540
protected function splitWords()
    {
        $indexedWords = [];
        $offset = 0;

        $words = explode($this->separator, $this->title);
        foreach ($words as $word) {
            if (mb_strlen($word, $this->encoding) == 0) {
                continue;
            }

            $wordIndex = $this->getWordIndex($word, $offset);

            if ($this->hasDash($word)) {
                $word = TextFormatter::titleCase($word, '-');
                $this->rebuildTitle($wordIndex, $word);
            }

            $indexedWords[$wordIndex] = $word;
            $offset += mb_strlen($word, $this->encoding) + 1; // plus space
        }

        return $this->indexedWords = $indexedWords;
    }
318961
def separator_line(cls, sep='-', size=10):
        
        if cls.intty():
            cls.echo(sep * size)
102873
def _split_line_with_offsets(line):
    
    for delimiter in re.finditer(r"[\.,:\;](?![^\s])", line):
        span = delimiter.span()
        line = line[:span[0]] + " " + line[span[1]:]

    for delimiter in re.finditer(r"[\"'\)\]\}>](?![^\.,\;:\"'\)\]\}>\s])",
                                 line):
        span = delimiter.span()
        line = line[:span[0]] + " " + line[span[1]:]

    for delimiter in re.finditer(r"(?<![^\.,\;:\"'\(\[\{<\s])[\"'\(\[\{<]",
                                 line):
        span = delimiter.span()
        line = line[:span[0]] + " " + line[span[1]:]

    # Treat hyphen separated words as separate words
    line = line.replace("-", " ")

    # Remove backticks
    line = line.replace("`", " ")

    for match in re.finditer(r"[^\s]+", line):
        content = match.group(0)
        if content.strip() != "":
            yield (match.span()[0], content)
238236
def splitter_support(py2enc):
    '''
    '''
    if sys.version < '3':
        def _fn_sentence(pattern, sentence):
            if REGEXTYPE == type(pattern):
                if pattern.flags & re.UNICODE:
                    return sentence.decode(py2enc)
                else:
                    return sentence
            else:
                return sentence
        def _fn_token2str(pattern):
            if REGEXTYPE == type(pattern):
                if pattern.flags & re.UNICODE:
                    def _fn(token):
                        return token.encode(py2enc)
                else:
                    def _fn(token):
                        return token
            else:
                def _fn(token):
                    return token
            return _fn
    else:
        def _fn_sentence(pattern, sentence):
            return sentence
        def _fn_token2str(pattern):
            def _fn(token):
                return token
            return _fn

    def _fn_tokenize_pattern(text, pattern):
        pos = 0
        sentence = _fn_sentence(pattern, text)
        postprocess = _fn_token2str(pattern)
        for m in re.finditer(pattern, sentence):
            if pos < m.start():
                token = postprocess(sentence[pos:m.start()])
                yield (token.strip(), False)
                pos = m.start()
            token = postprocess(sentence[pos:m.end()])
            yield (token.strip(), True)
            pos = m.end()
        if pos < len(sentence):
            token = postprocess(sentence[pos:])
            yield (token.strip(), False)

    def _fn_tokenize_features(text, features):
        acc = []
        acc.append((text.strip(), False))

        for feat in features:
            for i,e in enumerate(acc):
                if e[1]==False:
                    tmp = list(_fn_tokenize_pattern(e[0], feat))
                    if len(tmp) > 0:
                        acc.pop(i)
                        acc[i:i] = tmp
        return acc
                        
    return _fn_tokenize_pattern, _fn_tokenize_features
44129
def split(self, sep=None, maxsplit=None, regex=False):
        """"""
        if maxsplit is not None:
            raise NotImplementedError('no maxsplit yet')
        s = self.s
        if sep is None:
            sep = r'\s+'
        elif not regex:
            sep = re.escape(sep)
        matches = list(re.finditer(sep, s))
        return [self[start:end] for start, end in zip(
            [0] + [m.end() for m in matches],
            [m.start() for m in matches] + [len(s)])]
993257
def split_on(word: str, section: str) -> Tuple[str, str]:
    
    return word[:word.index(section)] + section, word[word.index(section) + len(section):]
487871
def delimit_words(self):
        
        bits_short = 8 - (len(self.buffer.getvalue()) % 8)
        
        #The string already falls on an byte boundary do nothing
        if bits_short == 0 or bits_short == 8:
            return None
        else:
            return self.binary_string(0, bits_short)
289532
def split(self, availWidth, availHeight):
        

        logger.debug("*** split (%f, %f)", availWidth, availHeight)

        splitted = []
        if self.splitIndex:
            text1 = self.text[:self.splitIndex]
            text2 = self.text[self.splitIndex:]
            p1 = Paragraph(Text(text1), self.style, debug=self.debug)
            p2 = Paragraph(Text(text2), self.style, debug=self.debug, splitted=True)
            splitted = [p1, p2]

            logger.debug("*** text1 %s / text %s", len(text1), len(text2))

        logger.debug('*** return %s', self.splitted)

        return splitted
1291747
static function wordSeparator()
    {
        if ( isset( $GLOBALS['eZCharTransform_wordSeparator'] ) )
        {
            return $GLOBALS['eZCharTransform_wordSeparator'];
        }
        else
        {
            $ini = eZINI::instance();
            $separator = strtolower( $ini->variable( "URLTranslator", "WordSeparator" ) );
            switch ( $separator )
            {
                case 'dash':
                    $separator = '-';
                    break;
                case 'underscore':
                    $separator = '_';
                    break;
                case 'space':
                    $separator = ' ';
                    break;
                default:
                    return '-';
            }
            $GLOBALS['eZCharTransform_wordSeparator'] = $separator;
            return $separator;
        }
    }
1218848
def _mpda(self, re_grammar, splitstring=0):
        
        cnfgrammar = CNFGenerator(re_grammar)

        if not self.alphabet:
            self._extract_alphabet(cnfgrammar)

        cnftopda = CnfPda(self.alphabet)
        productions = {}
        nonterminals = []
        nonterminals.append(cnfgrammar.init_symbol)
        for key in list(cnfgrammar.grammar_nonterminals):
            if key != cnfgrammar.init_symbol:
                nonterminals.append(key)
        for key in list(cnfgrammar.grammar_nonterminals):
            j = 0
            productions[key] = {}
            # print 'testing '+key
            for pair in cnfgrammar.grammar_rules:
                cnf_form = list(pair)
                if cnf_form[0] == key:
                    productions[key][j] = {}
                    if isinstance(cnf_form[1], type(())):
                        #            print list(p[1])
                        productions[key][j]['b0'] = list(cnf_form[1])[0]
                        productions[key][j]['b1'] = list(cnf_form[1])[1]
                    else:
                        #           print p[1]
                        productions[key][j]['a'] = cnf_form[1]
                    j = j + 1
        return cnftopda.initialize(
            nonterminals, productions, list(
                cnfgrammar.grammar_terminals), splitstring)
428298
def split_sentences(text):
    
    sentence_delimiters = re.compile(u'[.!?,;:\t\\\\"\\(\\)\\\'\u2019\u2013]|\\s\\-\\s')
    sentences = sentence_delimiters.split(text)
    return sentences