Static method __cutAll

@param string $sentence # input sentence
@param array  $options  # other options

@return array $words
8408
public static function __cutAll($sentence, $options = array())
    {
        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $words = array();

        $DAG = self::getDAG($sentence);
        $old_j = -1;

        foreach ($DAG as $k => $L) {
            if (count($L) == 1 && $k > $old_j) {
                $word = mb_substr($sentence, $k, (($L[0]-$k)+1), 'UTF-8');
                array_push($words, $word);
                $old_j = $L[0];
            } else {
                foreach ($L as $j) {
                    if ($j > $k) {
                        $word = mb_substr($sentence, $k, ($j-$k)+1, 'UTF-8');
                        array_push($words, $word);
                        $old_j = $j;
                    }
                }
            }
        }

        return $words;
    }
700786
public static function getWords($sentence, $count = 10) {
        $sentence = str_replace("\r", '', str_replace("\n", '', trim(strip_tags($sentence))));
        $words = explode(' ', $sentence);

        if (count($words) <= $count) {
            return $sentence;
        }

        $trimmed = '';
        for ($i = 0; $i < $count; $i++) {
            $trimmed .= $words[$i] . ' ';
        }

        $trimmed = trim($trimmed);

        return $trimmed;
    }
899173
private function getParagraphArr($sentences)
    {
        $wordsPer = $this->wordsPerParagraph;
        $sentenceAvg = $this->wordsPerSentence;
        $total = count($sentences);

        $paragraphs = [];
//        $pCount = 0;
        $currCount = 0;
        $curr = [];

        for ($i = 0; $i < $total; ++$i) {
            $s = $sentences[$i];
            $currCount += count($s);
            $curr[] = $s;
            if ($currCount >= ($wordsPer - round($sentenceAvg / 2.00)) || $i === $total - 1) {
                $currCount = 0;
                $paragraphs[] = $curr;
                $curr = [];
                //print_r($paragraphs);
            }
            //print_r($paragraphs);
        }

        return $paragraphs;
    }
1279793
public static function __cut($sentence, $options = array())
    {

        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $words = array();

        $viterbi_array = self::viterbi($sentence);
        $prob = $viterbi_array['prob'];
        $pos_list = $viterbi_array['pos_list'];

        $begin = 0;
        $next = 0;
        $len = mb_strlen($sentence, 'UTF-8');

        for ($i=0; $i<$len; $i++) {
            $char = mb_substr($sentence, $i, 1, 'UTF-8');
            $pos = $pos_list[$i];
            if ($pos=='B') {
                $begin = $i;
            } elseif ($pos=='E') {
                array_push($words, mb_substr($sentence, $begin, (($i+1)-$begin), 'UTF-8'));
                $next = $i+1;
            } elseif ($pos=='S') {
                array_push($words, $char);
                $next = $i+1;
            }
        }

        if ($next<$len) {
            array_push($words, mb_substr($sentence, $next, null, 'UTF-8'));
        }

        return $words;

    }
324543
public static function calc($sentence, $DAG, $options = array())
    {
        $N = mb_strlen($sentence, 'UTF-8');
        self::$route = array();
        self::$route[$N] = array($N => 1.0);
        for ($i=($N-1); $i>=0; $i--) {
            $candidates = array();
            foreach ($DAG[$i] as $x) {
                $w_c = mb_substr($sentence, $i, (($x+1)-$i), 'UTF-8');
                $previous_freq = current(self::$route[$x+1]);
                if (isset(self::$FREQ[$w_c])) {
                    $current_freq = (float) $previous_freq + self::$FREQ[$w_c];
                } else {
                    $current_freq = (float) $previous_freq + self::$min_freq;
                }
                $candidates[$x] = $current_freq;
            }
            arsort($candidates);
            $max_prob = reset($candidates);
            $max_key = key($candidates);
            self::$route[$i] = array($max_key => $max_prob);
        }

        return self::$route;
    }
501607
def _tokenize(sentence):
    ''''''

    _tokens = nltk.word_tokenize(sentence)
    tokens = [stemmer.stem(tk) for tk in _tokens]
    return tokens
915022
public function format_answer($answer, $context, $answerformat, $options = []) {

        if (is_object($options)) {
            $options = (array) $options;
        }

        if (empty($options['context'])) {
            $options['context'] = $context;
        }

        if (empty($options['para'])) {
            $options['para'] = true;
        }

        return format_text($answer, $answerformat, $options);
    }
663312
def cut_for_search(self, sentence, HMM=True):
        
        words = self.cut(sentence, HMM=HMM)
        for w in words:
            if len(w) > 2:
                for i in xrange(len(w) - 1):
                    gram2 = w[i:i + 2]
                    if self.FREQ.get(gram2):
                        yield gram2
            if len(w) > 3:
                for i in xrange(len(w) - 2):
                    gram3 = w[i:i + 3]
                    if self.FREQ.get(gram3):
                        yield gram3
            yield w
483594
public function tokenize(string $string) : array
    {
        $sentences = preg_split(self::SENTENCE_REGEX, $string) ?: [];

        $nGrams = [];

        foreach ($sentences as $sentence) {
            $words = [];

            preg_match_all(self::WORD_REGEX, $sentence, $words);

            $words = $words[0];

            $n = count($words);

            foreach ($words as $i => $word) {
                $p = min($n - $i, $this->max);

                for ($j = $this->min; $j <= $p; $j++) {
                    $nGram = $word;

                    for ($k = 1; $k < $j; $k++) {
                        $nGram .= self::SEPARATOR . $words[$i + $k];
                    }

                    $nGrams[] = $nGram;
                }
            }
        }

        return $nGrams;
    }
1218180
private function _punctuate(&$sentence)
    {
        $count = count($sentence);
        $sentence[$count - 1] .= '.';
        if ($count < 4)
            return ($sentence);
        $commas = $this->_numberOfCommas($count);
        for ($i = 1; $i <= $commas; $i++) {
            $index = (int)round($i * $count / ($commas + 1));
            if ($index < ($count - 1) && $index > 0)
                $sentence[$index] .= ',';
        }
        return ($sentence);
    }
503447
public function split($text)
    {
        $this->probabilityCalculator->setAbbreviations($this->getAbbreviations());

        $tokens = $this->lexer->run($text);
        $probabilities = $this->probabilityCalculator->calculate($tokens);
        $sentences = $this->sentenceBuilder->build($probabilities);

        return $sentences;
    }
1236920
public function getSnippets($text, $index, $terms, $options = array())
    {
        return $this->sphinx->BuildExcerpts($text, $index, $terms, $options);
    }
1051998
def slice(self, start, stop):
        
        s = Slice(token=self.token, language=self.language)
        for i, word in enumerate(self.words[start:stop]):
            # The easiest way to copy (part of) a sentence
            # is by unpacking all of the token tags and passing them to Sentence.append().
            p0 = word.string                                                       # WORD
            p1 = word.lemma                                                        # LEMMA
            p2 = word.type                                                         # POS
            p3 = word.chunk is not None and word.chunk.type or None                # CHUNK
            p4 = word.pnp is not None and "PNP" or None                            # PNP
            p5 = word.chunk is not None and unzip(0, word.chunk.relations) or None # REL            
            p6 = word.chunk is not None and unzip(1, word.chunk.relations) or None # ROLE
            p7 = word.chunk and word.chunk.anchor_id or None                       # ANCHOR
            p8 = word.chunk and word.chunk.start == start+i and BEGIN or None      # IOB
            p9 = word.custom_tags                                                  # User-defined tags.
            # If the given range does not contain the chunk head, remove the chunk tags.
            if word.chunk is not None and (word.chunk.stop > stop):
                p3, p4, p5, p6, p7, p8 = None, None, None, None, None, None
            # If the word starts the preposition, add the IOB B-prefix (i.e., B-PNP).
            if word.pnp is not None and word.pnp.start == start+i:
                p4 = BEGIN+"-"+"PNP"
            # If the given range does not contain the entire PNP, remove the PNP tags.
            # The range must contain the entire PNP, 
            # since it starts with the PP and ends with the chunk head (and is meaningless without these).
            if word.pnp is not None and (word.pnp.start < start or word.chunk.stop > stop):
                p4, p7 = None, None
            s.append(word=p0, lemma=p1, type=p2, chunk=p3, pnp=p4, relation=p5, role=p6, anchor=p7, iob=p8, custom=p9)
        s.parent = self
        s._start = start
        return s
358615
static List<GoogleToken> getGoogleTokens(String sentence, boolean addStartToken, Tokenizer wordTokenizer) {
    List<GoogleToken> result = new ArrayList<>();
    if (addStartToken) {
      result.add(new GoogleToken(LanguageModel.GOOGLE_SENTENCE_START, 0, 0));
    }
    List<String> tokens = wordTokenizer.tokenize(sentence);
    int startPos = 0;
    for (String token : tokens) {
      if (!StringTools.isWhitespace(token)) {
        result.add(new GoogleToken(token, startPos, startPos+token.length()));
      }
      startPos += token.length();
    }
    return result;
  }
1143991
public function getSnippet($text, $index, $terms, $options = array())
    {
        $results = $this->getSnippets(array($text), $index, $terms, $options);

        if ($results) {
            return $results[0];
        }

        return false;
    }
64120
public static List<String> splitSentence2(String text, SentenceDetector sd) {
        List<String> sentences = new ArrayList<String>();
        try {
            int sentenceOffsets[] = sd.sentPosDetect(text);

            int begin = 0;
            int end = 0;
            for (int i = 0; i < sentenceOffsets.length; i++) {
                end = begin
                        + (text.substring(begin, sentenceOffsets[i]).trim())
                                .length();
                sentences.add(text.substring(begin, end));
                begin = sentenceOffsets[i];
            }
        } catch (Exception e) {
            LOG.warn("failed to extract sentences from text '" + text + "'", e);
        }
        return sentences;
    }
272456
def _bpe_to_words(sentence, delimiter='@@'):
    """"""
    words = []
    word = ''
    delimiter_len = len(delimiter)
    for subwords in sentence:
        if len(subwords) >= delimiter_len and subwords[-delimiter_len:] == delimiter:
            word += subwords[:-delimiter_len]
        else:
            word += subwords
            words.append(word)
            word = ''
    return words