sentence to words

@param   sentence
@return  List<List<IWord>>
@throws  IOException
16843
List<List<IWord>> sentenceTokenize(List<Sentence> sentence) throws IOException
    {
        List<List<IWord>> senWords = new ArrayList<List<IWord>>();
        for ( Sentence sen : sentence ) {
            List<IWord> words = new ArrayList<IWord>();
            wordSeg.reset(new StringReader(sen.getValue()));
            IWord word = null;
            while ( (word = wordSeg.next()) != null ) {
                words.add(word);
            }
            
            senWords.add(words);
        }
        
        return senWords;
    }
1064469
def features(sentence, i):
    

    word = sentence[i]

    yield "word:{}" + word.lower()

    if word[0].isupper():
        yield "CAP"

    if i > 0:
        yield "word-1:{}" + sentence[i - 1].lower()
        if i > 1:
            yield "word-2:{}" + sentence[i - 2].lower()
    if i + 1 < len(sentence):
        yield "word+1:{}" + sentence[i + 1].lower()
        if i + 2 < len(sentence):
            yield "word+2:{}" + sentence[i + 2].lower()
767691
public AnalyzedSentence getAnalyzedSentence(String sentence) throws IOException {
    SimpleInputSentence cacheKey = new SimpleInputSentence(sentence, language);
    AnalyzedSentence cachedSentence = cache != null ? cache.getIfPresent(cacheKey) : null;
    if (cachedSentence != null) {
      return cachedSentence;
    } else {
      AnalyzedSentence raw = getRawAnalyzedSentence(sentence);
      AnalyzedSentence disambig = language.getDisambiguator().disambiguate(raw);
      AnalyzedSentence analyzedSentence = new AnalyzedSentence(disambig.getTokens(), raw.getTokens());
      if (language.getPostDisambiguationChunker() != null) {
        language.getPostDisambiguationChunker().addChunkTags(Arrays.asList(analyzedSentence.getTokens()));
      }
      if (cache != null) {
        cache.put(cacheKey, analyzedSentence);
      }
      return analyzedSentence;
    }
  }
1287688
public List<Token> findNouns(Sentence sentence) {
    List<Token> nouns = new ArrayList<Token>();

    List<SyntacticChunk> syntChunks = sentence.getSyntacticChunks();

    for (int i = 0; i < syntChunks.size(); i++) {
      String tag = syntChunks.get(i).getTag();

      if (tag.equals("PIV") || tag.equals("ACC") || tag.equals("SC")) {

        for (Token token : syntChunks.get(i).getTokens()) {

          if (token.getPOSTag().equals("n") || token.getPOSTag().equals("pron-pers") || token.getPOSTag().equals("prop")) {
              nouns.add(token);
            }
          }
        }
      }

    return nouns;
  }
1230158
public static List<String[]> convertSentenceToNER(Sentence sentence, NERTagSet tagSet)
    {
        List<String[]> collector = new LinkedList<String[]>();
        Set<String> nerLabels = tagSet.nerLabels;
        for (IWord word : sentence.wordList)
        {
            if (word instanceof CompoundWord)
            {
                List<Word> wordList = ((CompoundWord) word).innerList;
                Word[] words = wordList.toArray(new Word[0]);

                if (nerLabels.contains(word.getLabel()))
                {
                    collector.add(new String[]{words[0].value, words[0].label, tagSet.B_TAG_PREFIX + word.getLabel()});
                    for (int i = 1; i < words.length - 1; i++)
                    {
                        collector.add(new String[]{words[i].value, words[i].label, tagSet.M_TAG_PREFIX + word.getLabel()});
                    }
                    collector.add(new String[]{words[words.length - 1].value, words[words.length - 1].label,
                        tagSet.E_TAG_PREFIX + word.getLabel()});
                }
                else
                {
                    for (Word w : words)
                    {
                        collector.add(new String[]{w.value, w.label, tagSet.O_TAG});
                    }
                }
            }
            else
            {
                if (nerLabels.contains(word.getLabel()))
                {
                    // 单个实体
                    collector.add(new String[]{word.getValue(), word.getLabel(), tagSet.S_TAG});
                }
                else
                {
                    collector.add(new String[]{word.getValue(), word.getLabel(), tagSet.O_TAG});
                }
            }
        }
        return collector;
    }
1107991
public static List<Word> getWords(String text) {
        List<Word> result = new ArrayList<>();
        List<Word> words = WordParser.parse(text);
        for (Word word : words) {
            result.add(word);
        }
        return result;
    }
425128
public static ArrayList<Word> toUntaggedList(String... words) {
    ArrayList<Word> sent = new ArrayList<Word>();
    for (String str : words) {
      sent.add(new Word(str));
    }
    return sent;
  }
1281768
@Override
  public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens)
  throws IOException {
    List<AnalyzedToken> taggerTokens;
    List<AnalyzedToken> lowerTaggerTokens;
    List<AnalyzedToken> upperTaggerTokens;
    List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
    int pos = 0;

    Matcher matcher;
    for (String word : sentenceTokens) {
      String probeWord = word;

      // This loop happens when we need to retry probing the dictionary
      // which happens rarely when trying to remove suffixes -mañ, -se, etc.
      for (;;) {
        List<AnalyzedToken> l = new ArrayList<>();
        String lowerWord = probeWord.toLowerCase(conversionLocale);
        taggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(probeWord));
        lowerTaggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(lowerWord));
        boolean isLowercase = probeWord.equals(lowerWord);

        // Normal case.
        addTokens(taggerTokens, l);

        if (!isLowercase) {
          // Lowercase.
          addTokens(lowerTaggerTokens, l);
        }

        // Uppercase.
        if (lowerTaggerTokens.isEmpty() && taggerTokens.isEmpty()) {
          if (isLowercase) {
            upperTaggerTokens = asAnalyzedTokenListForTaggedWords(word,
                getWordTagger().tag(StringTools.uppercaseFirstChar(probeWord)));
            if (!upperTaggerTokens.isEmpty()) {
              addTokens(upperTaggerTokens, l);
            }
          }
          if (l.isEmpty()) {
            if ((matcher = patternSuffix.matcher(probeWord)).find()) {
              // Remove the suffix and probe dictionary again.
              // So given a word such as "xxx-mañ", we're going to
              // try to probe the dictionary again with "xxx" this time.
              probeWord = matcher.group(1);
              continue;
            }
            l.add(new AnalyzedToken(word, null, null));
          }
        }
        tokenReadings.add(new AnalyzedTokenReadings(l, pos));
        pos += word.length();
        break;
      }
    }

    return tokenReadings;

  }
207298
function tokenize_sentence(sentence) {
  var tokenized = tokenizer.tokenize(sentence);
  logger.info("tokenize_sentence: " + tokenized);
  return(tokenized);
}
817839
def sentences(self):
        """"""
        sents = []
        spans = self.sentence_tokenizer.span_tokenize(self.text)
        for span in spans:
            sent = Sentence(
                text=self.text[span[0]:span[1]],
                start=span[0],
                end=span[1],
                word_tokenizer=self.word_tokenizer,
                lexicon=self.lexicon,
                abbreviation_detector=self.abbreviation_detector,
                pos_tagger=self.pos_tagger,
                ner_tagger=self.ner_tagger,
                parsers=self.parsers,
                document=self.document
            )
            sents.append(sent)
        return sents
140952
def _create_sentence_objects(self):
        """"""
        sentence_objects = []
        sentences = sent_tokenize(self.raw, tokenizer=self.tokenizer)
        char_index = 0  # Keeps track of character index within the blob
        for sent in sentences:
            # Compute the start and end indices of the sentence
            # within the blob. This only works if the sentence splitter
            # does not perform any character replacements or changes to
            # white space.
            # Working: NLTKPunktTokenizer
            # Not working: PatternTokenizer
            try:
                start_index = self.raw.index(sent, char_index)
                char_index += len(sent)
                end_index = start_index + len(sent)
            except ValueError:
                start_index = None
                end_index = None
            # Sentences share the same models as their parent blob
            s = Sentence(
                sent,
                start_index=start_index,
                end_index=end_index,
                tokenizer=self.tokenizer,
                np_extractor=self.np_extractor,
                pos_tagger=self.pos_tagger,
                analyzer=self.analyzer,
                parser=self.parser,
                classifier=self.classifier)
            sentence_objects.append(s)
        return sentence_objects
17324
public AnalyzedSentence getRawAnalyzedSentence(String sentence) throws IOException {
    List<String> tokens = language.getWordTokenizer().tokenize(sentence);
    Map<Integer, String> softHyphenTokens = replaceSoftHyphens(tokens);

    List<AnalyzedTokenReadings> aTokens = language.getTagger().tag(tokens);
    if (language.getChunker() != null) {
      language.getChunker().addChunkTags(aTokens);
    }
    int numTokens = aTokens.size();
    int posFix = 0; 
    for (int i = 1; i < numTokens; i++) {
      aTokens.get(i).setWhitespaceBefore(aTokens.get(i - 1).isWhitespace());
      aTokens.get(i).setStartPos(aTokens.get(i).getStartPos() + posFix);
      if (!softHyphenTokens.isEmpty() && softHyphenTokens.get(i) != null) {
        aTokens.get(i).addReading(language.getTagger().createToken(softHyphenTokens.get(i), null));
        posFix += softHyphenTokens.get(i).length() - aTokens.get(i).getToken().length();
      }
    }
        
    AnalyzedTokenReadings[] tokenArray = new AnalyzedTokenReadings[tokens.size() + 1];
    AnalyzedToken[] startTokenArray = new AnalyzedToken[1];
    int toArrayCount = 0;
    AnalyzedToken sentenceStartToken = new AnalyzedToken("", SENTENCE_START_TAGNAME, null);
    startTokenArray[0] = sentenceStartToken;
    tokenArray[toArrayCount++] = new AnalyzedTokenReadings(startTokenArray, 0);
    int startPos = 0;
    for (AnalyzedTokenReadings posTag : aTokens) {
      posTag.setStartPos(startPos);
      tokenArray[toArrayCount++] = posTag;
      startPos += posTag.getToken().length();
    }

    // add additional tags
    int lastToken = toArrayCount - 1;
    // make SENT_END appear at last not whitespace token
    for (int i = 0; i < toArrayCount - 1; i++) {
      if (!tokenArray[lastToken - i].isWhitespace()) {
        lastToken -= i;
        break;
      }
    }

    tokenArray[lastToken].setSentEnd();

    if (tokenArray.length == lastToken + 1 && tokenArray[lastToken].isLinebreak()) {
      tokenArray[lastToken].setParagraphEnd();
    }
    return new AnalyzedSentence(tokenArray);
  }
487910
def sentences(total)
      list = []
      total.times do
        word_count = rand(word_count_range_in_a_sentence)
        sentence_len = word_count.times.inject(0){ |sum| sum + rand(char_count_range_in_a_word) }
        sentence_key = bsearch(sentence_map_keys, sentence_len)
        unless sentence_key
          sentence_key = sentence_map.keys.min
        end
        list << sentence_map[sentence_key].sample
      end
      return list.join(@line_break)
    end
200455
public void add(Sentence sentence) {
        if (sentence.getAlphabet() != alphabet) {
            throw new IllegalArgumentException("Alphabets do not match.");
        }
        addSentenceToAlphabet(sentence);
        sents.add(sentence);
        numTokens += sentence.size();
    }
638516
def _sentence(self, words):
        """"""
        db = self.database

        # Generate 2 words to start a sentence with
        seed = random.randint(0, db['word_count'] - 3)
        seed_word, next_word = db['words'][seed], db['words'][seed + 1]
        w1, w2 = seed_word, next_word

        # Generate the complete sentence
        sentence = []
        for i in range(0, words - 1):
            sentence.append(w1)
            w1, w2 = w2, random.choice(db['freqs'][(w1, w2)])
        sentence.append(w2)

        # Make the sentence respectable
        sentence = ' '.join(sentence)

        # Capitalize the sentence
        sentence = sentence.capitalize()

        # Remove additional sentence ending puntuation
        sentence = sentence.replace('.', '')
        sentence = sentence.replace('!', '')
        sentence = sentence.replace('?', '')
        sentence = sentence.replace(':', '')

        # Remove quote tags
        sentence = sentence.replace('.', '')
        sentence = sentence.replace('!', '')
        sentence = sentence.replace('?', '')
        sentence = sentence.replace(':', '')
        sentence = sentence.replace('"', '')

        # If the last character is not an alphanumeric remove it
        sentence = re.sub('[^a-zA-Z0-9]$', '', sentence)

        # Remove excess space
        sentence = re.sub('\s+', ' ', sentence)

        # Add a full stop
        sentence += '.'

        return sentence
1267428
def tokenize(self, sentence_str):
        '''
        
        '''
        mt = MeCab.Tagger("-Owakati")
        wordlist = mt.parse(sentence_str)
        token_list = wordlist.rstrip(" \n").split(" ")
        return token_list
1052752
def split_sentences(tokens):
    """"""
    begin = 0
    for i, token in enumerate(tokens):
        if is_end_of_sentence(tokens, i):
            yield tokens[begin:i+1]
            begin = i+1
    if begin <= len(tokens)-1:
        yield tokens[begin:]