Tokenizes the given {@code sentence} into words and analyzes it,
and then disambiguates POS tags.
@param sentence sentence to be analyzed
767691
public AnalyzedSentence getAnalyzedSentence(String sentence) throws IOException {
    SimpleInputSentence cacheKey = new SimpleInputSentence(sentence, language);
    AnalyzedSentence cachedSentence = cache != null ? cache.getIfPresent(cacheKey) : null;
    if (cachedSentence != null) {
      return cachedSentence;
    } else {
      AnalyzedSentence raw = getRawAnalyzedSentence(sentence);
      AnalyzedSentence disambig = language.getDisambiguator().disambiguate(raw);
      AnalyzedSentence analyzedSentence = new AnalyzedSentence(disambig.getTokens(), raw.getTokens());
      if (language.getPostDisambiguationChunker() != null) {
        language.getPostDisambiguationChunker().addChunkTags(Arrays.asList(analyzedSentence.getTokens()));
      }
      if (cache != null) {
        cache.put(cacheKey, analyzedSentence);
      }
      return analyzedSentence;
    }
  }
415064
public Token findVerb(Sentence sentence) {
    List<SyntacticChunk> syntChunks = sentence.getSyntacticChunks();

    for (int i = 0; i < syntChunks.size(); i++) {
      String tag = syntChunks.get(i).getTag();

      if (tag.equals("P") || tag.equals("MV") || tag.equals("PMV")
          || tag.equals("AUX") || tag.equals("PAUX"))
        return syntChunks.get(i).getTokens().get(0);
    }

    return null;
  }
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
1370626
static List<GoogleToken> getGoogleTokens(AnalyzedSentence sentence, boolean addStartToken, Tokenizer wordTokenizer) {
    List<GoogleToken> result = new ArrayList<>();
    if (addStartToken) {
      result.add(new GoogleToken(LanguageModel.GOOGLE_SENTENCE_START, 0, 0));
    }
    List<String> tokens = wordTokenizer.tokenize(sentence.getText());
    int startPos = 0;
    for (String token : tokens) {
      if (!StringTools.isWhitespace(token)) {
        int endPos = startPos + token.length();
        Set<AnalyzedToken> pos = findOriginalAnalyzedTokens(sentence, startPos, endPos);
        GoogleToken gToken = new GoogleToken(token, startPos, endPos, pos);
        result.add(gToken);
      }
      startPos += token.length();
    }
    return result;
  }
159071
def sentence_ends(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.ends(SENTENCES)
428298
def split_sentences(text):
    
    sentence_delimiters = re.compile(u'[.!?,;:\t\\\\"\\(\\)\\\'\u2019\u2013]|\\s\\-\\s')
    sentences = sentence_delimiters.split(text)
    return sentences
64120
public static List<String> splitSentence2(String text, SentenceDetector sd) {
        List<String> sentences = new ArrayList<String>();
        try {
            int sentenceOffsets[] = sd.sentPosDetect(text);

            int begin = 0;
            int end = 0;
            for (int i = 0; i < sentenceOffsets.length; i++) {
                end = begin
                        + (text.substring(begin, sentenceOffsets[i]).trim())
                                .length();
                sentences.add(text.substring(begin, end));
                begin = sentenceOffsets[i];
            }
        } catch (Exception e) {
            LOG.warn("failed to extract sentences from text '" + text + "'", e);
        }
        return sentences;
    }
376546
protected void segmentAfterRule(String sentence, String normalized, List<String> wordList)
    {
        if (!enableRuleBasedSegment)
        {
            segmenter.segment(sentence, normalized, wordList);
            return;
        }
        int start = 0;
        int end = start;
        byte preType = typeTable[normalized.charAt(end)];
        byte curType;
        while (++end < normalized.length())
        {
            curType = typeTable[normalized.charAt(end)];
            if (curType != preType)
            {
                if (preType == CharType.CT_NUM)
                {
                    // 浮点数识别
                    if ("，,．.".indexOf(normalized.charAt(end)) != -1)
                    {
                        if (end + 1 < normalized.length())
                        {
                            if (typeTable[normalized.charAt(end + 1)] == CharType.CT_NUM)
                            {
                                continue;
                            }
                        }
                    }
                    else if ("年月日时分秒".indexOf(normalized.charAt(end)) != -1)
                    {
                        preType = curType; // 交给统计分词
                        continue;
                    }
                }
                pushPiece(sentence, normalized, start, end, preType, wordList);
                start = end;
            }
            preType = curType;
        }
        if (end == normalized.length())
            pushPiece(sentence, normalized, start, end, preType, wordList);
    }
1082043
public void tokenizeToKAF(final KAFDocument kaf) throws IOException {

    int noSents = 0;
    int noParas = 1;
    
    List<List<Token>> tokens;

    if (isNoSeg) {
      String[] sentences = text.toArray(new String[text.size()]);
      tokens = tokenizer.tokenize(sentences);
    } else {
      final String[] sentences = segmenter.segmentSentence();
      tokens = tokenizer.tokenize(sentences);
    }
    for (final List<Token> tokenizedSentence : tokens) {
      noSents = noSents + 1;
      for (final Token token : tokenizedSentence) {
        if (token.getTokenValue().equals(RuleBasedSegmenter.PARAGRAPH)) {
          ++noParas;
          // TODO debug this
          if (noSents < noParas) {
            ++noSents;
          }
        } else {
          final WF wf = kaf.newWF(token.startOffset(), token.getTokenValue(),
              noSents);
          wf.setLength(token.tokenLength());
          wf.setPara(noParas);
        }
      }
    }
  }
96631
def p_contextualize_sentence(self, t):
        """"""

        if len(t) == 5:
            t[0] = contextualize(t[3], line=t.lineno(1))
        else:
            t[0] = contextualize(t[4], t[2], line=t.lineno(1))
1077156
public Pair<CoreMap, GrammaticalStructure> parseForCoref(String sentence) {
        List<CoreLabel> tokens = stanfordTokenize(sentence);
        Tree tree = parser.parse(tokens);
        GrammaticalStructure gs = tagDependencies(tree, true);
        tagPOS(tokens);
        tagLemma(tokens);
        tagNamedEntity(tokens);

        CoreMap result = new ArrayCoreMap();
        result.set(CoreAnnotations.TokensAnnotation.class, tokens);
        result.set(TreeCoreAnnotations.TreeAnnotation.class, tree);

        GrammaticalStructure.Extras extras = GrammaticalStructure.Extras.NONE;
        SemanticGraph deps = SemanticGraphFactory.generateCollapsedDependencies(gs, extras);
        SemanticGraph uncollapsedDeps = SemanticGraphFactory.generateUncollapsedDependencies(gs, extras);
        SemanticGraph ccDeps = SemanticGraphFactory.generateCCProcessedDependencies(gs, extras);

        result.set(SemanticGraphCoreAnnotations.CollapsedDependenciesAnnotation.class, deps);
        result.set(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class, uncollapsedDeps);
        result.set(SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation.class, ccDeps);
        return new ImmutablePair<>(result, gs);
    }
1205482
public synchronized List<Sentence> sentences(String text) {
        requireValidHandle();
        Libvoikko lib = getLib();
        List<Sentence> result = new ArrayList<Sentence>();
        if (!isValidInput(text)) {
            result.add(new Sentence(text, SentenceStartType.NONE));
            return result;
        }
        byte[] textBytes = s2n(text);
        int textLen = textBytes.length - 1;
        SizeTByReference sentenceLenByRef = new SizeTByReference();
        while (textLen > 0) {
            int sentenceTypeInt = lib.voikkoNextSentenceStartCstr(handle, textBytes, new SizeT(textLen), sentenceLenByRef);
            int sentenceLen = sentenceLenByRef.getValue().intValue();
            SentenceStartType sentenceType = SentenceStartType.values()[sentenceTypeInt];
            String tokenText = text.substring(0, sentenceLen);
            result.add(new Sentence(tokenText, sentenceType));
            text = text.substring(sentenceLen);
            textBytes = s2n(text);
            textLen = textBytes.length - 1;
        }
        return result;
    }
1102847
private void pushPiece(String sentence, String normalized, int start, int end, byte preType, List<String> wordList)
    {
        if (preType == CharType.CT_CHINESE)
        {
            segmenter.segment(sentence.substring(start, end), normalized.substring(start, end), wordList);
        }
        else
        {
            wordList.add(sentence.substring(start, end));
        }
    }
1250578
def remove_tokens_of_sentence(self,sentence_id):
        
        nodes_to_remove = set()
        for wf in self:
            if wf.get_sent() == sentence_id:
                nodes_to_remove.add(wf.get_node())
        
        for node in nodes_to_remove:
            self.node.remove(node)
687798
def sentence_starts(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.starts(SENTENCES)
951157
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      String namedEntityTag = null;
      int startOfNamedEntity = -1;

      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = leaf.getLexeme().split("_");
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3) {
          String tag = leaf.getFunctionalTag();
          if (tags != null) {
            if (tags.contains(tag)) {
              namedEntityTag = leaf.getFunctionalTag();
            }
          } else {
            namedEntityTag = leaf.getFunctionalTag();
          }
        }
      }

      if (namedEntityTag != null) {
        startOfNamedEntity = sentence.size();
      }

      sentence.addAll(Arrays.asList(leaf.getLexeme().split("_")));

      if (namedEntityTag != null) {
        names
            .add(new Span(startOfNamedEntity, sentence.size(), namedEntityTag));
      }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        right = leaf.getLexeme();
        String c = PortugueseContractionUtility.toContraction(
            leftContractionPart, right);

        if (c != null) {
          sentence.add(c);
        } else {
          System.err.println("missing " + leftContractionPart + " + " + right);
          sentence.add(leftContractionPart);
          sentence.add(right);
        }

      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
925596
def _add_sentence(self, sentence, ignore_traces=True):
        
        self.sentences.append(self._node_id)
        # add edge from document root to sentence root
        self.add_edge(self.root, self._node_id, edge_type=dg.EdgeTypes.dominance_relation)
        self._parse_sentencetree(sentence, ignore_traces=ignore_traces)
        self._node_id += 1