Tokenizes the given {@code sentence} into words and analyzes it.
This is the same as {@link #getAnalyzedSentence(String)} but it does not run
the disambiguator.
@param sentence sentence to be analyzed
@since 0.9.8
17324
public AnalyzedSentence getRawAnalyzedSentence(String sentence) throws IOException {
    List<String> tokens = language.getWordTokenizer().tokenize(sentence);
    Map<Integer, String> softHyphenTokens = replaceSoftHyphens(tokens);

    List<AnalyzedTokenReadings> aTokens = language.getTagger().tag(tokens);
    if (language.getChunker() != null) {
      language.getChunker().addChunkTags(aTokens);
    }
    int numTokens = aTokens.size();
    int posFix = 0; 
    for (int i = 1; i < numTokens; i++) {
      aTokens.get(i).setWhitespaceBefore(aTokens.get(i - 1).isWhitespace());
      aTokens.get(i).setStartPos(aTokens.get(i).getStartPos() + posFix);
      if (!softHyphenTokens.isEmpty() && softHyphenTokens.get(i) != null) {
        aTokens.get(i).addReading(language.getTagger().createToken(softHyphenTokens.get(i), null));
        posFix += softHyphenTokens.get(i).length() - aTokens.get(i).getToken().length();
      }
    }
        
    AnalyzedTokenReadings[] tokenArray = new AnalyzedTokenReadings[tokens.size() + 1];
    AnalyzedToken[] startTokenArray = new AnalyzedToken[1];
    int toArrayCount = 0;
    AnalyzedToken sentenceStartToken = new AnalyzedToken("", SENTENCE_START_TAGNAME, null);
    startTokenArray[0] = sentenceStartToken;
    tokenArray[toArrayCount++] = new AnalyzedTokenReadings(startTokenArray, 0);
    int startPos = 0;
    for (AnalyzedTokenReadings posTag : aTokens) {
      posTag.setStartPos(startPos);
      tokenArray[toArrayCount++] = posTag;
      startPos += posTag.getToken().length();
    }

    // add additional tags
    int lastToken = toArrayCount - 1;
    // make SENT_END appear at last not whitespace token
    for (int i = 0; i < toArrayCount - 1; i++) {
      if (!tokenArray[lastToken - i].isWhitespace()) {
        lastToken -= i;
        break;
      }
    }

    tokenArray[lastToken].setSentEnd();

    if (tokenArray.length == lastToken + 1 && tokenArray[lastToken].isLinebreak()) {
      tokenArray[lastToken].setParagraphEnd();
    }
    return new AnalyzedSentence(tokenArray);
  }
200092
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      int startOfNamedEntity = -1;
      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = underlinePattern.split(leaf.getLexeme());
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
              
              String[] conts = ContractionUtility.expand(lexemes[i]);
              if(conts != null) {
                int end = sentence.size();
                int start = end - 1;
                Span s = new Span(start, end, "default");
                names.add(s);
                {
                  Span[] ss = {s};
                  System.out.println(Arrays.toString(Span.spansToStrings(ss, sentence.toArray(new String[sentence.size()]))));
                }
              }
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        // if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3)
        // {
        // String tag = leaf.getFunctionalTag();
        // if (tags != null) {
        // if (tags.contains(tag)) {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // } else {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // }
      }

      // if (contraction) {
      // startOfNamedEntity = sentence.size();
      // }
      //
      sentence.addAll(Arrays.asList(leaf.getLexeme()));// .split("_")
      //
      // if (contraction) {
      // names
      // .add(new Span(startOfNamedEntity, sentence.size()));
      // }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        String[] parts = underlinePattern.split(leaf.getLexeme());
        if(parts != null) {
          // try to join only the first
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, parts[0]);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          }
          
          for (int i = 1; i < parts.length; i++) {
            sentence.add(parts[i]);
          }
        } else {
          right = leaf.getLexeme();
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, right);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          } else {
            System.err.println("missing " + leftContractionPart + " + " + right);
            sentence.add(leftContractionPart);
            sentence.add(right);
          }
        }
      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
200455
public void add(Sentence sentence) {
        if (sentence.getAlphabet() != alphabet) {
            throw new IllegalArgumentException("Alphabets do not match.");
        }
        addSentenceToAlphabet(sentence);
        sents.add(sentence);
        numTokens += sentence.size();
    }
640102
private boolean isRealSentence(AnalyzedSentence sentence) {
    AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
    if (tokens.length > 0) {
      AnalyzedTokenReadings lastToken = tokens[tokens.length - 1];
      return lastToken.hasPosTag("PKT") && StringUtils.equalsAny(lastToken.getToken(), ".", "?", "!");
    }
    return false;
  }
358873
def get_sentence(self, offset: int) -> BioCSentence or None:
        
        for sentence in self.sentences:
            if sentence.offset == offset:
                return sentence
        return None
773625
def add_sentence(self, sentence):
        
        sent_root_id = sentence.attrib[add_ns('id')]
        # add edge from document root to sentence root
        self.add_edge(self.root, sent_root_id, edge_type=dg.EdgeTypes.dominance_relation)
        self.sentences.append(sent_root_id)

        sentence_token_ids = []

        if 'span' in sentence.attrib:
            # the sentence element looks like this:
            # <sentence xml:id="s144" span="s144_1..s144_23">, which means that
            # there might be <word> elements which belong to this sentence but
            # occur after the closing </sentence> element
            span_str = sentence.attrib['span']
            sentence_token_ids.extend(convert_spanstring(span_str))
        else:  # a normal sentence element, i.e. <sentence xml:id="s143">
            for descendant in sentence.iterdescendants('word'):
                sentence_token_ids.append(self.get_element_id(descendant))

        self.node[sent_root_id]['tokens'] = sentence_token_ids
591426
public List<String> segment(final String sentence, final String normalized)
    {
        final List<String> wordList = new LinkedList<String>();
        segment(sentence, normalized, wordList);
        return wordList;
    }
882612
def enrich_sentences_with_NLP(self, all_sentences):
        
        if not self.has_NLP_support():
            raise NotImplementedError(
                f"Language {self.lang} not available in spacy beyond tokenization"
            )

        if len(all_sentences) == 0:
            return  # Nothing to parse

        if self.model.has_pipe("sentencizer"):
            self.model.remove_pipe("sentencizer")
            self.logger.debug(
                f"Removed sentencizer ('sentencizer') from model. "
                f"Now in pipeline: {self.model.pipe_names}"
            )

        if self.model.has_pipe("sentence_boundary_detector"):
            self.model.remove_pipe(name="sentence_boundary_detector")
        self.model.add_pipe(
            set_custom_boundary, before="parser", name="sentence_boundary_detector"
        )

        sentence_batches = self._split_sentences_by_char_limit(
            all_sentences, self.model.max_length
        )

        # TODO: We could do this in parallel. Test speedup in the future
        for sentence_batch in sentence_batches:
            custom_tokenizer = TokenPreservingTokenizer(self.model.vocab)
            # we circumvent redundant tokenization by using a custom
            # tokenizer that directly uses the already separated words
            # of each sentence as tokens
            doc = custom_tokenizer(sentence_batch)
            doc.user_data = sentence_batch
            for name, proc in self.model.pipeline:  # iterate over components in order
                doc = proc(doc)

            try:
                assert doc.is_parsed
            except Exception:
                self.logger.exception(f"{doc} was not parsed")

            for sent, current_sentence_obj in zip(doc.sents, sentence_batch):
                parts = defaultdict(list)

                for i, token in enumerate(sent):
                    parts["lemmas"].append(token.lemma_)
                    parts["pos_tags"].append(token.tag_)
                    parts["ner_tags"].append(
                        token.ent_type_ if token.ent_type_ else "O"
                    )
                    head_idx = (
                        0 if token.head is token else token.head.i - sent[0].i + 1
                    )
                    parts["dep_parents"].append(head_idx)
                    parts["dep_labels"].append(token.dep_)
                current_sentence_obj.pos_tags = parts["pos_tags"]
                current_sentence_obj.lemmas = parts["lemmas"]
                current_sentence_obj.ner_tags = parts["ner_tags"]
                current_sentence_obj.dep_parents = parts["dep_parents"]
                current_sentence_obj.dep_labels = parts["dep_labels"]
                yield current_sentence_obj
1024967
public List<Tokenizer.Token> tokenize(String sentence) {
      Sequence seq = Sequence.create(Chars.asList(sentence.toCharArray())
                                          .stream()
                                          .map(Object::toString));
      Labeling labels = model.label(seq);

      int n = sentence.length();
      int[][] spans = new int[n + 1][2];
      double[] best = new double[n + 1];
      best[0] = 1d;

      for (int i = 1; i <= n; i++) {
         for (int j = Math.max(0, i - maxSpanSize); j < i; j++) {
            int w = i - j;
            String span = sentence.substring(j, j + w);
            double score = scoreWord(j, j + w, span, labels);
            double segmentScore = multi(best[i - w], score);
            logFiner("'{0}' > '{1}' : {2} : {3}",
                     sentence.substring(spans[i - w][0], spans[i - w][1]),
                     span,
                     segmentScore,
                     best[i]);
            if (segmentScore >= best[i]) {
               best[i] = segmentScore;
               spans[i][0] = j;
               spans[i][1] = j + w;
            }
         }
      }
      int i = n;
      LinkedList<Tokenizer.Token> tokens = new LinkedList<>();
      while (i > 0) {
         String token = StringUtils.trim(sentence.substring(spans[i][0], spans[i][1]));
         if (!StringUtils.isNullOrBlank(token)) {
            tokens.addFirst(
               new Tokenizer.Token(token, TokenType.UNKNOWN, spans[i][0], spans[i][0] + token.length(), 0));
         }
         i = i - (spans[i][1] - spans[i][0]);
      }
      i = 0;
      for (Tokenizer.Token token : tokens) {
         token.index = i;
         i++;
      }

      return tokens;
   }
585359
public final AnalyzedSentence replace(AnalyzedSentence sentence) throws IOException {
    DisambiguationPatternRuleReplacer replacer = new DisambiguationPatternRuleReplacer(this);
    return replacer.replace(sentence);
  }
242056
public static void normalizeDoubleQuotes(final List<Token> sentence,
      final String lang) {

    boolean isLeft = true;
    for (int i = 0; i < sentence.size(); i++) {
      if (lang.equalsIgnoreCase("en")) {
        final Matcher doubleAsciiQuoteMatcher = doubleAsciiQuote
            .matcher(sentence.get(i).getTokenValue());
        final Matcher singleAsciiQuoteMatcher = singleAsciiQuote
            .matcher(sentence.get(i).getTokenValue());
        // if current token is "
        if (doubleAsciiQuoteMatcher.find()) {
          if (isLeft && i < sentence.size() - 1 && doubleAsciiQuoteAlphaNumeric
              .matcher(sentence.get(i + 1).getTokenValue()).find()) {
            sentence.get(i).setTokenValue("``");
            isLeft = false;
          } else if (!isLeft) {
            sentence.get(i).setTokenValue("''");
            isLeft = true;
          }
        } else if (singleAsciiQuoteMatcher.find()) {
          if (i < sentence.size() - 2
              && sentence.get(i + 1).getTokenValue().matches("[A-Za-z]")
              && sentence.get(i + 2).getTokenValue()
                  .matches("[^ \t\n\r\u00A0\u00B6]")) {
            sentence.get(i).setTokenValue("`");
          }
        }
      }
    }
  }
675930
@RequestMapping(produces = MediaType.APPLICATION_JSON_VALUE, value = "/parse", method = RequestMethod.GET)
  public String parse(@RequestParam("sentence") String sentence, HttpServletRequest request) {
    if (sentence == null || sentence.trim().isEmpty()) {
      return StringUtils.EMPTY;
    }

    sentence = sentence.trim();
    LOGGER.info("Parse [" + sentence + "]");
    DTree tree = PARSER.parse(sentence);

    DTreeEntity parseTreeEntity = new DTreeEntity(tree, "SAMPLE_AUTHOR");
    dNodeEntityRepository.save(parseTreeEntity.dNodeEntities);
    dTreeEntityRepository.save(parseTreeEntity);

    return "[" + toJSON(tree.get(0)) + "]";
  }
1116852
def tokenize(self,
                 sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_preprocess, bool, bool, bool, bool, Callable[[str], text_type])->Union[List[text_type], TokenizedSenetence]
        
        assert isinstance(normalize, bool)
        assert isinstance(sentence, text_type)
        normalized_sentence = func_normalizer(sentence)
        result = self.call_juman_interface(normalized_sentence)

        token_objects = [
            self.__extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects
            )
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects
1267428
def tokenize(self, sentence_str):
        '''
        
        '''
        mt = MeCab.Tagger("-Owakati")
        wordlist = mt.parse(sentence_str)
        token_list = wordlist.rstrip(" \n").split(" ")
        return token_list
769757
@Override
    protected List<Term> segSentence(char[] sentence)
    {
        if (sentence.length == 0) return Collections.emptyList();
        List<Term> termList = roughSegSentence(sentence);
        if (!(config.ner || config.useCustomDictionary || config.speechTagging))
            return termList;
        List<Vertex> vertexList = toVertexList(termList, true);
        if (config.speechTagging)
        {
            Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
            int i = 0;
            for (Term term : termList)
            {
                if (term.nature != null) term.nature = vertexList.get(i + 1).guessNature();
                ++i;
            }
        }
        if (config.useCustomDictionary)
        {
            combineByCustomDictionary(vertexList);
            termList = convert(vertexList, config.offset);
        }
        return termList;
    }
453299
private void improveGermanSentences(JCas jcas) {
		/* 
		 * these POS tag sequences will decide whether we want to merge two sentences
		 * that have (supposedly wrongfully) been split.
		 */
		HashSet<String[]> posRules = new HashSet<String[]>();
		posRules.add(new String[] {"CARD", "\\$.", "NN"});
		posRules.add(new String[] {"CARD", "\\$.", "NE"});
		
		FSIterator sentIter = jcas.getAnnotationIndex(Sentence.type).iterator();
		
		// compare two sentences at a time in order to have access to all POS tags
		HashSet<HashSet<Sentence>> toMerge = new HashSet<HashSet<Sentence>>();
		Sentence prevSent = null, thisSent = null;
		while(sentIter.hasNext()) {
			if(thisSent == null) {
				thisSent = (Sentence) sentIter.next();
				continue;
			}
			
			prevSent = thisSent;
			thisSent = (Sentence) sentIter.next();
			/* 
			 * select the last two tokens within the previous sentence as well as the
			 * first of the current one and check for matches.
			 */
			Token penultimateToken = null, ultimateToken = null, firstToken = null;
			FSIterator tokIter = jcas.getAnnotationIndex(Token.type).subiterator(thisSent);
			if(tokIter.hasNext()) {
				firstToken = (Token) tokIter.next();
			}
			
			tokIter = jcas.getAnnotationIndex(Token.type).subiterator(prevSent);
			while(tokIter.hasNext()) {
				if(ultimateToken == null) {
					ultimateToken = (Token) tokIter.next();
					continue;
				}
				penultimateToken = ultimateToken;
				ultimateToken = (Token) tokIter.next();
			}
			
			// check that all tokens for further analysis are present. if not: skip
			if(penultimateToken == null || ultimateToken == null || firstToken == null) {
				continue;
			}
			
			// check rules, memorize sentences to be merged
			for(String[] posRule : posRules) {
				/* 
				 * either one of the pre-defined POS rules fit, or the first token's 
				 * covered text begins with lower case characters.
				 */
				if((penultimateToken.getPos() != null && penultimateToken.getPos().matches(posRule[0]) &&
						ultimateToken.getPos() != null && ultimateToken.getPos().matches(posRule[1]) &&
						firstToken.getPos() != null && firstToken.getPos().matches(posRule[2]))
						||
						(firstToken.getCoveredText().matches("^[a-z/].*"))) {
					/* 
					 * check whether one of the previous candidate pairs already 
					 * contains one of our sentences.
					 */
					Boolean candidateExisted = false;
					for(HashSet<Sentence> mergeCandidate : toMerge) {
						if(mergeCandidate.contains(thisSent) || mergeCandidate.contains(prevSent)) {
							// we add both here because sets ignore duplicates
							mergeCandidate.add(prevSent);
							mergeCandidate.add(thisSent);
							
							candidateExisted = true;
							break;
						}
					}
					
					/* 
					 * if one of the sentences was not already to be merged with another,
					 * create a new merge candidate set
					 */
					if(!candidateExisted) {
						HashSet<Sentence> newCandidate = new HashSet<Sentence>();
						newCandidate.add(prevSent);
						newCandidate.add(thisSent);
						
						toMerge.add(newCandidate);
					}
					
					break; // don't need to do the next rules; already merging.
				}
			}
		}
		
		// iterate over the previously collected merge candidates
		
		for(HashSet<Sentence> mergeCandidate : toMerge) {
			// find the earliest beginning and latest end for the set of sentences
			Integer beginIndex = Integer.MAX_VALUE, endIndex = Integer.MIN_VALUE;

			Sentence mergedSent = new Sentence(jcas);
			for(Sentence s : mergeCandidate) {
				if(s.getBegin() < beginIndex) {
					beginIndex = s.getBegin();
				}
				
				if(s.getEnd() > endIndex) {
					endIndex = s.getEnd();
				}
				
				s.removeFromIndexes();
			}
			
			// set values, add to jcas
			mergedSent.setBegin(beginIndex);
			mergedSent.setEnd(endIndex);
			mergedSent.addToIndexes();
		}
	}
466789
protected void segment(final String sentence, final String normalized, final List<String> wordList, final List<CoreDictionary.Attribute> attributeList)
    {
        if (attributeList != null)
        {
            final int[] offset = new int[]{0};
            CustomDictionary.parseLongestText(sentence, new AhoCorasickDoubleArrayTrie.IHit<CoreDictionary.Attribute>()
            {
                @Override
                public void hit(int begin, int end, CoreDictionary.Attribute value)
                {
                    if (begin != offset[0])
                    {
                        segmentAfterRule(sentence.substring(offset[0], begin), normalized.substring(offset[0], begin), wordList);
                    }
                    while (attributeList.size() < wordList.size())
                        attributeList.add(null);
                    wordList.add(sentence.substring(begin, end));
                    attributeList.add(value);
                    assert wordList.size() == attributeList.size() : "词语列表与属性列表不等长";
                    offset[0] = end;
                }
            });
            if (offset[0] != sentence.length())
            {
                segmentAfterRule(sentence.substring(offset[0]), normalized.substring(offset[0]), wordList);
            }
        }
        else
        {
            segmentAfterRule(sentence, normalized, wordList);
        }
    }