Give an list of character offsets for chunk, fix tokenization so tokenization occurs at
boundary of chunks
@param docAnnotation
@param chunkCharOffsets
14942
public static boolean fixChunkTokenBoundaries(CoreMap docAnnotation, List<IntPair> chunkCharOffsets)
  {
    // First identify any tokens that need to be fixed
    String text = docAnnotation.get(CoreAnnotations.TextAnnotation.class);
    List<CoreLabel> tokens = docAnnotation.get(CoreAnnotations.TokensAnnotation.class);
    List<CoreLabel> output = new ArrayList<CoreLabel>(tokens.size());
    int i = 0;
    CoreLabel token = tokens.get(i);
    for (IntPair offsets:chunkCharOffsets) {
      assert(token.beginPosition() >= 0);
      assert(token.endPosition() >= 0);
      int offsetBegin = offsets.getSource();
      int offsetEnd = offsets.getTarget();
      // Find tokens where token begins after chunk starts
      // and token ends after chunk starts
      while (offsetBegin < token.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class)
              || offsetBegin >= token.get(CoreAnnotations.CharacterOffsetEndAnnotation.class)) {
        output.add(token);
        i++;
        if (i >= tokens.size()) { return false; }
        token = tokens.get(i);
      }
      // offsetBegin is now >= token begin and < token end
      // go until we find a token that starts after our chunk has ended
      while (offsetEnd > token.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class)) {
        // Check if chunk includes token
        if (offsetBegin > token.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class)) {
          // Chunk starts in the middle of the token
          if (offsetEnd < token.get(CoreAnnotations.CharacterOffsetEndAnnotation.class)) {
            output.add(tokenFactory.makeToken(text.substring(token.beginPosition(), offsetBegin),
                    token.beginPosition(), offsetBegin-token.beginPosition()));
            output.add(tokenFactory.makeToken(text.substring(offsetBegin,offsetEnd),
                    offsetBegin, offsetEnd-offsetBegin));
            output.add(tokenFactory.makeToken(text.substring(offsetEnd,token.endPosition()),
                    offsetEnd, token.endPosition()-offsetEnd));
          } else {
            output.add(tokenFactory.makeToken(text.substring(token.beginPosition(), offsetBegin),
                    token.beginPosition(), offsetBegin-token.beginPosition()));
            output.add(tokenFactory.makeToken(text.substring(offsetBegin,token.endPosition()),
                    offsetBegin, token.endPosition()-offsetBegin));
          }
        } else if (offsetEnd < token.get(CoreAnnotations.CharacterOffsetEndAnnotation.class)) {
          output.add(tokenFactory.makeToken(text.substring(token.beginPosition(),offsetEnd),
                  token.beginPosition(), offsetEnd-token.beginPosition()));
          output.add(tokenFactory.makeToken(text.substring(offsetEnd,token.endPosition()), offsetEnd,
                  token.endPosition()-offsetEnd));
        } else {
          // success!  chunk contains token
          output.add(token);
        }
        i++;
        if (i >= tokens.size()) { return false; }
        token = tokens.get(i);
      }
    }
    // Add rest of the tokens
    for (; i < tokens.size(); i++) {
      token = tokens.get(i);
      output.add(token);
    }
    docAnnotation.set(CoreAnnotations.TokensAnnotation.class, output);
    return true;
  }
184559
protected void formatRegion(IXtextDocument document, int offset, int length) {
		try {
			final int startLineIndex = document.getLineOfOffset(previousSiblingChar(document, offset));
			final int endLineIndex = document.getLineOfOffset(offset + length);
			int regionLength = 0;
			for (int i = startLineIndex; i <= endLineIndex; ++i) {
				regionLength += document.getLineLength(i);
			}
			if (regionLength > 0) {
				final int startOffset = document.getLineOffset(startLineIndex);
				for (final IRegion region : document.computePartitioning(startOffset, regionLength)) {
					this.contentFormatter.format(document, region);
				}
			}
		} catch (BadLocationException exception) {
			Exceptions.sneakyThrow(exception);
		}
	}
1290293
private <T extends TokenBase> List<T> createTokenList(int offset, String text) {
        ArrayList<T> result = new ArrayList<>();

        ViterbiLattice lattice = viterbiBuilder.build(text);
        List<ViterbiNode> bestPath = viterbiSearcher.search(lattice);

        for (ViterbiNode node : bestPath) {
            int wordId = node.getWordId();
            if (node.getType() == ViterbiNode.Type.KNOWN && wordId == -1) { // Do not include BOS/EOS
                continue;
            }
            @SuppressWarnings("unchecked")
            T token = (T) tokenFactory.createToken(wordId, node.getSurface(), node.getType(),
                            offset + node.getStartIndex(), dictionaryMap.get(node.getType()));
            result.add(token);
        }

        return result;
    }
352446
def _get_source_chunks(self, input_text, language=None):
    
    chunks = ChunkList()
    seek = 0
    result = self._get_annotations(input_text, language=language)
    tokens = result['tokens']
    language = result['language']
    for i, token in enumerate(tokens):
      word = token['text']['content']
      begin_offset = token['text']['beginOffset']
      label = token['dependencyEdge']['label']
      pos = token['partOfSpeech']['tag']
      if begin_offset > seek:
        chunks.append(Chunk.space())
        seek = begin_offset
      chunk = Chunk(word, pos, label)
      if chunk.label in _DEPENDENT_LABEL:
        # Determining concatenating direction based on syntax dependency.
        chunk.dependency = i < token['dependencyEdge']['headTokenIndex']
      if chunk.is_punct():
        chunk.dependency = chunk.is_open_punct()
      chunks.append(chunk)
      seek += len(word)
    return chunks, language
1074119
public String tokenizeToCoNLLOffsets() {

    final StringBuilder sb = new StringBuilder();
    final String[] sentences = segmenter.segmentSentence();
    final List<List<Token>> tokens = tokenizer.tokenize(sentences);
    for (final List<Token> tokSentence : tokens) {
      for (final Token token : tokSentence) {
        String tokenValue = token.getTokenValue();
        if (tokenValue.equals(RuleBasedSegmenter.PARAGRAPH)) {
          tokenValue = DEFAULT_TOKEN_VALUE;
        }
        sb.append(tokenValue.trim()).append(DELIMITER).append(token.startOffset())
            .append(DELIMITER).append(token.tokenLength()).append(LINE_BREAK);
      }
      sb.append(LINE_BREAK);
    }
    return sb.toString();
  }
517610
private void checkForTokens( String content, int startOffset, int endOffset )
  {
    while( startOffset <= endOffset )
    {
      // skip the delimiters to find the start of a new token
      while( isDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        if( isError( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _error, true );
        }
        else if( isWarning( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _warning, true );
        }

        if( startOffset < endOffset )
        {
          startOffset++;
        }
        else
        {
          return;
        }
      }
      // Extract and process the entire token
      if( isQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else if( isCharQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else
      {
        startOffset = getOtherToken( content, startOffset, endOffset );
      }
    }
  }
1270883
public void buildAnnotationTypeSignature(XMLNode node, Content annotationInfoTree) {
        writer.addAnnotationTypeSignature(utils.modifiersToString(annotationType, true),
                annotationInfoTree);
    }
129726
public static function tokenHasMultipleBoundaryCharactersTogether($token, &$tokens, $i, &$originalTokens)
    {
        return $token[Tokenizer::TOKEN_TYPE] === Tokenizer::TOKEN_TYPE_BOUNDARY
        && self::tokenPreviousCharacterIsBoundary($tokens, $i)
        && self::tokenPreviousCharacterIsWhiteSpace($token, $originalTokens);
    }
892684
@Override
	public void parse(final byte[] pData, final Collection<TagAndLength> pTags) {
		Collection<AnnotationData> set = getAnnotationSet(pTags);
		BitUtils bit = new BitUtils(pData);
		Iterator<AnnotationData> it = set.iterator();
		while (it.hasNext()) {
			AnnotationData data = it.next();
			if (data.isSkip()) {
				bit.addCurrentBitIndex(data.getSize());
			} else {
				Object obj = DataFactory.getObject(data, bit);
				setField(data.getField(), this, obj);
			}
		}
	}
461542
SegmentChunk withNewOffset(long newOffset) {
        SegmentChunk ns = new SegmentChunk(this.name, newOffset);
        ns.setLength(getLength());
        if (isSealed()) {
            ns.markSealed();
        }

        if (!exists()) {
            ns.markInexistent();
        }

        return ns;
    }
1142333
@Override
    public void addSourceTargetSet(UnicodeSet inputFilter, UnicodeSet sourceSet, UnicodeSet targetSet) {
        // Each form consists of a prefix, suffix,
        // * radix, minimum digit count, and maximum digit count.  These
        // * values are stored as a five character header. ...
        UnicodeSet myFilter = getFilterAsUnicodeSet(inputFilter);
        UnicodeSet items = new UnicodeSet();
        StringBuilder buffer = new StringBuilder();
        for (int i = 0; spec[i] != END;) {
            // first 5 items are header
            int end = i + spec[i] + spec[i+1] + 5;
            int radix = spec[i+2];
            for (int j = 0; j < radix; ++j) {
                Utility.appendNumber(buffer, j, radix, 0);
            }
            // then add the characters
            for (int j = i + 5; j < end; ++j) {
                items.add(spec[j]);
            }
            // and go to next block
            i = end;
        }
        items.addAll(buffer.toString());
        items.retainAll(myFilter);

        if (items.size() > 0) {
            sourceSet.addAll(items);
            targetSet.addAll(0,0x10FFFF); // assume we can produce any character
        }
    }
1024151
public void process(CAS tcas) {

		String text = tcas.getDocumentText();
		Document document = new DocumentImpl(text);

		cogroo.analyze(document);

		for (Sentence sentence : document.getSentences()) {

			// create sentence annotations
			AnnotationFS sentenceAnn = tcas.createAnnotation(mSentenceType,
					sentence.getStart(), sentence.getEnd());
			tcas.getIndexRepository().addFS(sentenceAnn);

			int sentenceOffset = sentence.getStart();

			AnnotationFS[] tokenAnnotationArr = new AnnotationFS[sentence
					.getTokens().size()];
			int i = 0;
			for (Token token : sentence.getTokens()) {
				// create token annotations
				tokenAnnotationArr[i] = tcas.createAnnotation(mTokenType,
						sentenceOffset + token.getStart(), sentenceOffset
								+ token.getEnd());

				// add POSTag annotations
				tokenAnnotationArr[i].setStringValue(this.mPosFeature,
						token.getPOSTag());

				// add lexeme annotations
				tokenAnnotationArr[i].setStringValue(this.mLexemeFeature,
						token.getLexeme());

				// add lemma annotations
				StringArrayFS lemmas = tcas.createStringArrayFS(token
						.getLemmas().length);
				lemmas.copyFromArray(token.getLemmas(), 0, 0,
						token.getLemmas().length);
				tokenAnnotationArr[i].setFeatureValue(this.mLemmaFeature,
						lemmas);

				tokenAnnotationArr[i].setStringValue(this.mFeaturesFeature,
						token.getFeatures());

				tcas.getIndexRepository().addFS(tokenAnnotationArr[i]);
				i++;
			}

			// chunks
			for (Chunk chunk : sentence.getChunks()) {

				int start = sentence.getTokens().get(chunk.getStart())
						.getStart()
						+ sentenceOffset;
				int end = sentence.getTokens().get(chunk.getEnd() - 1).getEnd()
						+ sentenceOffset;

				AnnotationFS chunkAnn = tcas.createAnnotation(mChunkType,
						start, end);

				chunkAnn.setStringValue(mChunkFeature, chunk.getTag());

				if(chunk.getHeadIndex() >= 0) {
					chunkAnn.setFeatureValue(mChunkHead,
							tokenAnnotationArr[chunk.getHeadIndex()]);
				}

				tcas.getIndexRepository().addFS(chunkAnn);
			}

			// syntactic chunk
			for (SyntacticChunk sc : sentence.getSyntacticChunks()) {
				int start = sentence.getTokens().get(sc.getStart()).getStart()
						+ sentenceOffset;
				int end = sentence.getTokens().get(sc.getEnd() - 1).getEnd()
						+ sentenceOffset;

				AnnotationFS syntChunkAnn = tcas.createAnnotation(
						mSyntacticChunkType, start, end);

				syntChunkAnn.setStringValue(mSyntacticChunkFeature, sc.getTag());

				tcas.getIndexRepository().addFS(syntChunkAnn);
			}
		}
	}
424227
protected final void addDefaultAnnotationValues(String annotation, Map<CharSequence, Object> values) {
        if (annotation != null) {
            Map<String, Map<CharSequence, Object>> annotationDefaults = this.annotationDefaultValues;
            if (annotationDefaults == null) {
                this.annotationDefaultValues = new HashMap<>();
                annotationDefaults = this.annotationDefaultValues;
            }

            putValues(annotation, values, annotationDefaults);
        }
    }
654181
private void tokenize(JCas jcas) {
		// read tokenized text to add tokens to the jcas
		Logger.printDetail(component, "TreeTagger (tokenization) with: " + ttprops.abbFileName);
		
		EnumSet<Flag> flags = Flag.getSet(ttprops.languageSwitch);
		TreeTaggerTokenizer ttt; ttprops.abbFileName = "english-abbreviations";
		if(ttprops.abbFileName != null) {
			ttt = new TreeTaggerTokenizer(ttprops.rootPath + ttprops.fileSeparator + "lib" + ttprops.fileSeparator + ttprops.abbFileName, flags);
		} else {
			ttt = new TreeTaggerTokenizer(null, flags);
		}
		
		String docText = jcas.getDocumentText().replaceAll("\n\n", "\nEMPTYLINE\n");
		List<String> tokenized = ttt.tokenize(docText);
		
		int tokenOffset = 0;
		// loop through all the lines in the treetagger output
		for(String s : tokenized) {
			// charset missmatch fallback: signal (invalid) s
			if ((!(s.equals("EMPTYLINE"))) && (jcas.getDocumentText().indexOf(s, tokenOffset) < 0)) {
				Logger.printError(component, "Tokenization was interrupted because the token \"" + s 
						+ "\" could not be found in the original text. The reason for this might be "
						+ "that the encoding of the document is not UTF-8. This token was skipped and "
						+ "if it was part of a temporal expression, will not be extracted.");
				continue;
			}
			// create tokens and add them to the jcas's indexes.
			Token newToken = new Token(jcas);
			if (s.equals("EMPTYLINE")){
				newToken.setBegin(tokenOffset);
				newToken.setEnd(tokenOffset);
				newToken.setPos("EMPTYLINE");
				if (annotate_partofspeech){
					newToken.addToIndexes();
				}
			}
			else{
				newToken.setBegin(jcas.getDocumentText().indexOf(s, tokenOffset));
				newToken.setEnd(newToken.getBegin() + s.length());
				newToken.addToIndexes();
				tokenOffset = newToken.getEnd();
			}
		}
	}
275729
private void deEndify(List<CoreLabel> tokens) {
    if (flags.retainEntitySubclassification) {
      return;
    }
    tokens = new PaddedList<CoreLabel>(tokens, new CoreLabel());
    int k = tokens.size();
    String[] newAnswers = new String[k];
    for (int i = 0; i < k; i++) {
      CoreLabel c = tokens.get(i);
      CoreLabel p = tokens.get(i - 1);
      if (c.get(AnswerAnnotation.class).length() > 1 && c.get(AnswerAnnotation.class).charAt(1) == '-') {
        String base = c.get(AnswerAnnotation.class).substring(2);
        String pBase = (p.get(AnswerAnnotation.class).length() <= 2 ? p.get(AnswerAnnotation.class) : p.get(AnswerAnnotation.class).substring(2));
        boolean isSecond = (base.equals(pBase));
        boolean isStart = (c.get(AnswerAnnotation.class).charAt(0) == 'B' || c.get(AnswerAnnotation.class).charAt(0) == 'S');
        if (isSecond && isStart) {
          newAnswers[i] = intern("B-" + base);
        } else {
          newAnswers[i] = intern("I-" + base);
        }
      } else {
        newAnswers[i] = c.get(AnswerAnnotation.class);
      }
    }
    for (int i = 0; i < k; i++) {
      CoreLabel c = tokens.get(i);
      c.set(AnswerAnnotation.class, newAnswers[i]);
    }
  }
1025886
public static Interval<Integer> getChunkOffsetsUsingCharOffsets(List<? extends CoreMap> chunkList,
                                       int charStart, int charEnd)
  {
    int chunkStart = 0;
    int chunkEnd = chunkList.size();
    // Find first chunk with start > charStart
    for (int i = 0; i < chunkList.size(); i++) {
      int start = chunkList.get(i).get(CoreAnnotations.CharacterOffsetBeginAnnotation.class);
      if (start > charStart) {
        break;
      }
      chunkStart = i;
    }
    // Find first chunk with start >= charEnd
    for (int i = chunkStart; i < chunkList.size(); i++) {
      int start = chunkList.get(i).get(CoreAnnotations.CharacterOffsetBeginAnnotation.class);
      if (start >= charEnd) {
        chunkEnd = i;
        break;
      }
    }
    return Interval.toInterval(chunkStart, chunkEnd, Interval.INTERVAL_OPEN_END);
  }
87894
private ChunkType getChunkType(List<ChunkTaggedToken> tokens, int chunkStartPos) {
    boolean isPlural = false;
    for (int i = chunkStartPos; i < tokens.size(); i++) {
      ChunkTaggedToken token = tokens.get(i);
      if (!isBeginningOfNounPhrase(token) && !isContinuationOfNounPhrase(token)) {
        break;
      }
      if (false && "and".equals(token.getToken())) {   // e.g. "Tarzan and Jane" is a plural noun phrase
        // TODO: "Additionally, there are over 500 college and university chapter."
        isPlural = true;
      } else if (hasNounWithPluralReading(token)) {   // e.g. "ten books" is a plural noun phrase
        isPlural = true;
      }
    }
    return isPlural ? ChunkType.PLURAL : ChunkType.SINGULAR;
  }