Fix token offsets of sentences to match those in the document (assumes tokens are shared)
sentence token indices may not match document token list if certain html elements are ignored
@param docAnnotation
@return true if fix was okay, false otherwise
769615
public static boolean fixTokenOffsets(CoreMap docAnnotation)
  {
    List<CoreLabel> docTokens = docAnnotation.get(CoreAnnotations.TokensAnnotation.class);
    List<CoreMap> sentences = docAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
    int i = 0;
    CoreLabel curDocToken = docTokens.get(0);
    for (CoreMap sentence:sentences) {
      List<CoreLabel> sentTokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
      CoreLabel sentTokenFirst = sentTokens.get(0);
      while (curDocToken != sentTokenFirst) {
        i++;
        if (i >= docTokens.size()) { return false; }
        curDocToken = docTokens.get(i);
      }
      int sentTokenBegin = i;
      CoreLabel sentTokenLast = sentTokens.get(sentTokens.size()-1);
      while (curDocToken != sentTokenLast) {
        i++;
        if (i >= docTokens.size()) { return false; }
        curDocToken = docTokens.get(i);
      }
      int sentTokenEnd = i+1;
      sentence.set(CoreAnnotations.TokenBeginAnnotation.class, sentTokenBegin);
      sentence.set(CoreAnnotations.TokenEndAnnotation.class, sentTokenEnd);
    }
    return true;
  }
646184
public static boolean fixSalt(Document doc) {
	Element root = doc.getDocumentElement();
	String salt = root.getAttribute("salt");
	if (StringUtil.isEmpty(salt, true) || !Decision.isUUId(salt)) {
	    // create salt
	    root.setAttribute("salt", CreateUUID.invoke());
	    return true;
	}
	return false;
    }
178752
public static boolean isDocumentedAnnotation(AnnotationTypeDoc annotationDoc) {
        AnnotationDesc[] annotationDescList = annotationDoc.annotations();
        for (int i = 0; i < annotationDescList.length; i++) {
            if (annotationDescList[i].annotationType().qualifiedName().equals(
                   java.lang.annotation.Documented.class.getName())){
                return true;
            }
        }
        return false;
    }
151038
private boolean goToNextDoc() throws IOException {
    if (docId == NO_MORE_DOCS) {
      return true;
    } else {
      docId = spans1.spans.nextDoc();
      if (docId == NO_MORE_DOCS) {
        return true;
      } else {
        int spans2DocId = spans2.spans.docID();
        if (spans2DocId < docId) {
          spans2DocId = spans2.spans.advance(docId);
        }
        if (docId != spans2DocId) {
          return spans1.spans.nextStartPosition() != NO_MORE_POSITIONS;
        } else if (goToNextStartPosition()) {
          return true;
        } else {
          reset();
          return false;
        }
      }
    }
  }
439348
public boolean compare(List<CoreLabel> doc1, List<CoreLabel> doc2) {
		int i1 = 0, i2 = 0; //
		String last1 = "O", last2 = "O";

		boolean cm_set;
	
		for (int i = 0; i < doc1.size(); i++) {
			CoreLabel c1 = doc1.get(i);
			CoreLabel c2 = doc2.get(i);
			
			String a1 = c1.getString(AnswerAnnotation.class);
			String a2 = c2.getString(AnswerAnnotation.class);
			
			if (a1 == null || a2 == null) return false;
		}
		// TODO
			
		return true;
	}
665588
boolean matchProcessingInstruction(
            final boolean blockMatching,
            final int markupLevel, final int markupBlockIndex) {

        checkMarkupLevel(markupLevel);

        if (this.markupSelectorItem.anyLevel() || markupLevel == 0 || (this.prev != null && this.prev.matchedMarkupLevels[markupLevel - 1])) {
            // This text has not matched yet, but might match, so we should check

            this.matchesThisLevel = this.markupSelectorItem.matchesProcessingInstruction(markupBlockIndex, this.markupBlockMatchingCounter);

            if (matchesPreviousOrCurrentLevel(markupLevel)) {
                // This filter was already matched by a previous level (through an "open" event), so just delegate to next.

                if (this.next != null) {
                    return this.next.matchProcessingInstruction(blockMatching, markupLevel, markupBlockIndex);
                }
                return (blockMatching? true : this.matchesThisLevel);

            } else if (this.matchesThisLevel) {
                // This filter was not matched before. So the fact that it matches now means we need to consume it,
                // therefore not delegating.

                return (this.next == null);

            }

        } else if (matchesPreviousOrCurrentLevel(markupLevel)) {
            // This filter was already matched by a previous level (through an "open" event), so just delegate to next.

            if (this.next != null) {
                return this.next.matchProcessingInstruction(blockMatching, markupLevel, markupBlockIndex);
            }
            return blockMatching;

        }

        return false;

    }
392417
boolean matchXmlDeclaration(
            final boolean blockMatching,
            final int markupLevel, final int markupBlockIndex) {

        checkMarkupLevel(markupLevel);

        if (this.markupSelectorItem.anyLevel() || markupLevel == 0 || (this.prev != null && this.prev.matchedMarkupLevels[markupLevel - 1])) {
            // This text has not matched yet, but might match, so we should check

            this.matchesThisLevel = this.markupSelectorItem.matchesXmlDeclaration(markupBlockIndex, this.markupBlockMatchingCounter);

            if (matchesPreviousOrCurrentLevel(markupLevel)) {
                // This filter was already matched by a previous level (through an "open" event), so just delegate to next.

                if (this.next != null) {
                    return this.next.matchXmlDeclaration(blockMatching, markupLevel, markupBlockIndex);
                }
                return (blockMatching? true : this.matchesThisLevel);

            } else if (this.matchesThisLevel) {
                // This filter was not matched before. So the fact that it matches now means we need to consume it,
                // therefore not delegating.

                return (this.next == null);

            }

        } else if (matchesPreviousOrCurrentLevel(markupLevel)) {
            // This filter was already matched by a previous level (through an "open" event), so just delegate to next.

            if (this.next != null) {
                return this.next.matchXmlDeclaration(blockMatching, markupLevel, markupBlockIndex);
            }
            return blockMatching;

        }

        return false;

    }
742117
private JsDocToken parseModifiesTag(JsDocToken token) {
    if (token == JsDocToken.LEFT_CURLY) {
      Set<String> modifies = new HashSet<>();
      while (true) {
        if (match(JsDocToken.STRING)) {
          String name = stream.getString();
          if (!modifiesAnnotationKeywords.contains(name)
              && !jsdocBuilder.hasParameter(name)) {
            addParserWarning("msg.jsdoc.modifies.unknown", name);
          }

          modifies.add(stream.getString());
          token = next();
        } else {
          addParserWarning("msg.jsdoc.modifies");
          return token;
        }

        if (match(JsDocToken.PIPE)) {
          token = next();
        } else {
          break;
        }
      }

      if (!match(JsDocToken.RIGHT_CURLY)) {
        addParserWarning("msg.jsdoc.modifies");
      } else {
        token = next();
        if (!jsdocBuilder.recordModifies(modifies)) {
          addParserWarning("msg.jsdoc.modifies.duplicate");
        }
      }
    }
    return token;
  }
1039171
private static Fix replaceTargetAnnotation(
      Target annotation, AnnotationTree targetAnnotationTree) {
    Set<ElementType> types = EnumSet.copyOf(REQUIRED_ELEMENT_TYPES);
    types.addAll(Arrays.asList(annotation.value()));

    return replaceTargetAnnotation(targetAnnotationTree, types);
  }
816126
protected void removeAnnotation(String annotIdentifier) {
		if (m_annotations != null) {
			Iterator it = m_annotations.iterator();
			while (it.hasNext()) {
				Annotation ann = (Annotation) it.next();
				if (ann.getIdentifier().equals(annotIdentifier))
					it.remove();
			}
		}
	}
150646
boolean documentParam(String parameter, String description) {
    if (!lazyInitDocumentation()) {
      return true;
    }

    if (documentation.parameters == null) {
      documentation.parameters = new LinkedHashMap<>();
    }

    if (!documentation.parameters.containsKey(parameter)) {
      documentation.parameters.put(parameter, description);
      return true;
    } else {
      return false;
    }
  }
537741
public void process(JCas jcas) throws AnalysisEngineProcessException {
		Integer offset = 0; // a cursor of sorts to keep up with the position in the document text
		
		// grab the document text
		String docText = jcas.getDocumentText();
		// get [sentence-tokens[word-tokens]] from the MaxentTagger
		TokenizerFactory<Word> fac = PTBTokenizerFactory.newTokenizerFactory();  
		fac.setOptions("ptb3Escaping=false,untokenizable=noneKeep");
		List<List<HasWord>> tokenArray = MaxentTagger.tokenizeText(new StringReader(docText), fac);
		
		// iterate over sentences in this document
		for(List<HasWord> sentenceToken : tokenArray) {
			List<TaggedWord> taggedSentence = mt.tagSentence(sentenceToken);
			ListIterator<TaggedWord> twit = taggedSentence.listIterator();
			
			// create a sentence object. gets added to index or discarded depending on configuration
			Sentence sentence = new Sentence(jcas);
			sentence.setBegin(offset);
			
			Integer wordCount = 0;
			// iterate over words in this sentence
			for(HasWord wordToken : sentenceToken) {
				Token t = new Token(jcas);
				TaggedWord tw = twit.next();
				
				// if pos is supposed to be added, iterate through the tagged tokens and set pos
				if(annotate_partofspeech) {
					t.setPos(tw.tag());
				}
				
				String thisWord = wordToken.word();
				
				if(docText.indexOf(thisWord, offset) < 0) {
					Logger.printDetail(component, "A previously tagged token wasn't found in the document text: \"" + thisWord + "\". " +
							"This may be due to unpredictable punctuation tokenization; hence this token isn't tagged.");
					continue; // jump to next token: discards token
				} else {
					offset = docText.indexOf(thisWord, offset); // set cursor to the starting position of token in docText
					t.setBegin(offset);
					++wordCount;
				}
				
				offset += thisWord.length(); // move cursor behind the word
				t.setEnd(offset);
				
				// add tokens to indexes.
				if(annotate_tokens) {
					t.addToIndexes();
				}
			}
			
			// if flag is set, also tag sentences
			if(annotate_sentences) {
				if(wordCount == 0)
					sentence.setEnd(offset);
				else
					sentence.setEnd(offset-1);
				sentence.addToIndexes();
			}
		}
		
		// TODO: DEBUG
		FSIterator fsi = jcas.getAnnotationIndex(Sentence.type).iterator();
		while(fsi.hasNext()) {
			Sentence s = (Sentence) fsi.next();
			if(s.getBegin() < 0 || s.getEnd() < 0) {
				System.err.println("Sentence: " + s.getBegin() + ":" + s.getEnd() + " = " + s.getCoveredText());
				System.err.println("wrong index in text: " + jcas.getDocumentText());
				System.exit(-1);
			}
		}
		FSIterator fsi2 = jcas.getAnnotationIndex(Token.type).iterator();
		while(fsi2.hasNext()) {
			Token t = (Token) fsi2.next();
			if(t.getBegin() < 0 || t.getEnd() < 0) {
				System.err.println("In text: " + jcas.getDocumentText());
				System.err.println("Token: " + t.getBegin() + ":" + t.getEnd());
				System.exit(-1);
			}
		}
	}
740540
public static void check(BioCDocument document) {
    String text = BioCUtils.getText(document);
    // check annotation offset and text
    checkAnnotations(document.getAnnotations(), text, 0, BioCLog.log(document));
    // check relation
    for (BioCRelation relation : document.getRelations()) {
      for (BioCNode node : relation.getNodes()) {
        checkArgument(document.getAnnotation(node.getRefid()).isPresent(),
            "Cannot find node %s in relation %s\n" +
                "Location: %s", node, relation, BioCLog.log(document));
      }
    }

    // check passage
    for (BioCPassage passage : document.getPassages()) {
      text = BioCUtils.getText(passage);
      // check annotation offset and text
      checkAnnotations(passage.getAnnotations(), text, passage.getOffset(),
          BioCLog.log(document, passage));
      // check relation
      for (BioCRelation relation : passage.getRelations()) {
        for (BioCNode node : relation.getNodes()) {
          checkArgument(passage.getAnnotation(node.getRefid()).isPresent(),
              "Cannot find node %s in relation %s\n" +
                  "Location: %s", node, relation, BioCLog.log(document, passage));
        }
      }

      // check sentence
      for (BioCSentence sentence : passage.getSentences()) {
        // check annotation offset and text
        checkAnnotations(sentence.getAnnotations(), sentence.getText().get(),
            sentence.getOffset(), BioCUtils.getXPathString(document, passage, sentence));
        // check relation
        for (BioCRelation relation : sentence.getRelations()) {
          for (BioCNode node : relation.getNodes()) {
            checkArgument(sentence.getAnnotation(node.getRefid()).isPresent(),
                "Cannot find node %s in relation %s\n" +
                    "  Location: %s", node, relation,
                BioCUtils.getXPathString(document, passage, sentence));
          }
        }
      }
    }
  }
48584
public function doesDocBlockContainAnnotation(string $annotation): bool
    {
        $docComment = (new \ReflectionClass($this->fullClassName))->getDocComment();

        if (false === $docComment) {
            return false;
        }

        return false !== strpos($docComment, $annotation);
    }
626038
private boolean isMagicComment() {
        assert reader.ch == '@';
        int parens = 0;
        boolean stringLit = false;
        int lbp = reader.bp;
        char lch = reader.buf[++lbp];

        if (!Character.isJavaIdentifierStart(lch)) {
            // The first thing after the @ has to be the annotation identifier
            return false;
        }

        while (lbp < reader.buflen) {
            lch = reader.buf[++lbp];
            // We are outside any annotation values
            if (Character.isWhitespace(lch) &&
                    !spacesincomments &&
                    parens == 0) {
                return false;
            } else if (lch == '@' &&
                    parens == 0) {
                // At most one annotation per magic comment
                return false;
            } else if (lch == '(' && !stringLit) {
                ++parens;
            } else if (lch == ')' && !stringLit) {
                --parens;
            } else if (lch == '"') {
                // TODO: handle more complicated string literals,
                // char literals, escape sequences, unicode, etc.
                stringLit = !stringLit;
            } else if (lch == '*' &&
                    !stringLit &&
                    lbp + 1 < reader.buflen &&
                    reader.buf[lbp+1] == '/') {
                // We reached the end of the comment, make sure no
                // parens are open
                return parens == 0;
            } else if (!Character.isJavaIdentifierPart(lch) &&
                    !Character.isWhitespace(lch) &&
                    lch != '.' && // separator in fully-qualified annotation name
                    // TODO: this also allows /*@A...*/ which should not be recognized.
                    !spacesincomments &&
                    parens == 0 &&
                    !stringLit) {
                return false;
            }
            // Do we need anything else for annotation values, e.g. String literals?
        }
        // came to end of file before '*/'
        return false;
    }
1191018
protected void addAnnotation(EObject element, XAnnotation annotation,
			IModificationContext context) throws Exception {
		final ICompositeNode node = NodeModelUtils.findActualNodeFor(annotation);
		if (node != null) {
			final IXtextDocument document = context.getXtextDocument();
			final int startOffset = node.getOffset();
			final int parameterOffset = getTools().getOffsetForPattern(document, startOffset,
					"@\\s*" + toPattern(annotation.getAnnotationType().getQualifiedName()) //$NON-NLS-1$
					+ "\\s*\\(\\s*"); //$NON-NLS-1$
			final int insertOffset;
			if (document.getChar(parameterOffset) == '{') {
				insertOffset = parameterOffset + getTools().getSpaceSize(document, parameterOffset + 1) + 1;
			} else {
				insertOffset = parameterOffset;
			}
			final ReplacingAppendable appendable = getTools().getAppendableFactory().create(document,
					(XtextResource) element.eResource(), insertOffset, 0);
			appendable.append("\""); //$NON-NLS-1$
			appendable.append(extractId(this.code));
			appendable.append("\", "); //$NON-NLS-1$
			appendable.commitChanges();
		}
	}
45230
def _prep_tags(t, annotations):
    """"""
    tags = [['O' for _ in sent.tokens] for sent in t.sentences]
    for start, end, text in annotations:
        done_first = False
        for i, sent in enumerate(t.sentences):
            for j, token in enumerate(sent.tokens):
                if start <= token.start < end or start < token.end <= end:
                    # Token start or end occurs within the annotation
                    tags[i][j] = 'I-CM' if done_first else 'B-CM'
                    done_first = True
    tagged = [[(token[0], token[1], tags[i][j]) for j, token in enumerate(sentence.pos_tagged_tokens)] for i, sentence in enumerate(t.sentences)]
    return tagged