Annotates a CoreMap representing a chunk with text information
TextAnnotation - String representing tokens in this chunks (token text separated by space)
@param chunk - CoreMap to be annotated
@param tokenTextKey - Key to use to find the token text
342078
public static void annotateChunkText(CoreMap chunk, Class tokenTextKey)
  {
    List<CoreLabel> chunkTokens = chunk.get(CoreAnnotations.TokensAnnotation.class);
    String text = getTokenText(chunkTokens, tokenTextKey);
    chunk.set(CoreAnnotations.TextAnnotation.class, text);
  }
404697
public static function phpTextChunk($key,$text) {
        $chunktype = "tEXt";
        $chunkdata = $key . "\0" . $text;
        $crc = pack("N", crc32($chunktype . $chunkdata));
        $len = pack("N",strlen($chunkdata));
        return $len .  $chunktype  . $chunkdata . $crc;
    }
884241
private SimpleOrderedMap<Object> createToken(ComponentToken token,
      Boolean encode) throws IOException {
    // System.out.println("Create stats position " + position.dataType + " "
    // + position.statsType + " " + position.statsItems + " --- " + encode);
    SimpleOrderedMap<Object> mtasTokenResponse = new SimpleOrderedMap<>();
    mtasTokenResponse.add("key", token.key);
    MtasSolrMtasResult data = new MtasSolrMtasResult(token.dataCollector,
        token.dataType, token.statsType, token.statsItems, null, null);
    if (encode) {
      mtasTokenResponse.add("_encoded_data", MtasSolrResultUtil.encode(data));
    } else {
      mtasTokenResponse.add(token.dataCollector.getCollectorType(), data);
      MtasSolrResultUtil.rewrite(mtasTokenResponse, searchComponent);
    }
    return mtasTokenResponse;
  }
1199757
function tokenBase(stream, state) {
    var ch = stream.next(), 
        mightBeFunction = false,
        isEQName = isEQNameAhead(stream);
    
    // an XML tag (if not in some sub, chained tokenizer)
    if (ch == "<") {
      if(stream.match("!--", true))
        return chain(stream, state, tokenXMLComment);
        
      if(stream.match("![CDATA", false)) {
        state.tokenize = tokenCDATA;
        return ret("tag", "tag");
      }
      
      if(stream.match("?", false)) {
        return chain(stream, state, tokenPreProcessing);
      }
      
      var isclose = stream.eat("/");
      stream.eatSpace();
      var tagName = "", c;
      while ((c = stream.eat(/[^\s\u00a0=<>\"\'\/?]/))) tagName += c;
      
      return chain(stream, state, tokenTag(tagName, isclose));
    }
    // start code block
    else if(ch == "{") {
      pushStateStack(state,{ type: "codeblock"});
      return ret("", "");
    }
    // end code block
    else if(ch == "}") {
      popStateStack(state);
      return ret("", "");
    }
    // if we're in an XML block
    else if(isInXmlBlock(state)) {
      if(ch == ">")
        return ret("tag", "tag");
      else if(ch == "/" && stream.eat(">")) {
        popStateStack(state);
        return ret("tag", "tag");
      }
      else  
        return ret("word", "variable");
    }
    // if a number
    else if (/\d/.test(ch)) {
      stream.match(/^\d*(?:\.\d*)?(?:E[+\-]?\d+)?/);
      return ret("number", "atom");
    }
    // comment start
    else if (ch === "(" && stream.eat(":")) {
      pushStateStack(state, { type: "comment"});
      return chain(stream, state, tokenComment);
    }
    // quoted string
    else if (  !isEQName && (ch === '"' || ch === "'"))
      return chain(stream, state, tokenString(ch));
    // variable
    else if(ch === "$") {
      return chain(stream, state, tokenVariable);
    }
    // assignment
    else if(ch ===":" && stream.eat("=")) {
      return ret("operator", "keyword");
    }
    // open paren
    else if(ch === "(") {
      pushStateStack(state, { type: "paren"});
      return ret("", "");
    }
    // close paren
    else if(ch === ")") {
      popStateStack(state);
      return ret("", "");
    }
    // open paren
    else if(ch === "[") {
      pushStateStack(state, { type: "bracket"});
      return ret("", "");
    }
    // close paren
    else if(ch === "]") {
      popStateStack(state);
      return ret("", "");
    }
    else {
      var known = keywords.propertyIsEnumerable(ch) && keywords[ch];

      // if there's a EQName ahead, consume the rest of the string portion, it's likely a function
      if(isEQName && ch === '\"') while(stream.next() !== '"'){}
      if(isEQName && ch === '\'') while(stream.next() !== '\''){}
      
      // gobble up a word if the character is not known
      if(!known) stream.eatWhile(/[\w\$_-]/);
      
      // gobble a colon in the case that is a lib func type call fn:doc
      var foundColon = stream.eat(":");
      
      // if there's not a second colon, gobble another word. Otherwise, it's probably an axis specifier
      // which should get matched as a keyword
      if(!stream.eat(":") && foundColon) {
        stream.eatWhile(/[\w\$_-]/);
      }
      // if the next non whitespace character is an open paren, this is probably a function (if not a keyword of other sort)
      if(stream.match(/^[ \t]*\(/, false)) {
        mightBeFunction = true;
      }
      // is the word a keyword?
      var word = stream.current();
      known = keywords.propertyIsEnumerable(word) && keywords[word];
      
      // if we think it's a function call but not yet known, 
      // set style to variable for now for lack of something better
      if(mightBeFunction && !known) known = {type: "function_call", style: "variable def"};
      
      // if the previous word was element, attribute, axis specifier, this word should be the name of that
      if(isInXmlConstructor(state)) {
        popStateStack(state);
        return ret("word", "variable", word);
      }
      // as previously checked, if the word is element,attribute, axis specifier, call it an "xmlconstructor" and 
      // push the stack so we know to look for it on the next word
      if(word == "element" || word == "attribute" || known.type == "axis_specifier") pushStateStack(state, {type: "xmlconstructor"});
      
      // if the word is known, return the details of that else just call this a generic 'word'
      return known ? ret(known.type, known.style, word) :
                     ret("word", "variable", word);
    }
  }
841716
private static function readToken(ServerRequestInterface $request, string $tokenKey): ?string
    {
        $token = is_array($form = $request->getParsedBody()) === true ? ($form[$tokenKey] ?? null) : null;

        return $token;
    }
86588
private boolean tokenTextsDoNotMatch(Token token1, Token token2) {
        return !token1.getText().equals(token2.getText());
    }
886761
function(context: ParserContext): ?Token {
    const delimiter = context.reader.peek();
    if (delimiter !== '"' && delimiter !== "'") return null;
    if (!isInsideOpenBracket(context)) return null;

    // read until the delimiter
    let stringValue = context.reader.read();
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      stringValue += next;

      if (next === delimiter) break;
    }

    return context.createToken("string", stringValue);
  }
487759
public static boolean isFirstCoveredToken(final Token token, final Annotation annotation)
    {
        final JCas jCas = getJCas(annotation);
        final List<Token> coveredTokens = JCasUtil.selectCovered(jCas, Token.class, annotation);

        if (coveredTokens.isEmpty()) {
            return false;
        }
        else {
            final Token firstCoveredToken = coveredTokens.get(0);
            return haveSameSpan(token, firstCoveredToken);
        }
    }
362719
public void addText(final String _text)
    {
        this.tokens.add(new Token(ValueList.TokenType.TEXT, _text));
    }
895742
function TokenStreamBase(input, tokenData){

    /**
     * The string reader for easy access to the text.
     * @type StringReader
     * @property _reader
     * @private
     */
    this._reader = input ? new StringReader(input.toString()) : null;

    /**
     * Token object for the last consumed token.
     * @type Token
     * @property _token
     * @private
     */
    this._token = null;

    /**
     * The array of token information.
     * @type Array
     * @property _tokenData
     * @private
     */
    this._tokenData = tokenData;

    /**
     * Lookahead token buffer.
     * @type Array
     * @property _lt
     * @private
     */
    this._lt = [];

    /**
     * Lookahead token buffer index.
     * @type int
     * @property _ltIndex
     * @private
     */
    this._ltIndex = 0;

    this._ltIndexCache = [];
}
892684
@Override
	public void parse(final byte[] pData, final Collection<TagAndLength> pTags) {
		Collection<AnnotationData> set = getAnnotationSet(pTags);
		BitUtils bit = new BitUtils(pData);
		Iterator<AnnotationData> it = set.iterator();
		while (it.hasNext()) {
			AnnotationData data = it.next();
			if (data.isSkip()) {
				bit.addCurrentBitIndex(data.getSize());
			} else {
				Object obj = DataFactory.getObject(data, bit);
				setField(data.getField(), this, obj);
			}
		}
	}
530390
protected void parse( String content ) {
        Tokenizer tokenizer = new CndTokenizer(false, true);
        TokenStream tokens = new TokenStream(content, tokenizer, false);
        tokens.start();
        while (tokens.hasNext()) {
            // Keep reading while we can recognize one of the two types of statements ...
            if (tokens.matches("<", ANY_VALUE, "=", ANY_VALUE, ">")) {
                parseNamespaceMapping(tokens);
            } else if (tokens.matches("[", ANY_VALUE, "]")) {
                parseNodeTypeDefinition(tokens);
            } else {
                Position position = tokens.previousPosition();
                throw new ParsingException(position, CndI18n.expectedNamespaceOrNodeDefinition.text(tokens.consume(),
                                                                                                    position.getLine(),
                                                                                                    position.getColumn()));
            }
        }
    }
1179082
function tokenAttribute(stream, state) {
    var ch = stream.next();
    
    if(ch == "/" && stream.eat(">")) {
      if(isInXmlAttributeBlock(state)) popStateStack(state);
      if(isInXmlBlock(state)) popStateStack(state);
      return ret("tag", "tag");
    }
    if(ch == ">") {
      if(isInXmlAttributeBlock(state)) popStateStack(state);
      return ret("tag", "tag");
    }
    if(ch == "=")
      return ret("", "");
    // quoted string
    if (ch == '"' || ch == "'")
      return chain(stream, state, tokenString(ch, tokenAttribute));

    if(!isInXmlAttributeBlock(state)) 
      pushStateStack(state, { type: "attribute", name: name, tokenize: tokenAttribute});

    stream.eat(/[a-zA-Z_:]/);
    stream.eatWhile(/[-a-zA-Z0-9_:.]/);
    stream.eatSpace();

    // the case where the attribute has not value and the tag was closed
    if(stream.match(">", false) || stream.match("/", false)) {
      popStateStack(state);
      state.tokenize = tokenBase;      
    }

    return ret("attribute", "attribute");
  }
991238
@Override
    public Object addTokenToDistributedMap(Object key, Object value) {
        String keyStr = (String) key;
        TokenManager tm = LoggedOutTokenCacheImpl.tokenManager.getService();
        int timeOut = -1;
        try {
            byte[] tokenBytes = AuthenticationHelper.copyCredToken(Base64Coder.base64DecodeString(keyStr));
            Token token = tm.recreateTokenFromBytes(tokenBytes);
            if (token != null)
            {
                long tokenExp = token.getExpiration();
                long calcTimeOut = tokenExp - System.currentTimeMillis();
                timeOut = (int) calcTimeOut / 1000;
                String userName = token.getAttributes("u")[0];
                if (userName != null)
                    value = userName;
            }

        } catch (InvalidTokenException e) {
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                Tr.debug(tc, "Token is not valid so do not cache it " + e.getMessage());
            }
            return null;
        } catch (TokenExpiredException e) {
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                Tr.debug(tc, "Token is exipired so do not cache it " + e.getMessage());
            }
            return null;
        }

        return putDistributedObjectLoggedOutToken(key, value, timeOut);
    }
579234
Token match( TokenStream input,
                 int ttype,
                 String text,
                 int[] follow,
                 DroolsEditorType etype ) throws RecognitionException {
        Token matchedSymbol = null;
        matchedSymbol = input.LT(1);
        if (input.LA(1) == ttype && (text == null || text.equals(matchedSymbol.getText()))) {
            input.consume();
            state.errorRecovery = false;
            state.failed = false;
            helper.emit(matchedSymbol,
                    etype);
            return matchedSymbol;
        }
        if (state.backtracking > 0) {
            state.failed = true;
            return matchedSymbol;
        }
        matchedSymbol = recoverFromMismatchedToken(input,
                ttype,
                text,
                follow);
        helper.emit(matchedSymbol,
                etype);
        return matchedSymbol;
    }
826244
function onStdin( chunk ) {
	var i , j , buffer , startBuffer , char , codepoint ,
		keymapCode , keymapStartCode , keymap , keymapList ,
		regexp , matches , bytes , found , handlerResult ,
		index = 0 , length = chunk.length ;

	if ( this.shutdown ) { return ; }

	while ( index < length ) {
		found = false ;
		bytes = 1 ;

		if ( chunk[ index ] <= 0x1f || chunk[ index ] === 0x7f ) {
			// Those are ASCII control character and DEL key

			for ( i = Math.min( length , Math.max( this.rKeymapMaxSize , this.rKeymapStarterMaxSize ) ) ; i > 0 ; i -- ) {
				buffer = chunk.slice( index ) ;
				keymapCode = buffer.toString() ;
				startBuffer = chunk.slice( index , index + i ) ;
				keymapStartCode = startBuffer.toString() ;


				if ( this.rKeymap[ i ] && this.rKeymap[ i ][ keymapStartCode ] ) {
					// First test fixed sequences

					keymap = this.rKeymap[ i ][ keymapStartCode ] ;
					found = true ;

					if ( keymap.handler ) {
						handlerResult = keymap.handler.call( this , keymap.name , chunk.slice( index + i ) ) ;
						bytes = i + handlerResult.eaten ;

						if ( ! handlerResult.disable ) {
							this.emit( keymap.event , handlerResult.name , handlerResult.data ) ;
						}
					}
					else if ( keymap.event ) {
						bytes = i ;
						this.emit( keymap.event , keymap.name , keymap.data , { code: startBuffer } ) ;
					}
					else {
						bytes = i ;
						this.emit( 'key' , keymap.name , keymap.matches , { isCharacter: false , code: startBuffer } ) ;
					}

					break ;
				}
				else if ( this.rKeymapStarter[ i ] && this.rKeymapStarter[ i ][ keymapStartCode ] ) {
					// Then test pattern sequences

					keymapList = this.rKeymapStarter[ i ][ keymapStartCode ] ;

					//console.log( 'for i:' , keymapList ) ;

					for ( j = 0 ; j < keymapList.length ; j ++ ) {
						keymap = keymapList[ j ] ;

						regexp = '^' +
							string.escape.regExp( keymap.starter ) +
							'([ -~]*)' +	// [ -~] match only all ASCII non-control character
							string.escape.regExp( keymap.ender ) ;

						matches = keymapCode.match( new RegExp( regexp ) , 'g' ) ;

						//console.log( 'for j:' , keymap , regexp , matches ) ;

						if ( matches ) {
							found = true ;

							handlerResult = keymap.handler.call( this , keymap.name , matches[ 1 ] ) ;
							bytes = matches[ 0 ].length ;
							this.emit( keymap.event , handlerResult.name , handlerResult.data ) ;

							break ;
						}
					}

					if ( found ) { break ; }
				}
			}

			// Nothing was found, so to not emit trash, we just abort the current buffer processing
			if ( ! found ) { this.emit( 'unknown' , chunk ) ; return ; }
		}
		else if ( chunk[ index ] >= 0x80 ) {
			// Unicode bytes per char guessing
			if ( chunk[ index ] < 0xc0 ) { continue ; }	// We are in a middle of an unicode multibyte sequence... Something fails somewhere, we will just continue for now...
			else if ( chunk[ index ] < 0xe0 ) { bytes = 2 ; }
			else if ( chunk[ index ] < 0xf0 ) { bytes = 3 ; }
			else if ( chunk[ index ] < 0xf8 ) { bytes = 4 ; }
			else if ( chunk[ index ] < 0xfc ) { bytes = 5 ; }
			else { bytes = 6 ; }

			buffer = chunk.slice( index , index + bytes ) ;
			char = buffer.toString( 'utf8' ) ;

			//if ( bytes > 2 ) { codepoint = punycode.ucs2.decode( char )[ 0 ] ; }
			if ( bytes > 2 ) { codepoint = string.unicode.firstCodePoint( char ) ; }
			else { codepoint = char.charCodeAt( 0 ) ; }

			this.emit( 'key' , char , [ char ] , { isCharacter: true , codepoint: codepoint , code: buffer } ) ;
		}
		else {
			// Standard ASCII
			char = String.fromCharCode( chunk[ index ] ) ;
			this.emit( 'key' , char , [ char ] , { isCharacter: true , codepoint: chunk[ index ] , code: chunk[ index ] } ) ;
		}

		index += bytes ;
	}
}
1134029
private void chunkMap(final Element root) {
        // create the reference to the new file on root element.
        String newFilename = replaceExtension(new File(currentFile).getName(), FILE_EXTENSION_DITA);
        URI newFile = currentFile.resolve(newFilename);
        if (new File(newFile).exists()) {
            final URI oldFile = newFile;
            newFilename = chunkFilenameGenerator.generateFilename(CHUNK_PREFIX, FILE_EXTENSION_DITA);
            newFile = currentFile.resolve(newFilename);
            // Mark up the possible name changing, in case that references might be updated.
            conflictTable.put(newFile, oldFile.normalize());
        }
        changeTable.put(newFile, newFile);

        // change the class attribute to "topicref"
        final String origCls = root.getAttribute(ATTRIBUTE_NAME_CLASS);
        root.setAttribute(ATTRIBUTE_NAME_CLASS, origCls + MAP_TOPICREF.matcher);
        root.setAttribute(ATTRIBUTE_NAME_HREF, toURI(newFilename).toString());

        createTopicStump(newFile);

        // process chunk
        processTopicref(root);

        // restore original root element
        if (origCls != null) {
            root.setAttribute(ATTRIBUTE_NAME_CLASS, origCls);
        }
        root.removeAttribute(ATTRIBUTE_NAME_HREF);
    }