Helper function for token handling.

@param tokenizer Tokenizer
@throws IOException
10600
private void nextToken(StreamTokenizer tokenizer) throws IOException {
    tokenizer.nextToken();
    if((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
      tokenizer.ttype = StreamTokenizer.TT_WORD;
    }
    else if((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
      tokenizer.ttype = '?';
    }
    if(LOG.isDebugging()) {
      if(tokenizer.ttype == StreamTokenizer.TT_NUMBER) {
        LOG.debug("token: " + tokenizer.nval);
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_WORD) {
        LOG.debug("token: " + tokenizer.sval);
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_EOF) {
        LOG.debug("token: EOF");
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_EOL) {
        LOG.debug("token: EOL");
      }
      else {
        LOG.debug("token type: " + tokenizer.ttype);
      }
    }
  }
157860
private Attribute parseAttribute(StreamTokenizer tokenizer) throws IOException, ParseException {
        Attribute attribute = null;

        // Get attribute name.
        getNextToken(tokenizer);
        String attributeName = tokenizer.sval;
        getNextToken(tokenizer);

        // Check if attribute is nominal.
        if (tokenizer.ttype == StreamTokenizer.TT_WORD) {

            // Attribute is real, integer, or string.
            if (tokenizer.sval.equalsIgnoreCase(ARFF_ATTRIBUTE_REAL) ||
                    tokenizer.sval.equalsIgnoreCase(ARFF_ATTRIBUTE_INTEGER) ||
                    tokenizer.sval.equalsIgnoreCase(ARFF_ATTRIBUTE_NUMERIC)) {
                attribute = new NumericAttribute(attributeName);
                readTillEOL(tokenizer);

            } else if (tokenizer.sval.equalsIgnoreCase(ARFF_ATTRIBUTE_STRING)) {
                attribute = new StringAttribute(attributeName);
                readTillEOL(tokenizer);

            } else if (tokenizer.sval.equalsIgnoreCase(ARFF_ATTRIBUTE_DATE)) {
                String format = null;
                if (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
                    if ((tokenizer.ttype != StreamTokenizer.TT_WORD) && (tokenizer.ttype != '\'') && (tokenizer.ttype != '\"')) {
                        throw new ParseException("not a valid date format", tokenizer.lineno());
                    }
                    format = tokenizer.sval;
                    readTillEOL(tokenizer);
                } else {
                    tokenizer.pushBack();
                }
                attribute = new DateAttribute(attributeName, null, format);
                readTillEOL(tokenizer);

            } else if (tokenizer.sval.equalsIgnoreCase(ARFF_ATTRIBUTE_RELATIONAL)) {
                readTillEOL(tokenizer);

            } else if (tokenizer.sval.equalsIgnoreCase(ARFF_END_SUBRELATION)) {
                getNextToken(tokenizer);

            } else {
                throw new ParseException("Invalid attribute type or invalid enumeration", tokenizer.lineno());
            }
        } else {

            // Attribute is nominal.
            List<String> attributeValues = new ArrayList<>();
            tokenizer.pushBack();

            // Get values for nominal attribute.
            if (tokenizer.nextToken() != '{') {
                throw new ParseException("{ expected at beginning of enumeration", tokenizer.lineno());
            }
            while (tokenizer.nextToken() != '}') {
                if (tokenizer.ttype == StreamTokenizer.TT_EOL) {
                    throw new ParseException("} expected at end of enumeration", tokenizer.lineno());
                } else {
                    attributeValues.add(tokenizer.sval.trim());
                }
            }

            String[] values = new String[attributeValues.size()];
            for (int i = 0; i < values.length; i++) {
                values[i] = attributeValues.get(i);
            }
            attribute = new NominalAttribute(attributeName, values);
        }

        getLastToken(tokenizer, false);
        getFirstToken(tokenizer);
        if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
            throw new ParseException(PREMATURE_END_OF_FILE, tokenizer.lineno());
        }

        return attribute;
    }
391318
public boolean nextValidToken() throws IOException {
        while (tokeniser.nextToken()) {
            if (tokeniser.getTokenType() == PRTokeniser.TK_COMMENT)
                continue;
            return true;
        }
        return false;
    }
1290863
protected void doSplitTokenImpl(Token token, ITokenAcceptor result) {
		String text = token.getText();
		int indentation = computeIndentation(text);
		if (indentation == -1 || indentation == currentIndentation) {
			// no change of indentation level detected simply process the token
			result.accept(token);
		} else if (indentation > currentIndentation) {
			// indentation level increased
			splitIntoBeginToken(token, indentation, result);
		} else if (indentation < currentIndentation) {
			// indentation level decreased
			int charCount = computeIndentationRelevantCharCount(text);
			if (charCount > 0) {
				// emit whitespace including newline
				splitWithText(token, text.substring(0, charCount), result);	
			}
			// emit end tokens at the beginning of the line
			decreaseIndentation(indentation, result);
			if (charCount != text.length()) {
				handleRemainingText(token, text.substring(charCount), indentation, result);
			}
		} else {
			throw new IllegalStateException(String.valueOf(indentation));
		}
	}
937905
static Parser<Token[]> tokens(final Parser<? extends Collection<Token>> parser) {
    return parser.map(list -> list.toArray(new Token[list.size()]));
  }
693040
function Tokenizer(dic) {
    this.token_info_dictionary = dic.token_info_dictionary;
    this.unknown_dictionary = dic.unknown_dictionary;
    this.viterbi_builder = new ViterbiBuilder(dic);
    this.viterbi_searcher = new ViterbiSearcher(dic.connection_costs);
    this.formatter = new IpadicFormatter();  // TODO Other dictionaries
}
238946
private static boolean expect(StringTokenizer tokenizer, String token) {
        if (!tokenizer.hasMoreTokens()) {
            return false;
        }
        String s = tokenizer.nextToken();
        if (DEBUG) {
            System.out.println("token=" + s);
        }
        return s.equals(token);
    }
263476
@VisibleForTesting
  Collection<Token<? extends TokenIdentifier>> readDelegationTokens(Path tokenFilePath) throws IOException {
    LOGGER.info("Reading updated token from token file: " + tokenFilePath);
    return YarnHelixUtils.readTokensFromFile(tokenFilePath, this.fs.getConf());
  }
1256050
@Override
  Tokenizer.Token visit(Tokenizer.Token value) throws IOException {
    // count the opCodes
    if(value.getEditsElement() == EditsElement.OPCODE) {
      if(value instanceof Tokenizer.ByteToken) {
        incrementOpCodeCount(((Tokenizer.ByteToken)value).value);
      } else {
        throw new IOException("Token for EditsElement.OPCODE should be " +
          "of type Tokenizer.ByteToken, not " + value.getClass());
      }
    }
    return value;
  }
693788
void addTokensFromFile(final UserGroupInformation ugi) throws IOException {
    LOG.log(Level.FINE, "Reading security tokens from file: {0}", this.securityTokensFile);

    try (final FileInputStream stream = new FileInputStream(securityTokensFile)) {
      final BinaryDecoder decoder = decoderFactory.binaryDecoder(stream, null);

      while (!decoder.isEnd()) {
        final SecurityToken token = tokenDatumReader.read(null, decoder);

        final Token<TokenIdentifier> yarnToken = new Token<>(
            token.getKey().array(),
            token.getPassword().array(),
            new Text(token.getKind().toString()),
            new Text(token.getService().toString()));

        LOG.log(Level.FINE, "addToken for {0}", yarnToken.getKind());

        ugi.addToken(yarnToken);
      }
    }
  }
344274
public void setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
        this.tokenizerFactory = tokenizerFactory;

        if (sentenceIter != null) {
            SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIter)
                            .tokenizerFactory(this.tokenizerFactory).build();
            this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
        }
    }
831022
public void tokenise() throws IOException {
		int nextChar = 0;
//		char[] nextChar = new char[1]; // input variable
//		nextChar[0]=0;	// set to 0
		this.setTokeniserState(TOKENISER_NORMAL);	// set initial tokeniser state
		
		
//		while(this.pbReader.read(nextChar) != -1) {
		while((nextChar = this.pbReader.read()) != -1) {
			this.byteCount++;
			
	        if (this.getTokeniserState() == TOKENISER_BINARY)                      // if we're parsing binary data, handle it directly
	        {
	            if (parseChar(nextChar) != errOK)
	                return; 
	        }  else {
//				switch(nextChar[0]) {
				switch(nextChar) {
					case '{':	// scope delimiter - Open
						this.handleOpenGroup();
						break;
					case '}':  // scope delimiter - Close
						this.handleCloseGroup();
						break;
					case 0x0a:	// noise character
					case 0x0d:	// noise character
//						if(this.isImport()) {
//							this.rtfDoc.add(new RtfDirectContent(new String(nextChar)));
//						}
						break;
					case '\\':	// Control word start delimiter
							if(parseCtrlWord(pbReader) != errOK) {
							// TODO: Indicate some type of error
							return;
						}
						break;
					default:
						if(groupLevel == 0) { // BOMs
							break;
						}
						if(this.getTokeniserState() == TOKENISER_HEX) {
							StringBuffer hexChars = new StringBuffer();
							hexChars.append(nextChar);
//							if(pbReader.read(nextChar) == -1) {
							if((nextChar = pbReader.read()) == -1) {
								return;
							}
							this.byteCount++;
							hexChars.append(nextChar);
	                    	try {
//								nextChar[0]=(char)Integer.parseInt(hexChars.toString(), 16);
								nextChar=Integer.parseInt(hexChars.toString(), 16);
							} catch (NumberFormatException e) {
								return;
							}
		                    this.setTokeniserState(TOKENISER_NORMAL);
						}
						if (parseChar(nextChar) != errOK) {
							// some error occurred. we should send a real error
							return; 							
						}
						break;
				}	// switch(nextChar[0])
			}	// end if (this.getTokeniserState() == TOKENISER_BINARY)
	        
//	        if(groupLevel < 1 && this.isImportFragment()) return; //return errOK;
//	        if(groupLevel < 0 && this.isImportFull()) return; //return errStackUnderflow;
//	        if(groupLevel < 0 && this.isConvert()) return; //return errStackUnderflow;
	        
		}// end while(reader.read(nextChar) != -1)
		RtfDestination dest = this.getCurrentDestination();
		if(dest != null) {
			dest.closeDestination();
		}
	}
439226
@Override
  public boolean tokenizeLine(String line, BufferedReader reader, Record record) throws IOException {
    char separator = separatorChar;
    int len = line.length();
    int start = 0; 
    int j = 0;
    for (int i = 0; i < len; i++) {
      if (line.charAt(i) == separator) {
        put(line, start, i, j, record);
        start = i+1;
        j++;
      }
    }
    put(line, start, len, j, record);
    return true;
  }
654181
private void tokenize(JCas jcas) {
		// read tokenized text to add tokens to the jcas
		Logger.printDetail(component, "TreeTagger (tokenization) with: " + ttprops.abbFileName);
		
		EnumSet<Flag> flags = Flag.getSet(ttprops.languageSwitch);
		TreeTaggerTokenizer ttt; ttprops.abbFileName = "english-abbreviations";
		if(ttprops.abbFileName != null) {
			ttt = new TreeTaggerTokenizer(ttprops.rootPath + ttprops.fileSeparator + "lib" + ttprops.fileSeparator + ttprops.abbFileName, flags);
		} else {
			ttt = new TreeTaggerTokenizer(null, flags);
		}
		
		String docText = jcas.getDocumentText().replaceAll("\n\n", "\nEMPTYLINE\n");
		List<String> tokenized = ttt.tokenize(docText);
		
		int tokenOffset = 0;
		// loop through all the lines in the treetagger output
		for(String s : tokenized) {
			// charset missmatch fallback: signal (invalid) s
			if ((!(s.equals("EMPTYLINE"))) && (jcas.getDocumentText().indexOf(s, tokenOffset) < 0)) {
				Logger.printError(component, "Tokenization was interrupted because the token \"" + s 
						+ "\" could not be found in the original text. The reason for this might be "
						+ "that the encoding of the document is not UTF-8. This token was skipped and "
						+ "if it was part of a temporal expression, will not be extracted.");
				continue;
			}
			// create tokens and add them to the jcas's indexes.
			Token newToken = new Token(jcas);
			if (s.equals("EMPTYLINE")){
				newToken.setBegin(tokenOffset);
				newToken.setEnd(tokenOffset);
				newToken.setPos("EMPTYLINE");
				if (annotate_partofspeech){
					newToken.addToIndexes();
				}
			}
			else{
				newToken.setBegin(jcas.getDocumentText().indexOf(s, tokenOffset));
				newToken.setEnd(newToken.getBegin() + s.length());
				newToken.addToIndexes();
				tokenOffset = newToken.getEnd();
			}
		}
	}
763002
private void readSparseInstance(StreamTokenizer tokenizer, AttributeDataset data, Attribute[] attributes) throws IOException, ParseException {
        double[] x = responseIndex >= 0 ? new double[attributes.length - 1] : new double[attributes.length];
        double y = Double.NaN;
        int index = -1;
        
        // Get values for all attributes.
        do {
            getNextToken(tokenizer);
            
            // end of instance
            if (tokenizer.ttype == '}') {
                break;
            }
            
            String s = tokenizer.sval.trim();
            if (index < 0) {
                index = Integer.parseInt(s);
                if (index < 0 || index >= attributes.length) {
                    throw new ParseException("Invalid attribute index: " + index, tokenizer.lineno());
                }
                
            } else {
                
                String val = s;
                if (index != responseIndex) {
                    if (val.equals("?")) {
                        x[index] = Double.NaN;
                    } else {
                        x[index] = attributes[index].valueOf(val);
                    }
                } else {
                    if (val.equals("?")) {
                        y = Double.NaN;
                    } else {
                        y = attributes[index].valueOf(val);
                    }
                }
                
                index = -1;
            }
            
        } while (tokenizer.ttype == StreamTokenizer.TT_WORD);
        
        if (Double.isNaN(y)) data.add(x); else data.add(x, y);
    }
217762
public void write(@NonNull Normalizer normalizer, @NonNull OutputStream stream) throws IOException {
        NormalizerSerializerStrategy strategy = getStrategy(normalizer);

        writeHeader(stream, Header.fromStrategy(strategy));
        //noinspection unchecked
        strategy.write(normalizer, stream);
    }
605746
public Token<DelegationTokenIdentifier> createDelegationToken(String renewer) throws IOException {
    Text renewerText = new Text();
    if (renewer != null) {
      renewerText.set(renewer);
    }

    DelegationTokenIdentifier tokenIdentifier =
        requireNonNull(createTokenIdentifier(renewerText), "Token identifier");

    Token<DelegationTokenIdentifier> token = new Token<>(tokenIdentifier, secretManager);
    token.setKind(getKind());
    token.setService(service);
    logger.atFine().log("Created token %s with token identifier %s", token, tokenIdentifier);
    return token;
  }