Advances the specified number of tokens in the stream and places them in
the buffer.

@param tokens the number of tokens to advance

@return {@true} if the stream contained at least that many tokens, {@code
false} if the stream contained fewer
767200
private boolean advance(int tokens) {
	while (buffer.size() < tokens && tokenizer.hasNext())
	    buffer.add(tokenizer.next());
	return buffer.size() >= tokens;
    }
29517
public static synchronized void updateTokens(Collection<Token> tokens)
    {
        assert !tokens.isEmpty() : "removeEndpoint should be used instead";
        String req = "INSERT INTO system.%s (key, tokens) VALUES ('%s', ?)";
        executeInternal(String.format(req, LOCAL_CF, LOCAL_KEY), tokensAsSet(tokens));
        forceBlockingFlush(LOCAL_CF);
    }
102881
public Span[] seqToSpans(final String[] tokens) {
    final Span[] origSpans = this.sequenceLabeler.tag(tokens);
    final Span[] seqSpans = SequenceLabelerME.dropOverlappingSpans(origSpans);
    return seqSpans;
  }
673397
private static boolean hasBreakToken(AnalyzedTokenReadings[] tokens) {
    for (int i = 0; i < tokens.length && i < MAX_TOKEN_TO_CHECK; i++) {
      if (tokens[i].getToken().equals("-") || tokens[i].getToken().equals("—") || tokens[i].getToken().equals("–")) {
        return true;
      }
    }
    return false;
  }
1321027
protected boolean ignoreToken(AnalyzedTokenReadings[] tokens, int idx) throws IOException {
    List<String> words = new ArrayList<>();
    for (AnalyzedTokenReadings token : tokens) {
      words.add(token.getToken());
    }
    return ignoreWord(words, idx);
  }
629177
public static List<TokensLine> convert(List<TokenEntry> tokens) {
    List<TokensLine> result = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    int startLine = Integer.MIN_VALUE;
    int startIndex = 0;
    int currentIndex = 0;
    for (TokenEntry token : tokens) {
      if (token != TokenEntry.EOF) {
        String value = token.getValue();
        int line = token.getBeginLine();
        if (line != startLine) {
          addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
          startIndex = currentIndex + 1;
          startLine = line;
        }
        currentIndex++;
        sb.append(value);
      }
    }
    addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
    return result;
  }
1362214
private static boolean areTokensValid(Token... tokens)
    {
        for (Token token : tokens)
        {
            if (!isTokenValid(token))
                return false;
        }
        return true;
    }
1352541
public static String getStringFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String tok : tokens) {
      sb.append(tok).append(" ");
    }
    return sb.toString().trim();
  }
568230
private boolean isAtEOL() {
		int next = skipLineSpace(index);
		return next >= tokens.size() || tokens.get(next).kind == NewLine;
	}
526157
Stream<Token> tokenize(String input) {
        List<Token> tokens = new ArrayList<Token>();
        int tokenPos = 0;
        while (!input.isEmpty()) {
            boolean matched = false;
            for (Token.Type tokenType : Token.Type.values()) {
                Matcher matcher = tokenType.pattern.matcher(input);
                if (matcher.find()) {
                    matched = true;
                    input = matcher.replaceFirst("");
                    if (tokenType != Token.Type.WHITESPACE) {
                        tokens.add(new Token(
                            tokenType,
                            matcher.group(),
                            tokenPos
                        ));
                    }
                    tokenPos += matcher.end();
                    break;
                }
            }
            if (!matched) {
                throw new LexerException(input);
            }
        }
        tokens.add(new Token(Token.Type.EOI, null, tokenPos));
        return new Stream<Token>(tokens.toArray(new Token[tokens.size()]));
    }
95377
@Override
    public final boolean incrementToken() throws IOException {


        if (!done) {
            if (readTerm()) return true;
        }

        if (done) {
            resetVariable();
            if (!input.incrementToken()) {
                return false;
            }
            done = false;
        }
        readTerm();
        return true;
    }
433507
private boolean hasTokenInBuffer() {
        matchValid = false;
        matcher.usePattern(delimPattern);
        matcher.region(position, buf.limit());

        // Skip delims first
        if (matcher.lookingAt())
            position = matcher.end();

        // If we are sitting at the end, no more tokens in buffer
        if (position == buf.limit())
            return false;

        return true;
    }
359816
public void setSyntaxTokens(java.util.Collection<SyntaxToken> syntaxTokens) {
        if (syntaxTokens == null) {
            this.syntaxTokens = null;
            return;
        }

        this.syntaxTokens = new java.util.ArrayList<SyntaxToken>(syntaxTokens);
    }
897045
@Override
    public boolean incrementToken() throws IOException {
        //清除所有的词元属性
        clearAttributes();
        Lexeme nextLexeme = _IKImplement.next();
        if (nextLexeme != null) {
            //将Lexeme转成Attributes
            //设置词元文本
            termAtt.append(nextLexeme.getLexemeText());
            //设置词元长度
            termAtt.setLength(nextLexeme.getLength());
            //设置词元位移
            offsetAtt.setOffset(nextLexeme.getBeginPosition(), nextLexeme.getEndPosition());
            //记录分词的最后位置
            endPosition = nextLexeme.getEndPosition();
            //记录词元分类
            typeAtt.setType(nextLexeme.getLexemeTypeString());
            //返会true告知还有下个词元
            return true;
        }
        //返会false告知词元输出完毕
        return false;
    }
1336598
function(token) {
      if (Array.isArray(token)) {
        var tokens = /**@type {!Array.<number>}*/(token);
        while (tokens.length)
          this.tokens.push(tokens.pop());
      } else {
        this.tokens.push(token);
      }
    }
705061
public void push( Token token ) {
        size++;
        if( first == null ) {
            first = token;
            last = token;
            token.previous = null;
            token.next = null;
        } else {
            last.next = token;
            token.previous = last;
            token.next = null;
            last = token;
        }
    }
1166726
synchronized long tryReserveTokens(long tokens, long maxWaitMillis) {
    long now = System.currentTimeMillis();
    long waitUntilNextTokenAvailable = Math.max(0, this.nextTokenAvailableMillis - now);

    updateTokensStored(now);
    if (tokens <= this.tokensStored) {
      this.tokensStored -= tokens;
      return waitUntilNextTokenAvailable;
    }

    double additionalNeededTokens = tokens - this.tokensStored;
    // casting to long will round towards 0
    long additionalWaitForEnoughTokens = (long) (additionalNeededTokens / this.tokensPerMilli) + 1;
    long totalWait = waitUntilNextTokenAvailable + additionalWaitForEnoughTokens;
    if (totalWait > maxWaitMillis) {
      return -1;
    }
    this.tokensStored = this.tokensPerMilli * additionalWaitForEnoughTokens - additionalNeededTokens;
    this.nextTokenAvailableMillis = this.nextTokenAvailableMillis + additionalWaitForEnoughTokens;
    return totalWait;
  }