Consume a token.

        Returns a tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.
507351
def consume_token(self, tokens, index, tokens_len):
        
        del tokens_len

        if tokens[index].type == TokenType.EndInlineRST:
            return _paste_tokens_line_by_line(tokens,
                                              TokenType.RST,
                                              self.begin,
                                              index + 1)
952102
def tokens(cls, tokens):
        
        return cls(Lnk.TOKENS, tuple(map(int, tokens)))
1301784
def consume_token(self, tokens, index, tokens_len):
        
        del tokens_len

        consumption_ended = False

        q_type = self.quote_type

        begin_literal_type = getattr(TokenType,
                                     "Begin{0}QuotedLiteral".format(q_type))
        end_literal_type = getattr(TokenType,
                                   "End{0}QuotedLiteral".format(q_type))

        if (index != self.begin and
                tokens[index].type == begin_literal_type):
            # This is an edge case where a quote begins a line and matched
            # as a quoted region beginning and a quoted region ending.
            # Split the token before and after the quote, mark the
            # quote character itself as an ending and insert both
            # tokens back in, handling the ending afterwards.
            assert _RE_QUOTE_TYPE.match(tokens[index].content[0])

            # Mini-tokenize everything after the first token
            line_tokens = _scan_for_tokens(tokens[index].content[1:])
            end_type = getattr(TokenType,
                               "End{0}QuotedLiteral".format(q_type))
            replacement = [Token(type=end_type,
                                 content=tokens[index].content[0],
                                 line=tokens[index].line,
                                 col=tokens[index].col)]

            for after in line_tokens:
                replacement.append(Token(type=after.type,
                                         content=after.content,
                                         line=(tokens[index].line +
                                               after.line - 1),
                                         col=(tokens[index].col +
                                              after.col - 1)))

            tokens = _replace_token_range(tokens,
                                          index,
                                          index + 1,
                                          replacement)
            consumption_ended = True

        if tokens[index].type == end_literal_type:
            consumption_ended = True

        if consumption_ended:
            end = index + 1
            pasted = ""
            for i in range(self.begin, end):
                pasted += tokens[i].content

            tokens = _replace_token_range(tokens, self.begin, end,
                                          [Token(type=TokenType.QuotedLiteral,
                                                 content=pasted,
                                                 line=tokens[self.begin].line,
                                                 col=tokens[self.begin].col)])

            return (self.begin, len(tokens), tokens)
389047
private void gobble(Iterator iter)
    {
        if (eatTheRest)
        {
            while (iter.hasNext())
            {
                tokens.add(iter.next());
            }
        }
    }
386788
def consume_comments
      if @s.peek(2) == '/*'
        @s.consume
        @s.consume

        if text = @s.scan_until(RE_COMMENT_CLOSE)
          text.slice!(-2, 2)
        else
          # Parse error.
          text = @s.consume_rest
        end

        return create_token(:comment, :value => text)
      end

      nil
    end
1306617
private Token consumeNumber() {
    StringBuilder sb = new StringBuilder();
    int l = line;
    int c = column;
    Token T;
    boolean err;
    boolean isDouble = false;
    if(ch == '-') {
      sb.append(ch);
      nextChar();
    }
    if(ch != '0') {
      err = consumeDigits(sb);
      if(err) {
        return new Token(TokenType.ERROR, sb.toString(), line, column);
      }
    } else {
      sb.append(ch);
      nextChar();
    }
    if(ch == '.') {
      isDouble = true;
      sb.append(ch);
      nextChar();
      err = consumeDigits(sb);
      if(err) {
        return new Token(TokenType.ERROR, sb.toString(), line, column);
      }
    }
    if(ch == 'E' || ch == 'e') {
      isDouble = true;
      sb.append(ch);
      nextChar();
      if(ch == '-') {
        sb.append(ch);
        nextChar();
      } else if(ch == '+') {
        sb.append(ch);
        nextChar();
      }
      err = consumeDigits(sb);
      if(err) {
        return new Token(TokenType.ERROR, sb.toString(), line, column);
      }
    }
    if(isDouble) {
      T = new Token(TokenType.DOUBLE, sb.toString(), l, c);
    } else {
      T = new Token(TokenType.INTEGER, sb.toString(), l, c);
    }
    return T;
  }
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
1095594
def consume(self):
        
        try:
            bytes = self.read()
        except IOError:
            bytes = ''

        self.truncate()

        if len(bytes) > 0:
            self.write(bytes)
            self.seek(0)
256205
def get_cache_token(self, token):
        

        if self.conn is None:
            raise CacheException('Redis is not connected')

        token_data = self.conn.get(token)
        token_data = json.loads(token_data) if token_data else None

        return token_data
997294
def consume(self, kind):
        """"""
        next_token = self.stream.move()
        if next_token.kind != kind:
            raise UnexpectedTokenError(token=next_token, expected_kind=kind)
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
140897
def _consume(self):
        
        self._stream_offset += self._buff_i - self._buf_checkpoint
        self._buf_checkpoint = self._buff_i
853658
def consume_simple_block(input = @tokens)
      start_token = input.current[:node]
      end_token   = BLOCK_END_TOKENS[start_token]

      block = {
        :start  => start_token.to_s,
        :end    => end_token.to_s,
        :value  => [],
        :tokens => [input.current] # Non-standard. Used for serialization.
      }

      block[:tokens].concat(input.collect do
        while token = input.consume
          break if token[:node] == end_token

          input.reconsume
          block[:value] << consume_component_value(input)
        end
      end)

      create_node(:simple_block, block)
    end
178023
def consume(self, seq):
        ''''''
        for kmer in iter_kmers(seq, self.k, canonical=self.canonical):
            self._incr(kmer)
923545
def collect_tokens_until(self, token_type):
        """"""
        self.next()
        if self.current_token.type == token_type:
            return

        while True:
            yield self.current_token

            self.next()
            if self.current_token.type == token_type:
                return

            if self.current_token.type != 'COMMA':
                raise self.error(f'Expected comma but got '
                                 f'{self.current_token.value!r}')
            self.next()
1047609
def consume_declaration(input = @tokens)
      declaration = {}
      value       = []

      declaration[:tokens] = input.collect do
        declaration[:name] = input.consume[:value]

        next_token = input.peek

        while next_token && next_token[:node] == :whitespace
          input.consume
          next_token = input.peek
        end

        unless next_token && next_token[:node] == :colon
          # Parse error.
          #
          # Note: The spec explicitly says to return nothing here, but Simon
          # Sapin's CSS parsing tests expect an error node.
          return create_node(:error, :value => 'invalid')
        end

        input.consume

        until input.peek.nil?
          value << consume_component_value(input)
        end
      end

      # Look for !important.
      important_tokens = value.reject {|token|
        node = token[:node]
        node == :whitespace || node == :comment || node == :semicolon
      }.last(2)

      if important_tokens.size == 2 &&
          important_tokens[0][:node] == :delim &&
          important_tokens[0][:value] == '!' &&
          important_tokens[1][:node] == :ident &&
          important_tokens[1][:value].downcase == 'important'

        declaration[:important] = true
        excl_index = value.index(important_tokens[0])

        # Technically the spec doesn't require us to trim trailing tokens after
        # the !important, but Simon Sapin's CSS parsing tests expect it and
        # tinycss2 does it, so we'll go along with the cool kids.
        value.slice!(excl_index, value.size - excl_index)
      else
        declaration[:important] = false
      end

      declaration[:value] = value
      create_node(:declaration, declaration)
    end
1161883
def NextToken(self):
    
    # If we don't have enough data - feed ourselves: We assume
    # that we must have at least one sector in our buffer.
    if len(self.buffer) < 512:
      if self.Feed() == 0 and not self.buffer:
        return None

    return Lexer.NextToken(self)