Yield tokens and offsets.
766321
def token_offsets(tokens):
    """"""
    end_offset = 0
    previous_end_row = 0
    previous_end_column = 0
    for t in tokens:
        token_type = t[0]
        token_string = t[1]
        (start_row, start_column) = t[2]
        (end_row, end_column) = t[3]

        # Account for the whitespace between tokens.
        end_offset += start_column
        if previous_end_row == start_row:
            end_offset -= previous_end_column

        # Record the start offset of the token.
        start_offset = end_offset

        # Account for the length of the token itself.
        end_offset += len(token_string)

        yield (token_type,
               token_string,
               start_offset,
               end_offset)

        previous_end_row = end_row
        previous_end_column = end_column
366506
def pieces(array, chunk_size):
        """"""
        for i in range(0, len(array), chunk_size): yield array[i:i+chunk_size]
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char
48568
def _lex_fortran(self, match, ctx=None):
        """"""
        lexer = FortranLexer()
        text = match.group(0) + "\n"
        for index, token, value in lexer.get_tokens_unprocessed(text):
            value = value.replace('\n', '')
            if value != '':
                yield index, token, value
1052752
def split_sentences(tokens):
    """"""
    begin = 0
    for i, token in enumerate(tokens):
        if is_end_of_sentence(tokens, i):
            yield tokens[begin:i+1]
            begin = i+1
    if begin <= len(tokens)-1:
        yield tokens[begin:]
306189
def iflatten(L):
    """"""
    for sublist in L:
        if hasattr(sublist, '__iter__'):
            for item in iflatten(sublist): yield item
        else: yield sublist
736305
def tokenize(self):
        
        def readlines(s):
            while True:
                line = s.readline()
                if line != "":
                    yield line.rstrip()
                else:
                    break

        for self.lineno, line in enumerate(readlines(self.stream), 1):
            for self.column, part in self.split(line):
                if part[0] == "#": # COMMENT
                    break
                yield (self.lineno, self.column, self.tokentype(part))
61501
def _tokens_from_patsy(node):
    
    for n in node.args:
        for t in _tokens_from_patsy(n):
            yield t

    if node.token:
        yield node.token
1176346
def stream_lines(tokens, line_num)
      yield line_start(line_num)

      tokens.each do |token, value|
        yield @inner.span(token, value)
      end

      yield line_end(line_num)
    end
826595
def _generate_tokens(pat: GenericAny, text: str) -> Iterator[Token]:
    
    rules = _TYPE_RULES
    keys = _TYPE_KEYS
    groupindex = pat.groupindex
    scanner = pat.scanner(text)
    for m in iter(scanner.match, None):
        lastgroup = m.lastgroup
        func = rules[keys[groupindex[lastgroup] - 1]]
        if func is not None:
            yield func(m.group(lastgroup))
437414
def process_tokens(self, tokens):
        
        """
        for (tok_type, token, (start_row, _), _, _) in tokens:
            if tok_type == tokenize.STRING:
                self._check_string(token, start_row)
99250
def __get_tokens(self, row):
        """"""
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            # print("RETURN line", line[0:100])
            yield line
            line = row_tokenizer.next()
819035
def chunks(l, n):
    
    for i in _range(0, len(l), n):
        yield l[i:i + n]
430200
def _iterate_subsequences(self, tokens):
        
        for start_idx in xrange(len(tokens)):
            for end_idx in xrange(start_idx + 1, len(tokens) + 1):
                yield ' '.join(tokens[start_idx:end_idx]), start_idx
402359
def offset_gen(offset, iterable, skip_signal=None):
  '''
  '''
  offset = int(offset)
  assert offset >= 0, 'negative offset'

  for item in iterable:
    if offset > 0:
      offset -= 1
      if callable(skip_signal):
        skip_signal(item)
    else:
      yield item
1267331
def _get_batches(self, items):
        """"""
        for i in xrange(0, len(items), self.max_batch_size):
            yield items[i:i + self.max_batch_size]
1080395
def combine(self, a, b):
        """"""

        for l in (a, b):
            for x in l:
                yield x