Returns a sequence of Tokens.

    Args:
      source: string of C++ source code.

    Yields:
      Token that represents the next token in the source.
178103
def get_tokens(source):
    
    if not source.endswith('\n'):
        source += '\n'

    # Cache various valid character sets for speed.
    valid_identifier_first_chars = VALID_IDENTIFIER_FIRST_CHARS
    valid_identifier_chars = VALID_IDENTIFIER_CHARS
    hex_digits = HEX_DIGITS
    int_or_float_digits = INT_OR_FLOAT_DIGITS
    int_or_float_digits2 = int_or_float_digits | set('.')

    # Ignore tokens while in a #if 0 block.
    count_ifs = 0

    i = 0
    end = len(source)
    while i < end:
        # Skip whitespace.
        while i < end and source[i].isspace():
            i += 1
        if i >= end:
            return

        token_type = UNKNOWN
        start = i
        c = source[i]
        # Find a string token.
        if c in valid_identifier_first_chars or c == '_':
            token_type = NAME
            while source[i] in valid_identifier_chars:
                i += 1
            # String and character constants can look like a name if
            # they are something like L"".
            if source[i] == "'" and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_char(source, start, i)
            elif source[i] == '"' and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_string(source, i)
        elif c == '/' and source[i + 1] == '/':  # Find // comments.
            i = _find(source, '\n', i)
            continue
        elif c == '/' and source[i + 1] == '*':  # Find /* comments. */
            i = _find(source, '*/', i) + 2
            continue
        elif c in '<>':                          # Handle '<' and '>' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            # Do not merge '>>' or '>>=' into a single token
            if new_ch == c and c != '>':
                i += 1
                new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in ':+-&|=':                      # Handle 'XX' and 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == c:
                i += 1
            elif c == '-' and new_ch == '>':
                i += 1
            elif new_ch == '=':
                i += 1
        elif c in '!*^%/':                       # Handle 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in '()[]{}~?;.,':                 # Handle single char tokens.
            token_type = SYNTAX
            i += 1
            if c == '.' and source[i].isdigit():
                token_type = CONSTANT
                i += 1
                while source[i] in int_or_float_digits:
                    i += 1
                # Handle float suffixes.
                for suffix in ('l', 'f'):
                    if suffix == source[i:i + 1].lower():
                        i += 1
                        break
        elif c.isdigit():                        # Find integer.
            token_type = CONSTANT
            if c == '0' and source[i + 1] in 'xX':
                # Handle hex digits.
                i += 2
                while source[i] in hex_digits:
                    i += 1
            else:
                while source[i] in int_or_float_digits2:
                    i += 1
            # Handle integer (and float) suffixes.
            if source[i].isalpha():
                for suffix in ('ull', 'll', 'ul', 'l', 'f', 'u'):
                    size = len(suffix)
                    if suffix == source[i:i + size].lower():
                        i += size
                        break
        elif c == '"':                           # Find string.
            token_type = CONSTANT
            i = _get_string(source, i)
        elif c == "'":                           # Find char.
            token_type = CONSTANT
            i = _get_char(source, start, i)
        elif c == '#':                           # Find pre-processor command.
            token_type = PREPROCESSOR
            got_if = source[i:i + 3] == '#if'
            if count_ifs and source[i:i + 6] == '#endif':
                count_ifs -= 1
                if count_ifs == 0:
                    source = source[:i].ljust(i + 6) + source[i + 6:]
                    continue

            # Handle preprocessor statements (\ continuations).
            while True:
                i1 = source.find('\n', i)
                i2 = source.find('//', i)
                i3 = source.find('/*', i)
                i4 = source.find('"', i)
                # Get the first important symbol (newline, comment, EOF/end).
                i = min([x for x in (i1, i2, i3, i4, end) if x != -1])

                # Handle comments in #define macros.
                if i == i3:
                    i = _find(source, '*/', i) + 2
                    source = source[:i3].ljust(i) + source[i:]
                    continue

                # Handle #include "dir//foo.h" properly.
                if source[i] == '"':
                    i = _find(source, '"', i + 1) + 1
                    continue

                # Keep going if end of the line and the line ends with \.
                if i == i1 and source[i - 1] == '\\':
                    i += 1
                    continue

                if got_if:
                    begin = source.find('(', start, i)
                    if begin == -1:
                        begin = source.find(' ', start)
                    begin = begin + 1
                    s1 = source.find(' ', begin)
                    s2 = source.find(')', begin)
                    s3 = source.find('\n', begin)
                    s = min([x for x in (s1, s2, s3, end) if x != -1])

                    condition = source[begin:s]
                    if (
                        count_ifs or
                        condition == '0' or
                        condition == '__OBJC__'
                    ):
                        count_ifs += 1
                break
        elif c == '\\':                          # Handle \ in code.
            # This is different from the pre-processor \ handling.
            i += 1
            continue
        elif count_ifs:
            # Ignore bogus code when we are inside an #if block.
            i += 1
            continue
        else:
            raise TokenError("unexpected token '{0}'".format(c))

        if count_ifs:
            continue

        assert i > 0
        yield Token(token_type, source[start:i], start, i)
273190
def _get_tokens(morph_fd):
    '''
    '''
    for line in morph_fd:
        line = line.rstrip()   # remove \r\n
        line = line.split(';', 1)[0]  # strip comments
        squash_token = []  # quoted strings get squashed into one token

        if '<(' in line:  # skip spines, which exist on a single line
            assert ')>' in line, 'Missing end of spine'
            continue

        for token in line.replace('(', ' ( ').replace(')', ' ) ').split():
            if squash_token:
                squash_token.append(token)
                if token.endswith('"'):
                    token = ' '.join(squash_token)
                    squash_token = []
                    yield token
            elif token.startswith('"') and not token.endswith('"'):
                squash_token.append(token)
            else:
                yield token
1350278
def _next_token(self, skipws=True):
        """"""
        self._token = next(self._tokens).group(0)
        return self._next_token() if skipws and self._token.isspace() else self._token
1090390
function getChainHeadToken(node) {
            let target = node
            const type = target.type
            while (target.parent.type === type) {
                target = target.parent
            }
            return sourceCode.getFirstToken(target)
        }
533148
def iterSourceCode(paths):
    
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if filename.endswith('.py'):
                        yield os.path.join(dirpath, filename)
        else:
            yield path
1167657
public final List<SequenceLabel> getSequencesFromSpans(final String[] tokens,
      final Span[] seqSpans) {
    final List<SequenceLabel> sequences = new ArrayList<SequenceLabel>();
    for (final Span seqSpan : seqSpans) {
      final String seqString = seqSpan.getCoveredText(tokens);
      final String seqType = seqSpan.getType();
      final SequenceLabel sequence = this.sequenceFactory
          .createSequence(seqString, seqType, seqSpan);
      sequences.add(sequence);
    }
    return sequences;
  }
436984
def get_tokens_unprocessed(self, text):
        
        tokens = self._block_re.split(text)
        tokens.reverse()
        state = idx = 0
        try:
            while True:
                # text
                if state == 0:
                    val = tokens.pop()
                    yield idx, Other, val
                    idx += len(val)
                    state = 1
                # block starts
                elif state == 1:
                    tag = tokens.pop()
                    # literals
                    if tag in ('<%%', '%%>'):
                        yield idx, Other, tag
                        idx += 3
                        state = 0
                    # comment
                    elif tag == '<%#':
                        yield idx, Comment.Preproc, tag
                        val = tokens.pop()
                        yield idx + 3, Comment, val
                        idx += 3 + len(val)
                        state = 2
                    # blocks or output
                    elif tag in ('<%', '<%=', '<%-'):
                        yield idx, Comment.Preproc, tag
                        idx += len(tag)
                        data = tokens.pop()
                        r_idx = 0
                        for r_idx, r_token, r_value in \
                                self.ruby_lexer.get_tokens_unprocessed(data):
                            yield r_idx + idx, r_token, r_value
                        idx += len(data)
                        state = 2
                    elif tag in ('%>', '-%>'):
                        yield idx, Error, tag
                        idx += len(tag)
                        state = 0
                    # % raw ruby statements
                    else:
                        yield idx, Comment.Preproc, tag[0]
                        r_idx = 0
                        for r_idx, r_token, r_value in \
                                self.ruby_lexer.get_tokens_unprocessed(tag[1:]):
                            yield idx + 1 + r_idx, r_token, r_value
                        idx += len(tag)
                        state = 0
                # block ends
                elif state == 2:
                    tag = tokens.pop()
                    if tag not in ('%>', '-%>'):
                        yield idx, Other, tag
                    else:
                        yield idx, Comment.Preproc, tag
                    idx += len(tag)
                    state = 0
        except IndexError:
            return
1244484
def token_load(self, line_number, tokens):
        self.line_number = line_number
        assert tokens[-1] == 0x00, "line code %s doesn't ends with \\x00: %s" % (
            repr(tokens), repr(tokens[-1])
        )

        
        for src, dst in self.tokens_replace_rules:
            log.info("Relace tokens %s with $%02x",
                pformat_byte_hex_list(src), dst
            )
            log.debug("Before..: %s", pformat_byte_hex_list(tokens))
            tokens = list_replace(tokens, src, dst)
            log.debug("After...: %s", pformat_byte_hex_list(tokens))

        self.line_code = tokens[:-1]
1201727
def tokenize_string(source):
    
    line_reader = StringIO(source).readline
    token_generator = tokenize.generate_tokens(line_reader)

    # Loop over all tokens till a backtick (`) is found.
    # Then, take all tokens till the next backtick to form a backtick quoted
    # string.
    for toknum, tokval, _, _, _ in token_generator:
        if tokval == '`':
            tokval = " ".join(it.takewhile(
                lambda tokval: tokval != '`',
                map(operator.itemgetter(1), token_generator)))
            toknum = _BACKTICK_QUOTED_STRING
        yield toknum, tokval
464289
def process_tokens(
    tokens: Sequence[Token], *, end: str = "\n", sep: str = " "
) -> Tuple[str, str]:
    
    # Flatten the list of tokens in case some of them are of
    # class UnicodeSequence:
    flat_tokens = list()  # type: List[Token]
    for token in tokens:
        if isinstance(token, UnicodeSequence):
            flat_tokens.extend(token.tuple())
        else:
            flat_tokens.append(token)

    with_color = _process_tokens(flat_tokens, end=end, sep=sep, color=True)
    without_color = _process_tokens(flat_tokens, end=end, sep=sep, color=False)
    return (with_color, without_color)
1168517
async def chunks(source, n):
    
    async with streamcontext(source) as streamer:
        async for first in streamer:
            xs = select.take(create.preserve(streamer), n-1)
            yield [first] + await aggregate.list(xs)
826595
def _generate_tokens(pat: GenericAny, text: str) -> Iterator[Token]:
    
    rules = _TYPE_RULES
    keys = _TYPE_KEYS
    groupindex = pat.groupindex
    scanner = pat.scanner(text)
    for m in iter(scanner.match, None):
        lastgroup = m.lastgroup
        func = rules[keys[groupindex[lastgroup] - 1]]
        if func is not None:
            yield func(m.group(lastgroup))
1338714
def _tokens_to_subtoken(self, tokens):
        
        ret = []
        for token in tokens:
            ret.extend(
                self._escaped_token_to_subtoken_strings(_escape_token(token, self._alphabet)))
        return ret
840200
def _generate_tokens(self, text):
    
    # This is technically an undocumented API for Python3, but allows us to use the same API as for
    # Python2. See http://stackoverflow.com/a/4952291/328565.
    for index, tok in enumerate(tokenize.generate_tokens(io.StringIO(text).readline)):
      tok_type, tok_str, start, end, line = tok
      yield Token(tok_type, tok_str, start, end, line, index,
                  self._line_numbers.line_to_offset(start[0], start[1]),
                  self._line_numbers.line_to_offset(end[0], end[1]))
923545
def collect_tokens_until(self, token_type):
        """"""
        self.next()
        if self.current_token.type == token_type:
            return

        while True:
            yield self.current_token

            self.next()
            if self.current_token.type == token_type:
                return

            if self.current_token.type != 'COMMA':
                raise self.error(f'Expected comma but got '
                                 f'{self.current_token.value!r}')
            self.next()
379749
def tokens(self):
        
        for subsegment_or_token in self:
            if isinstance(subsegment_or_token, Segment):
                subsegment = subsegment_or_token
                for token in subsegment.tokens():
                    yield token
            else:
                token = subsegment_or_token
                yield token
263339
def recover_codes(tokens: Iterator[Tokenizer]) -> str:
    

    tokens = iter(tokens)

    s = []

    try:
        head = next(tokens)

    except StopIteration:
        return s

    append = s.append
    lineno = head.lineno

    start_indent = colno = head.colno

    append(head.value)

    colno += len(s[-1])

    for each in tokens:
        n = each.lineno - lineno

        if n:
            append('\n' * n)
            lineno = each.lineno
            colno = each.colno

            if colno - start_indent > 0:
                append(' ' * colno)

        else:
            c = each.colno - colno
            if c:
                colno = each.colno
                if c > 0:
                    append(' ' * c)

        append(each.value)

        colno += len(s[-1])

    return s