Yields all tokens in order from first_token through and including last_token. If
    include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT.
502593
def token_range(self, first_token, last_token, include_extra=False):
    
    for i in xrange(first_token.index, last_token.index + 1):
      if include_extra or not is_non_coding_token(self._tokens[i].type):
        yield self._tokens[i]
234042
def _shorten_line(tokens, source, indentation, indent_word,
                  aggressive=False, previous_line=''):
    
    for (token_type,
         token_string,
         start_offset,
         end_offset) in token_offsets(tokens):

        if (
            token_type == tokenize.COMMENT and
            not is_probably_part_of_multiline(previous_line) and
            not is_probably_part_of_multiline(source) and
            not source[start_offset + 1:].strip().lower().startswith(
                ('noqa', 'pragma:', 'pylint:'))
        ):
            # Move inline comments to previous line.
            first = source[:start_offset]
            second = source[start_offset:]
            yield (indentation + second.strip() + '\n' +
                   indentation + first.strip() + '\n')
        elif token_type == token.OP and token_string != '=':
            # Don't break on '=' after keyword as this violates PEP 8.

            assert token_type != token.INDENT

            first = source[:end_offset]

            second_indent = indentation
            if (first.rstrip().endswith('(') and
                    source[end_offset:].lstrip().startswith(')')):
                pass
            elif first.rstrip().endswith('('):
                second_indent += indent_word
            elif '(' in first:
                second_indent += ' ' * (1 + first.find('('))
            else:
                second_indent += indent_word

            second = (second_indent + source[end_offset:].lstrip())
            if (
                not second.strip() or
                second.lstrip().startswith('#')
            ):
                continue

            # Do not begin a line with a comma
            if second.lstrip().startswith(','):
                continue
            # Do end a line with a dot
            if first.rstrip().endswith('.'):
                continue
            if token_string in '+-*/':
                fixed = first + ' \\' + '\n' + second
            else:
                fixed = first + '\n' + second

            # Only fix if syntax is okay.
            if check_syntax(normalize_multiline(fixed)
                            if aggressive else fixed):
                yield indentation + fixed
959485
def consume_token(self, tokens, index, tokens_len):
        
        finished = False

        if tokens[index].line > self.line:
            finished = True
            end = index
        elif index == tokens_len - 1:
            finished = True
            end = index + 1

        if finished:
            pasted_together_contents = ""
            for i in range(self.begin, end):
                pasted_together_contents += tokens[i].content

            replacement = [Token(type=TokenType.Comment,
                                 content=pasted_together_contents,
                                 line=tokens[self.begin].line,
                                 col=tokens[self.begin].col)]

            tokens = _replace_token_range(tokens,
                                          self.begin,
                                          end,
                                          replacement)

            return (self.begin, len(tokens), tokens)
239507
function handleEscaping(tokens, token, index) {
	if (index === 0) {
		return [token];
	}

	const previousToken = tokens[index - 1];

	if (!previousToken.endsWith('\\')) {
		return [...tokens, token];
	}

	return [...tokens.slice(0, index - 1), `${previousToken.slice(0, -1)} ${token}`];
}
343278
function getInclude() {
  const type = tokens[pos].include_type;

  if (type === 1) return getInclude1();
  if (type === 2) return getInclude2();
  if (type === 3) return getInclude3();
  if (type === 4) return getInclude4();
  if (type === 5) return getInclude5();
}
601721
def _yield_all(self, l):
        '''
        
        '''
        if l is not None:
            if type(l) in [list, tuple]:
                for f in l:
                    for x in self._yield(f): yield x
            else:
                for x in self._yield(l): yield x
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
893149
def operatorOperands(tokenlist):
    ""
    it = iter(tokenlist)
    while 1:
        try:
            yield (next(it), next(it))
        except StopIteration:
            break
923545
def collect_tokens_until(self, token_type):
        """"""
        self.next()
        if self.current_token.type == token_type:
            return

        while True:
            yield self.current_token

            self.next()
            if self.current_token.type == token_type:
                return

            if self.current_token.type != 'COMMA':
                raise self.error(f'Expected comma but got '
                                 f'{self.current_token.value!r}')
            self.next()
827310
function checkInclude(i) {
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkIncludeWithKeyframes1(i)) tokens[i].include_type = 9;
  else if (l = checkInclude1(i)) tokens[i].include_type = 1;
  else if (l = checkInclude2(i)) tokens[i].include_type = 2;
  else if (l = checkInclude3(i)) tokens[i].include_type = 3;
  else if (l = checkInclude4(i)) tokens[i].include_type = 4;
  else if (l = checkIncludeWithKeyframes2(i)) tokens[i].include_type = 10;
  else if (l = checkInclude5(i)) tokens[i].include_type = 5;
  else if (l = checkInclude6(i)) tokens[i].include_type = 6;
  else if (l = checkInclude7(i)) tokens[i].include_type = 7;
  else if (l = checkInclude8(i)) tokens[i].include_type = 8;

  return l;
}
1131072
def tokenise(template):
    ''''''
    upto = 0
    lineno = 0

    for m in tag_re.finditer(template):

        start, end = m.span()
        lineno = template.count('\n', 0, start) + 1  # Humans count from 1
        # If there's a gap between our start and the end of the last match,
        # there's a Text node between.
        if upto < start:
            yield Token(TokenType.text, template[upto:start], lineno)
        upto = end

        mode = m.lastgroup
        content = m.group(mode)
        yield Token(TokenType[mode], content, lineno)

    # if the last match ended before the end of the source, we have a tail Text
    # node.
    if upto < len(template):
        yield Token(TokenType.text, template[upto:], lineno)
870548
def commented_out_code_lines(source):
    
    line_numbers = []
    try:
        for t in generate_tokens(source):
            token_type = t[0]
            token_string = t[1]
            start_row = t[2][0]
            line = t[4]

            # Ignore inline comments.
            if not line.lstrip().startswith('#'):
                continue

            if token_type == tokenize.COMMENT:
                stripped_line = token_string.lstrip('#').strip()
                if (
                    ' ' in stripped_line and
                    '#' not in stripped_line and
                    check_syntax(stripped_line)
                ):
                    line_numbers.append(start_row)
    except (SyntaxError, tokenize.TokenError):
        pass

    return line_numbers
1131597
def _Tokenize(template_str, meta_left, meta_right, whitespace):
    """"""
    
    trimlen = len(meta_left)
    token_re = MakeTokenRegex(meta_left, meta_right)
    do_strip = (whitespace == 'strip-line')  # Do this outside loop
    do_strip_part = False
    
    for line in template_str.splitlines(True):  # retain newlines
        if do_strip or do_strip_part:
            line = line.strip()
        
        tokens = token_re.split(line)
        
        # Check for a special case first.  If a comment or "block" directive is on a
        # line by itself (with only space surrounding it), then the space is
        # omitted.  For simplicity, we don't handle the case where we have 2
        # directives, say '{.end} # {#comment}' on a line.
        if len(tokens) == 3:
            # ''.isspace() == False, so work around that
            if (tokens[0].isspace() or not tokens[0]) and \
                    (tokens[2].isspace() or not tokens[2]):
                token = tokens[1][trimlen: -trimlen]
                
                # Check the ones that begin with ## before #
                if token == COMMENT_BEGIN:
                    yield COMMENT_BEGIN_TOKEN, None
                    continue
                if token == COMMENT_END:
                    yield COMMENT_END_TOKEN, None
                    continue
                if token == OPTION_STRIP_LINE:
                    do_strip_part = True
                    continue
                if token == OPTION_END:
                    do_strip_part = False
                    continue
                
                if token.startswith('#'):
                    continue  # The whole line is omitted
                
                token_type, token = _MatchDirective(token)
                if token_type is not None:
                    yield token_type, token  # Only yield the token, not space
                    continue
        
        # The line isn't special; process it normally.
        for i, token in enumerate(tokens):
            if i % 2 == 0:
                yield LITERAL_TOKEN, token
            
            else:  # It's a "directive" in metachracters
                assert token.startswith(meta_left), repr(token)
                assert token.endswith(meta_right), repr(token)
                token = token[trimlen: -trimlen]
                
                # Check the ones that begin with ## before #
                if token == COMMENT_BEGIN:
                    yield COMMENT_BEGIN_TOKEN, None
                    continue
                if token == COMMENT_END:
                    yield COMMENT_END_TOKEN, None
                    continue
                if token == OPTION_STRIP_LINE:
                    do_strip_part = True
                    continue
                if token == OPTION_END:
                    do_strip_part = False
                    continue
                
                # A single-line comment
                if token.startswith('#'):
                    continue
                
                if token.startswith('.'):
                    literal = {
                        '.meta-left': meta_left,
                        '.meta-right': meta_right,
                        '.space': ' ',
                        '.tab': '\t',
                        '.newline': '\n',
                    }.get(token)
                    
                    if literal is not None:
                        yield META_LITERAL_TOKEN, literal
                        continue
                    
                    token_type, token = _MatchDirective(token)
                    if token_type is not None:
                        yield token_type, token
                
                else:  # Now we know the directive is a substitution.
                    yield SUBST_TOKEN, token
993419
def build_tokens_line(self):
        """"""
        logical = []
        comments = []
        length = 0
        prev_row = prev_col = mapping = None
        for token_type, text, start, end, line in self.tokens:
            if token_type in SKIP_TOKENS:
                continue
            if not mapping:
                mapping = [(0, start)]
            if token_type == tokenize.COMMENT:
                comments.append(text)
                continue
            if token_type == tokenize.STRING:
                text = mute_string(text)
            if prev_row:
                (start_row, start_col) = start
                if prev_row != start_row:    # different row
                    prev_text = self.lines[prev_row - 1][prev_col - 1]
                    if prev_text == ',' or (prev_text not in '{[(' and
                                            text not in '}])'):
                        text = ' ' + text
                elif prev_col != start_col:  # different column
                    text = line[prev_col:start_col] + text
            logical.append(text)
            length += len(text)
            mapping.append((length, end))
            (prev_row, prev_col) = end
        self.logical_line = ''.join(logical)
        self.noqa = comments and noqa(''.join(comments))
        return mapping
543116
def _shorten_line_at_tokens_new(tokens, source, indentation,
                                max_line_length):
    
    # Yield the original source so to see if it's a better choice than the
    # shortened candidate lines we generate here.
    yield indentation + source

    parsed_tokens = _parse_tokens(tokens)

    if parsed_tokens:
        # Perform two reflows. The first one starts on the same line as the
        # prefix. The second starts on the line after the prefix.
        fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
                              start_on_prefix_line=True)
        if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
            yield fixed

        fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
                              start_on_prefix_line=False)
        if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
            yield fixed
264174
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 is_training):
    """"""

    # Swag is a multiple choice task. To perform this task using Bert,
    # we will use the formatting proposed in "Improving Language
    # Understanding by Generative Pre-Training" and suggested by
    # @jacobdevlin-google in this issue
    # https://github.com/google-research/bert/issues/38.
    #
    # Each choice will correspond to a sample on which we run the
    # inference. For a given Swag example, we will create the 4
    # following inputs:
    # - [CLS] context [SEP] choice_1 [SEP]
    # - [CLS] context [SEP] choice_2 [SEP]
    # - [CLS] context [SEP] choice_3 [SEP]
    # - [CLS] context [SEP] choice_4 [SEP]
    # The model will output a single value for each input. To get the
    # final decision of the model, we will run a softmax over these 4
    # outputs.
    features = []
    for example_index, example in enumerate(examples):
        context_tokens = tokenizer.tokenize(example.context_sentence)
        start_ending_tokens = tokenizer.tokenize(example.start_ending)

        choices_features = []
        for ending_index, ending in enumerate(example.endings):
            # We create a copy of the context tokens in order to be
            # able to shrink it according to ending_tokens
            context_tokens_choice = context_tokens[:]
            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)
            # Modifies `context_tokens_choice` and `ending_tokens` in
            # place so that the total length is less than the
            # specified length.  Account for [CLS], [SEP], [SEP] with
            # "- 3"
            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

            tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]
            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            padding = [0] * (max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            segment_ids += padding

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            choices_features.append((tokens, input_ids, input_mask, segment_ids))

        label = example.label
        if example_index < 5:
            logger.info("*** Example ***")
            logger.info("swag_id: {}".format(example.swag_id))
            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):
                logger.info("choice: {}".format(choice_idx))
                logger.info("tokens: {}".format(' '.join(tokens)))
                logger.info("input_ids: {}".format(' '.join(map(str, input_ids))))
                logger.info("input_mask: {}".format(' '.join(map(str, input_mask))))
                logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids))))
            if is_training:
                logger.info("label: {}".format(label))

        features.append(
            InputFeatures(
                example_id = example.swag_id,
                choices_features = choices_features,
                label = label
            )
        )

    return features
495231
def including(self, sequence) -> Generator:
        """"""
        return (element for element in sequence
                if self.indexer(element) in self.predicates)