Returns the next token after the given one. If include_extra is True, includes non-coding
    tokens from the tokenize module, such as NL and COMMENT.
177567
def next_token(self, tok, include_extra=False):
    
    i = tok.index + 1
    if not include_extra:
      while is_non_coding_token(self._tokens[i].type):
        i += 1
    return self._tokens[i]
521734
function stuff_after(comment, ast) { 
  var code = code_after(comment["range"], ast);

  if (code && comment["next"])
    return code["range"][0] < comment["next"]["range"][0] ? code : "";
  else
    return code;
}
431944
def next_chunk(self):
        
        raise NotImplementedError("%s not implemented for %s" % (self.next_chunk.__func__.__name__,
                                                                 self.__class__.__name__))
724002
def next_token(self, tokenum, value, scol):
        """"""

        # Make self.current reflect these values
        self.current.set(tokenum, value, scol)

        # Determine indent_type based on this token
        if self.current.tokenum == INDENT and self.current.value:
            self.indent_type = self.current.value[0]

        # Only proceed if we shouldn't ignore this token
        if not self.ignore_token():
            # Determining if this token is whitespace
            self.determine_if_whitespace()

            # Determine if inside a container
            self.determine_inside_container()

            # Change indentation as necessary
            self.determine_indentation()

            # See if we are force inserting this token
            if self.forced_insert():
                return

            # If we have a newline after an inserted line, then we don't need to worry about semicolons
            if self.inserted_line and self.current.tokenum == NEWLINE:
                self.inserted_line = False

            # If we have a non space, non comment after an inserted line, then insert a semicolon
            if self.result and not self.is_space and self.inserted_line:
                if self.current.tokenum != COMMENT:
                    self.result.append((OP, ';'))
                self.inserted_line = False

            # Progress the tracker
            self.progress()

            # Add a newline if we just skipped a single
            if self.single and self.single.skipped:
                self.single.skipped = False
                self.result.append((NEWLINE, '\n'))

            # Set after_space so next line knows if it is after space
            self.after_space = self.is_space
874148
def next_token(self):
        """"""
        if self.lookahead:
            self.current_token = self.lookahead.popleft()
            return self.current_token

        self.current_token = self._parse_next_token()
        return self.current_token
522420
def get_next_token(self, length=15, **kwargs):
        
        return self.get_available_tokens(count=1,
                                         token_length=length,
                                         **kwargs)[0]
69555
public void nextToken() {
        byte ct[] = ctype;
        int c;
        int lctype;
        sval = null;
        isQuoted = false;

        do {
            c = read();
            if (c < 0) {
                return;
            }
            lctype = (c < 256) ? ct[c] : unicode2ctype(c);
        } while (lctype == CT_WHITESPACE);

        if (lctype == CT_ALPHA) {
            int i = 0;
            do {
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char) c;
                c = read();
                lctype = c < 0 ? CT_WHITESPACE : (c < 256)? ct[c] : unicode2ctype(c);
            } while (lctype == CT_ALPHA);
            if (c >= 0) --next; // push last back
            sval = String.copyValueOf(buf, 0, i);
            return;
        }

        if (lctype == CT_QUOTE) {
            int quote = c;
            int i = 0;
            /* Invariants (because \Octal needs a lookahead):
             *   (i)  c contains char value
             *   (ii) d contains the lookahead
             */
            int d = read();
            while (d >= 0 && d != quote) {
                if (d == '\\') {
                    c = read();
                    int first = c;   /* To allow \377, but not \477 */
                    if (c >= '0' && c <= '7') {
                        c = c - '0';
                        int c2 = read();
                        if ('0' <= c2 && c2 <= '7') {
                            c = (c << 3) + (c2 - '0');
                            c2 = read();
                            if ('0' <= c2 && c2 <= '7' && first <= '3') {
                                c = (c << 3) + (c2 - '0');
                                d = read();
                            } else
                                d = c2;
                        } else
                          d = c2;
                    } else {
                        switch (c) {
                        case 'a':
                            c = 0x7;
                            break;
                        case 'b':
                            c = '\b';
                            break;
                        case 'f':
                            c = 0xC;
                            break;
                        case 'n':
                            c = '\n';
                            break;
                        case 'r':
                            c = '\r';
                            break;
                        case 't':
                            c = '\t';
                            break;
                        case 'v':
                            c = 0xB;
                            break;
                        }
                        d = read();
                    }
                } else {
                    c = d;
                    d = read();
                }
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char)c;
            }

            if (d == quote) {
                isQuoted = true;
            }
            sval = String.copyValueOf(buf, 0, i);
        }
    }
744953
protected function _getNextToken()
    {
        $this->_token      = self::EOF;
        $this->_tokenValue = null;
        $this->_eatWhitespace();

        if ($this->_offset >= $this->_sourceLength) {
            return(self::EOF);
        }

        $str        = $this->_source;
        $str_length = $this->_sourceLength;
        $i          = $this->_offset;
        $start      = $i;

        switch ($str{$i}) {
        case '{':
            $this->_token = self::LBRACE;
            break;
        case '}':
            $this->_token = self::RBRACE;
            break;
        case '[':
            $this->_token = self::LBRACKET;
            break;
        case ']':
            $this->_token = self::RBRACKET;
            break;
        case ',':
            $this->_token = self::COMMA;
            break;
        case ':':
            $this->_token = self::COLON;
            break;
        case  '"':
            $result = '';
            do {
                $i++;
                if ($i >= $str_length) {
                    break;
                }

                $chr = $str{$i};
                if ($chr == '\\') {
                    $i++;
                    if ($i >= $str_length) {
                        break;
                    }
                    $chr = $str{$i};
                    switch ($chr) {
                    case '"' :
                        $result .= '"';
                        break;
                    case '\\':
                        $result .= '\\';
                        break;
                    case '/' :
                        $result .= '/';
                        break;
                    case 'b' :
                        $result .= chr(8);
                        break;
                    case 'f' :
                        $result .= chr(12);
                        break;
                    case 'n' :
                        $result .= chr(10);
                        break;
                    case 'r' :
                        $result .= chr(13);
                        break;
                    case 't' :
                        $result .= chr(9);
                        break;
                    case '\'' :
                        $result .= '\'';
                        break;
                    default:
                        throw new Zend_Json_Exception(
                            "Illegal escape "
                                .  "sequence '" . $chr . "'"
                        );
                    }
                } elseif ($chr == '"') {
                    break;
                } else {
                    $result .= $chr;
                }
            } while ($i < $str_length);

            $this->_token = self::DATUM;
            //$this->_tokenValue = substr($str, $start + 1, $i - $start - 1);
            $this->_tokenValue = $result;
            break;
        case 't':
            if (($i+ 3) < $str_length && substr($str, $start, 4) == "true") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = true;
            $i += 3;
            break;
        case 'f':
            if (($i+ 4) < $str_length && substr($str, $start, 5) == "false") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = false;
            $i += 4;
            break;
        case 'n':
            if (($i+ 3) < $str_length && substr($str, $start, 4) == "null") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = null;
            $i += 3;
            break;
        }

        if ($this->_token != self::EOF) {
            $this->_offset = $i + 1; // Consume the last token character
            return($this->_token);
        }

        $chr = $str{$i};
        if ($chr == '-' || $chr == '.' || ($chr >= '0' && $chr <= '9')) {
            if (preg_match(
                '/-?([0-9])*(\.[0-9]*)?((e|E)((-|\+)?)[0-9]+)?/s',
                $str, $matches, PREG_OFFSET_CAPTURE, $start
            ) && $matches[0][1] == $start
            ) {

                $datum = $matches[0][0];

                if (is_numeric($datum)) {
                    if (preg_match('/^0\d+$/', $datum)) {
                        throw new Zend_Json_Exception("Octal notation not supported by JSON (value: $datum)");
                    } else {
                        $val  = intval($datum);
                        $fVal = floatval($datum);
                        $this->_tokenValue = ($val == $fVal ? $val : $fVal);
                    }
                } else {
                    throw new Zend_Json_Exception("Illegal number format: $datum");
                }

                $this->_token = self::DATUM;
                $this->_offset = $start + strlen($datum);
            }
        } else {
            throw new Zend_Json_Exception('Illegal Token');
        }

        return($this->_token);
    }
337462
function processNonSpecifier() {
                    const secondToken = sourceCode.getFirstToken(node, 1)
                    if (isLeftBrace(secondToken)) {
                        setOffsetToToken(
                            [
                                secondToken,
                                sourceCode.getTokenAfter(secondToken),
                            ],
                            0,
                            importToken
                        )
                        tokens.push(
                            sourceCode.getLastToken(node, hasSemi ? 2 : 1), // from
                            sourceCode.getLastToken(node, hasSemi ? 1 : 0) // "foo"
                        )
                    } else {
                        tokens.push(
                            sourceCode.getLastToken(node, hasSemi ? 1 : 0)
                        )
                    }
                }
96962
function isLastTokenInCurrentLine(token) {
            const tokenAfter = sourceCode.getTokenAfter(token);

            return !(tokenAfter && astUtils.isTokenOnSameLine(token, tokenAfter));
        }
1037626
function getFirstBlockToken(token) {
      let prev;
      let first = token;

      do {
        prev = first;
        first = sourceCode.getTokenAfter(first, { includeComments: true });
      } while (isComment(first) && first.loc.start.line === prev.loc.end.line);

      return first;
    }
870467
def _advance(self):
        
        cur_val = None if self.cur_token is None else self.cur_token.value
        try:
            self.cur_token = next(self._tokenizer)
        except StopIteration:
            self.cur_token = None
        return cur_val
994469
def next(self):
        
        try:
            result = self.data[self.index]
        except IndexError:
            self.index = 0
            raise StopIteration
        self.index += 1
        return result
664530
def extra(self, **kwargs):
        
        s = self._clone()
        if 'from_' in kwargs:
            kwargs['from'] = kwargs.pop('from_')
        s._extra.update(kwargs)
        return s
454068
private Token lookahead(int n) {
        Token current = token;
        for (int i = 0; i < n; i++) {
            if (current.next == null) {
                current.next = token_source.getNextToken();
            }
            current = current.next;
        }
        return current;
    }
590878
public String nextToken() {
        /*
         * If next position already computed in hasMoreElements() and
         * delimiters have changed between the computation and this invocation,
         * then use the computed value.
         */

        currentPosition = (newPosition >= 0 && !delimsChanged) ?
            newPosition : skipDelimiters(currentPosition);

        /* Reset these anyway */
        delimsChanged = false;
        newPosition = -1;

        if (currentPosition >= maxPosition)
            throw new NoSuchElementException();
        int start = currentPosition;
        currentPosition = scanToken(currentPosition);
        return str.substring(start, currentPosition);
    }
1040991
def _next(self):
        
        c = self._get()
        if c == '/':
            p = self._peek()
            if p == '/':
                c = self._get()
                while c > '\n':
                    c = self._get()
                return c
            if p == '*':
                c = self._get()
                while 1:
                    c = self._get()
                    if c == '*':
                        if self._peek() == '/':
                            self._get()
                            return ' '
                    if c == '\000':
                        raise UnterminatedComment()

        return c