Analyse next token.
771013
public final void nextToken() {
        skipIgnoredToken();
        if (isVariableBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
        } else if (isNCharBegin()) {
            currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
        } else if (isIdentifierBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
        } else if (isHexDecimalBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
        } else if (isNumberBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
        } else if (isSymbolBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
        } else if (isCharsBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanChars();
        } else if (isEnd()) {
            currentToken = new Token(Assist.END, "", offset);
        } else {
            throw new SQLParsingException(this, Assist.ERROR);
        }
        offset = currentToken.getEndPosition();
    }
831319
def parse(self):
        """"""
        expr = self.expression()
        if not isinstance(self.current_token, EndToken):
            raise InvalidTokenError("Unconsumed trailing tokens.")
        return expr
901949
def peek(self, steps=1):
        """"""
        try:
            tokens = iter(self)
            for _ in six.moves.range(steps):
                next(tokens)

            return next(tokens)
        except StopIteration:
            return None
691366
def next(self):
        """"""
        if self.tokens:
            other = self.copy()
            tok = other.tokens.pop()
            other.tokens.append(tok.next())
            return other
        else:
            return Version.inf
867795
def parse(self, text):
        """"""
        token_stream = self.lexer.tokenize(text)
        return self.expr(token_stream, next(token_stream))
340610
def analyse(string = nil)
      raise(ArgumentError, 'Lexer: failed to start analysis: no source given!') unless
        string || @scanner

      self.data = string || @scanner.string

      until @scanner.eos?
        send("parse_#{MODE[@mode]}")
      end

      push([false, '$end'])
    end
1123043
def dump(self):
    """"""
    subtoken_strings = [(i, s)
                        for s, i in six.iteritems(self._subtoken_string_to_id)]
    print(u", ".join(u"{0} : '{1}'".format(i, s)
                     for i, s in sorted(subtoken_strings)))
657625
public function readNextToken()
    {
        if (TokenType::EOF === $this->getTokenType())
        {
            throw new TokenizerException("Cannot read past the end of the stream.", $this->line, $this->linePos, $this->input);
        }

        // GET CURRENT; THEN MOVE POINTER
        $this->tokenValue = $c = $this->nextChar();

        if(null === $c)
        {
            $this->tokenType = TokenType::EOF;
        }

        // SPACE CHARACTER?
        elseif(ctype_space($c))
        {
            // IS LINEBREAK
            if(in_array($c, array("\r", "\n")))
            {
                $this->tokenType = TokenType::LINEBREAK;

                // IS "\r\n"?
                if("\r" === $c && "\n" === $this->currentChar())
                {
                    $this->tokenValue .= "\n";

                    $this->nextChar();
                }

                $this->line ++;

                $this->linePos = 0;
            }

            // IS OTHER CTRL CHAR, LET'S ASSUME IT IS WHITESPACE
            else
            {
                $this->tokenType = TokenType::WHITESPACE;

                while(null !== ($n = $this->currentChar()) && ctype_space($n) && ! in_array($n, array("\r", "\n")))
                {
                    $this->tokenValue .= $n;

                    $this->nextChar();
                }
            }

            // SKIP DA WHITESPACE
            if($this->skipWhitespace)
            {
                $this->readNextToken();
            }
        }
        elseif(in_array($c, array('+', '-')))
        {
            $this->tokenType = TokenType::SIGN;
        }

        // NUMBER, COULD BE
        // INT,
        // RATIONAL NUMNER (DECIMAL, CURRENCY)
        // DATE LITERAL
        // DATE TIME LITERAL
        // SIGNED NUMBER (+/-)
        elseif(ctype_digit($c))
        {
            // ASSERT: NUMBER
            $this->tokenType = TokenType::NUMBER;

            while(null !== ($n = $this->currentChar()) && ! ctype_space($n) && ! in_array($n, self::$rightDelimiters))
            {
                $this->nextChar();

                if( ! ctype_digit($n) && ! in_array($n, array(
                    '.', '-', 'T', 'Z', ':', '+'
                ))) {
                    throw new TokenizerException('Unexpected char "' . $n . '"', $this->line, $this->linePos, $this->input);
                }

                if('-' === $n && $this->tokenType !== TokenType::DATETIME_LITERAL)
                {
                    $this->tokenType = TokenType::DATE_LITERAL;
                }

                if('T' === $n)
                {
                    $this->tokenType = TokenType::DATETIME_LITERAL;
                }

                // PUSH TOKEN CHAR
                $this->tokenValue .= $n;

                // CHECK FOR COLON (INCLUDED IN DELIMITERS, SO ADVANCE IF GOT AND IS TIMESTAMP
                if(':' === $this->currentChar() && $this->tokenType === TokenType::DATETIME_LITERAL)
                {
                    $this->tokenValue .= $this->currentChar();

                    $this->nextChar();
                }
            }

            // VALIDATE DATE/DATETIME LITERALS
            if(TokenType::DATE_LITERAL === $this->getTokenType() && false === ($this->tokenValue = \DateTime::createFromFormat('Y-m-d', $this->getTokenValue())))
            {
                throw new TokenizerException('Unexpected literal format "' . $this->getTokenValue() . '"', $this->line, $this->linePos, $this->input);
            }

            // VALIDATE DATE/DATETIME LITERALS
            if(TokenType::DATETIME_LITERAL === $this->getTokenType() && false === ($this->tokenValue = \DateTime::createFromFormat(\DateTime::W3C, $this->getTokenValue())))
            {
                throw new TokenizerException('Unexpected literal format "' . $this->getTokenValue() . '"', $this->line, $this->linePos, $this->input);
            }
        }

        // STRING LITERALS
        elseif(in_array($c, array('\'', '"')))
        {
            $closed = false;

            $this->tokenType = TokenType::STRING_LITERAL;

            while(null !== ($n = $this->currentChar()))
            {
                $this->nextChar();

                $n2 = $this->currentChar();

                if('\\' === $n)
                {
                    // CHECK ALLOWED ESCAPE SEQUENCES
                    if(in_array($n2, array('n', 'M', 'r', 'R', 't', 'T', 'b', 'B', 'f', 'F', $c, '\\', '_', '%')))
                    {
                        $this->tokenValue .= '\\' . $n2;

                        continue;
                    }
                    else
                    {
                        throw new TokenizerException(sprintf('Unrecognized escape sequence "%s"', '\\' . $n2), $this->line, $this->linePos, $this->input);
                    }
                }
                else
                {
                    $this->tokenValue .= $n;

                    if($c === $n2)
                    {
                        $closed = true;

                        $this->tokenValue .= $n2;

                        $this->nextChar();

                        break;
                    }
                }
            }

            if( ! $closed)
            {
                throw new TokenizerException('Unterminated string literal', $this->line, $this->linePos, $this->input);
            }
        }

        // ARBITRARY CHAR (paranthesis, comma separator, ...)
        elseif(isset(self::$charMap[$c]))
        {
            $this->tokenType = self::$charMap[$c];
        }

        // (MATH.) [COMPOUNT] COMPARISON OPERATORS
        elseif(in_array($c, array('<', '>', '=', '!')))
        {
            $n = $this->currentChar();

            switch($c)
            {
                case '!':
                    if($n !== '=')
                    {
                        throw new TokenizerException('Unexpected "'.$n.'"', $this->line, $this->linePos, $this->input);
                    }
                    $this->tokenValue = '!=';

                    $this->tokenType = TokenType::OP_NE;

                    $this->nextChar();

                    break;

                case '<':
                    if('=' === $n)
                    {
                        $this->tokenValue = '<=';

                        $this->tokenType = TokenType::OP_LTE;

                        $this->nextChar();
                    }
                    else
                    {
                        $this->tokenType = TokenType::OP_LT;
                    }
                    break;

                case '>':
                    if('=' === $n)
                    {
                        $this->tokenValue = '>=';

                        $this->tokenType = TokenType::OP_GTE;

                        $this->nextChar();
                    }
                    else
                    {
                        $this->tokenType = TokenType::OP_GT;
                    }
                    break;

                case '=':
                    $this->tokenType = TokenType::OP_EQ;

                    break;
            }
        }

        // KEYWORDS;
        // - DATE LITERALS ("NEXT_N_YEARS:231"),
        // - ARBRITRARY EXPRESSIONS
        //   - LOGICAL OPERATORS
        //   - COMPARISON OPERATORS
        //   - FIELDNAMES
        //   - SOQL-KEYWORDS ("SELECT")
        else
        {
            // "DYNAMIC" EXPRESSIONS, CURRENTLY ONLY SUPPORTED BY DATE LITERALS
            // EXAMPLE: WHERE dateColumn < LAST_N_MONTHS:3
            $is_dynamic = false;

            $p1 = $c;

            $p2 = '';

            while(null !== ($n = $this->currentChar()) && ! ctype_space($n) && ! in_array($n, self::$rightDelimiters))
            {
                $this->nextChar();

                if( ! $is_dynamic)
                {
                    $p1 .= $n;
                }

                else
                {
                    $p2 .= $n;
                }

                $this->tokenValue .= $n;

                if(':' === $n)
                {
                    if($is_dynamic)
                    {
                        throw new TokenizerException('Unexpected ":"', $this->line, $this->linePos, $this->input);
                    }
                    $is_dynamic = true;
                }
            }

            if($is_dynamic && ! $p2)
            {
                throw new TokenizerException('Unexpected ":"', $this->line, $this->linePos, $this->input);
            }

            $t = strtoupper($p1);

            // SOQL KEYWORDS
            if(in_array($t, self::$keywords))
            {
                $this->tokenType = TokenType::KEYWORD;
            }

            else
            {
                $this->tokenType = TokenType::EXPRESSION;
            }
        }
    }
225981
public function getNext()
    {
        for (; $this->idx < $this->count; ++$this->idx) {
            if (($this->tokens[$this->idx]->type !== Token::TYPE_WHITESPACE)
                && ($this->tokens[$this->idx]->type !== Token::TYPE_COMMENT)
            ) {
                return $this->tokens[$this->idx++];
            }
        }

        return null;
    }
612431
def analysis(self):
        """"""
        if not self.is_tagged(ANALYSIS):
            self.tag_analysis()
        return [word[ANALYSIS] for word in self.words]
731933
@Requires("lookahead >= 0")
  public Token peek(int lookahead) {
    ArrayDeque<Token> tmp = new ArrayDeque<Token>();
    for (int i = 0; i <= lookahead; ++i) {
      if (hasNext()) {
        tmp.push(next());
      }
    }
    Token token = tmp.peek();
    while (!tmp.isEmpty()) {
      pushback(tmp.pop());
    }
    return token;
  }
268039
def next(self):
        """"""
        if self._current_index < len(self._collection):
            value = self._collection[self._current_index]
            self._current_index += 1
            return value
        elif self._next_cursor:
            self.__fetch_next()
            return self.next()
        else:
            self._current_index = 0
            raise StopIteration
603189
def _getPOS( self, token, onlyFirst = True ):
        ''' 
        '''
        if onlyFirst:
            return token[ANALYSIS][0][POSTAG]
        else:
            return [ a[POSTAG] for a in token[ANALYSIS] ]
1084467
private final void nextToken()
  {

    if (m_queueMark < m_ops.getTokenQueueSize())
    {
      m_token = (String) m_ops.m_tokenQueue.elementAt(m_queueMark++);
      m_tokenChar = m_token.charAt(0);
    }
    else
    {
      m_token = null;
      m_tokenChar = 0;
    }
  }
734043
def contains(self, token: str) -> bool:
        """"""
        self._validate_token(token)
        return token in self
870467
def _advance(self):
        
        cur_val = None if self.cur_token is None else self.cur_token.value
        try:
            self.cur_token = next(self._tokenizer)
        except StopIteration:
            self.cur_token = None
        return cur_val
69555
public void nextToken() {
        byte ct[] = ctype;
        int c;
        int lctype;
        sval = null;
        isQuoted = false;

        do {
            c = read();
            if (c < 0) {
                return;
            }
            lctype = (c < 256) ? ct[c] : unicode2ctype(c);
        } while (lctype == CT_WHITESPACE);

        if (lctype == CT_ALPHA) {
            int i = 0;
            do {
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char) c;
                c = read();
                lctype = c < 0 ? CT_WHITESPACE : (c < 256)? ct[c] : unicode2ctype(c);
            } while (lctype == CT_ALPHA);
            if (c >= 0) --next; // push last back
            sval = String.copyValueOf(buf, 0, i);
            return;
        }

        if (lctype == CT_QUOTE) {
            int quote = c;
            int i = 0;
            /* Invariants (because \Octal needs a lookahead):
             *   (i)  c contains char value
             *   (ii) d contains the lookahead
             */
            int d = read();
            while (d >= 0 && d != quote) {
                if (d == '\\') {
                    c = read();
                    int first = c;   /* To allow \377, but not \477 */
                    if (c >= '0' && c <= '7') {
                        c = c - '0';
                        int c2 = read();
                        if ('0' <= c2 && c2 <= '7') {
                            c = (c << 3) + (c2 - '0');
                            c2 = read();
                            if ('0' <= c2 && c2 <= '7' && first <= '3') {
                                c = (c << 3) + (c2 - '0');
                                d = read();
                            } else
                                d = c2;
                        } else
                          d = c2;
                    } else {
                        switch (c) {
                        case 'a':
                            c = 0x7;
                            break;
                        case 'b':
                            c = '\b';
                            break;
                        case 'f':
                            c = 0xC;
                            break;
                        case 'n':
                            c = '\n';
                            break;
                        case 'r':
                            c = '\r';
                            break;
                        case 't':
                            c = '\t';
                            break;
                        case 'v':
                            c = 0xB;
                            break;
                        }
                        d = read();
                    }
                } else {
                    c = d;
                    d = read();
                }
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char)c;
            }

            if (d == quote) {
                isQuoted = true;
            }
            sval = String.copyValueOf(buf, 0, i);
        }
    }