Reads next token from text.

@return Token
767694
public function readToken(): Token
    {
        // List of types is reverse-sorted by frequency for typical locator.
        $typeList = [
            Token::TYPE_UNESCAPED,
            Token::TYPE_SLASH,
            Token::TYPE_ESCAPED,
        ];
        $token = $this->matchNextToken($typeList);
        if (null === $token) {
            // Checking for known syntax errors.
            $errorTypeList = [
                Token::TYPE_ERROR_INVALID_ESCAPE,
            ];
            $token = $this->matchNextToken($errorTypeList);
        }
        if (null === $token) {
            throw new UnknownSyntaxException("Unknown syntax error in source text");
        }
        return $token;
    }
238253
public String nextToken() 
  {
    int delPos;
    /* Scan for the next delimiter or end of data */
    for (delPos = position; delPos < dataLength; delPos++)
    {
      char c = data.charAt (delPos);
      if (delimiters.indexOf (c) != -1)
        break;
    }
    /* Find out what the token should be */
    String ret = data.substring (position, delPos);
    /* Set the position to the next position */
    position = delPos+1;
    return ret;
  }
20481
public function NextToken()
	{
		$anchor = $this->_offset;
		$ch = '\0';
		$this->Value = "";
		while ( $this->_offset < strlen( $this->_text ) )
		{
			$ch = $this->_text[ $this->_offset ];
			if ( preg_match( "/\s/", $ch ) && $ch != '\n' && $ch != '\r' )
				$this->_offset++;
			else
				break;
		}
		switch ( $ch )
		{
			case '\0':

				$this->Token = $ch;
				break;

			case '\n':

				$this->_offset++;
				if ($this->_offset < strlen( $this->_text ) - 1 && $this->_text[ $this->_offset ] == '\r' )
					$this->_offset++;
				$this->LineCount++;
				$this->Token = $ch;
				break;

			case '\r':

				$this->_offset++;
				if ( $this->_offset < strlen( $this->_text.Length ) - 1 && $this->_text[ $this->_offset ] == '\n')
					$this->_offset++;
				$this->LineCount++;
				$this->Token = '\n';
				break;

			default:

				if ( is_numeric( $ch ) )
				{
					$sb = array();
					while ( $this->_offset < strlen( $this->_text ) )
					{
						$ch = $this->_text[ $this->_offset ];
						if ( ! is_numeric( $ch ) )
							break;

						$sb[] = $ch;
						$this->_offset++;
					}
					$this->Token = StringTokenizer::TokenInt;
					$this->Value = implode( "", $sb );
				}
				else
				{
					$this->Token = $ch;
					$this->_offset++;
				}
				break;

		}
		return $this->Token;
	}
1015494
protected LexToken readToken() throws LexException
	{
		LexToken tok = reader.getLast();
		reader.nextToken();
		return tok;
	}
840394
public function getToken()
    {
        if(isset($this->_tokens[$this->_position])) {
            return $this->_tokens[$this->_position];
        } else {
            return null;
        }
    }
632370
def next(self):
        
        if self.current_token == len(self.tokens):
            return None
        token = self.tokens[self.current_token]
        if token["type"] == "newline":
            self.line += 1
            self.line_start = token["start"]
        self.current_token += 1
        if token["type"] == "unknown":
            self.error("Unknown token")
        return token
363201
public function parseLabel()
    {
        $token = '';

        /**
         * Value to be returned.
         *
         * @var Token
         */
        $ret = null;

        /**
         * The value of `$this->last` where `$token` ends in `$this->str`.
         *
         * @var int
         */
        $iEnd = $this->last;
        for ($j = 1; $j < Context::LABEL_MAX_LENGTH && $this->last < $this->len; ++$j, ++$this->last) {
            if ($this->str[$this->last] === ':' && $j > 1) {
                // End of label
                $token .= $this->str[$this->last];
                $ret = new Token($token, Token::TYPE_LABEL);
                $iEnd = $this->last;
                break;
            } elseif (Context::isWhitespace($this->str[$this->last]) && $j > 1) {
                // Whitespace between label and :
                // The size of the keyword didn't increase.
                --$j;
            } elseif (Context::isSeparator($this->str[$this->last])) {
                // Any other separator
                break;
            }
            $token .= $this->str[$this->last];
        }

        $this->last = $iEnd;

        return $ret;
    }
745120
private Object getNext(String txt, String originalText) {
    if (invertible) {
      String str = prevWordAfter.toString();
      prevWordAfter.setLength(0);
      CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength());
      word.set(OriginalTextAnnotation.class, originalText);
      word.set(BeforeAnnotation.class, str);
      prevWord.set(AfterAnnotation.class, str);
      prevWord = word;
      return word;
    } else {
      return tokenFactory.makeToken(txt, yychar, yylength());
   }
  }
76862
public function nextToken()
    {
        if ($this->_input === null) {
            return null;
        }

        do {
            if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
                // It covers both cases a) there are no matches (preg_match(...) === 0)
                // b) error occured (preg_match(...) === FALSE)
                return null;
            }

            $str = $match[0][0];
            $pos = $match[0][1];
            $endpos = $pos + strlen($str);

            $this->_position = $endpos;

            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
        } while ($token === null); // try again if token is skipped

        return $token;
    }
1328355
public function next()
    {
        if (!mb_strlen($this->input)) {
            return false;
        }

        foreach ($this->engine->getPatterns() as $pattern) {
            if ($token = $this->scan($pattern->regex, $pattern->type)) {
                if (in_array($pattern->type, $this->disallow)) {
                    throw new Exception($pattern->type . ' is disallowed.', 3);
                }

                return $token;
            }
        }

        throw new Exception('Unknow pattern found at: ' . mb_substr($this->input, 0, 100), 12);
    }
642874
protected function advance()
    {
        $this->cursor = $this->token->getEnd();
        return $this->token = $this->lexer->readToken($this->cursor);
    }
294783
public Token getNextToken() {
    if (nextToken == null) {
      try {
        if (!lex()) {
          throw new NoSuchElementException();
        }
      } catch (IOException e) {
        errorMessage = e.getMessage();
        hasErrors_ = true;
        throw new NoSuchElementException();
      }
    }
    return nextToken;
  }
250651
protected function nextToken() {
		if ( $this->pos >= $this->end ) {
			return false;
		}

		// Whitespace
		$length = strspn( $this->rule, self::WHITESPACE_CLASS, $this->pos );
		$this->pos += $length;

		if ( $this->pos >= $this->end ) {
			return false;
		}

		// Number
		$length = strspn( $this->rule, self::NUMBER_CLASS, $this->pos );
		if ( $length !== 0 ) {
			$token = $this->newNumber( substr( $this->rule, $this->pos, $length ), $this->pos );
			$this->pos += $length;

			return $token;
		}

		// Two-character operators
		$op2 = substr( $this->rule, $this->pos, 2 );
		if ( $op2 === '..' || $op2 === '!=' ) {
			$token = $this->newOperator( $op2, $this->pos, 2 );
			$this->pos += 2;

			return $token;
		}

		// Single-character operators
		$op1 = $this->rule[$this->pos];
		if ( $op1 === ',' || $op1 === '=' || $op1 === '%' ) {
			$token = $this->newOperator( $op1, $this->pos, 1 );
			$this->pos++;

			return $token;
		}

		// Word
		if ( !preg_match( self::WORD_REGEX, $this->rule, $m, 0, $this->pos ) ) {
			$this->error( 'unexpected character "' . $this->rule[$this->pos] . '"' );
		}
		$word1 = strtolower( $m[0] );
		$word2 = '';
		$nextTokenPos = $this->pos + strlen( $word1 );
		if ( $word1 === 'not' || $word1 === 'is' ) {
			// Look ahead one word
			$nextTokenPos += strspn( $this->rule, self::WHITESPACE_CLASS, $nextTokenPos );
			if ( $nextTokenPos < $this->end
				&& preg_match( self::WORD_REGEX, $this->rule, $m, 0, $nextTokenPos )
			) {
				$word2 = strtolower( $m[0] );
				$nextTokenPos += strlen( $word2 );
			}
		}

		// Two-word operators like "is not" take precedence over single-word operators like "is"
		if ( $word2 !== '' ) {
			$bothWords = "{$word1}-{$word2}";
			if ( isset( self::$precedence[$bothWords] ) ) {
				$token = $this->newOperator( $bothWords, $this->pos, $nextTokenPos - $this->pos );
				$this->pos = $nextTokenPos;

				return $token;
			}
		}

		// Single-word operators
		if ( isset( self::$precedence[$word1] ) ) {
			$token = $this->newOperator( $word1, $this->pos, strlen( $word1 ) );
			$this->pos += strlen( $word1 );

			return $token;
		}

		// The single-character operand symbols
		if ( strpos( self::OPERAND_SYMBOLS, $word1 ) !== false ) {
			$token = $this->newNumber( $word1, $this->pos );
			$this->pos++;

			return $token;
		}

		// Samples
		if ( $word1 === '@integer' || $word1 === '@decimal' ) {
			// Samples are like comments, they have no effect on rule evaluation.
			// They run from the first sample indicator to the end of the string.
			$this->pos = $this->end;

			return false;
		}

		$this->error( 'unrecognised word' );
	}
1226080
public function tokenize()
    {
        $this->inputStream->reset();

        $tokens = [];

        while ($token = $this->readToken()) {
            $tokens[] = $token;
        }

        return $tokens;
    }
1225745
public function getFirstToken()
    {
        if (! $this->_firstToken) {
            if (! $this->_tokens) {
                //No cache yet, but we might need all tokens later
                $this->getTokens();
            }
            $this->_firstToken = reset($this->_tokens);
        }

        return $this->_firstToken;
    }
1078896
protected final Token consumeToken() throws ScanException, ParseException {
		Token result = token;
		if (lookahead.isEmpty()) {
			token = scanner.next();
			position = scanner.getPosition();
		} else {
			LookaheadToken next = lookahead.remove(0);
			token = next.token;
			position = next.position;
		}
		return result;
	}
50279
public function nextToken()
    {
        if ($this->lookahead !== null) {
            $token = $this->lookahead;

            $this->lookahead = null;
        } else {
            $token = trim(array_shift($this->tokens));
        }


        if (strlen($token) === 1 && Character::isLetter($token{0})
            && $this->dosStyleFilesystem
            && !empty($this->tokens)
        ) {
            // we are on a dos style system so this path could be a drive

            // spec. We look at the next token

            $nextToken = trim(array_shift($this->tokens));

            if (StringHelper::startsWith('\\', $nextToken) || StringHelper::startsWith('/', $nextToken)) {
                // we know we are on a DOS style platform and the next path

                // starts with a slash or backslash, so we know this is a

                // drive spec

                $token .= ':' . $nextToken;
            } else {
                // store the token just read for next time

                $this->lookahead = $nextToken;
            }
        }

        return $token;
    }