return the next token from the text
354673
Token next() throws ScanException {
        text = null;
        switch (c) {
            case '\n':
                return EOL;
            case ' ':
            case '\t':
                return whiteSpace();
            case ',':
                nextChar();
                return COMMA;
            case '/':
                nextChar();
                return SLASH;
            case '<':
                nextChar();
                return LESS;
            case '>':
                nextChar();
                return MORE;
            case '=':
                nextChar();
                return EQUALS;
            case '!':
                return pipe();
            case '"':
                return quotedText('"');
            case '\'':
                return quotedText('\'');
            default:
                return name();
        }

    }
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
444686
def get_tokens(line: str) -> Iterator[str]:
    
    for token in line.rstrip().split():
        if len(token) > 0:
            yield token
905105
def parse(self, text):
        
        tokens = self.lex(text)
        parser = Parser(tokens)
        return parser.parse()
1093217
function getTextBetween(node1, node2) {
            const allTokens = [node1].concat(sourceCode.getTokensBetween(node1, node2)).concat(node2);
            const sourceText = sourceCode.getText();

            return allTokens.slice(0, -1).reduce((accumulator, token, index) => accumulator + sourceText.slice(token.range[1], allTokens[index + 1].range[0]), "");
        }
289351
public Token next() throws IOException {

        // skip whitespace
        while (Character.isWhitespace((char)lastChar))
        {
            readChar();
        }

        // handle punctuation
        switch (lastChar)
        {
            case -1:  readChar(); return Token.TokenEOF;
            case '{': readChar(); return Token.TokenBraceL;
            case '}': readChar(); return Token.TokenBraceR;
            case '[': readChar(); return Token.TokenBrackL;
            case ']': readChar(); return Token.TokenBrackR;
            case ':': readChar(); return Token.TokenColon;
            case ',': readChar(); return Token.TokenComma;

            case '"':
            case '\'':
                String stringValue = readString();
                return new Token(stringValue);

            case '-':
            case '.':
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                Number numberValue = readNumber();
                return new Token(numberValue);

            case 'n': 
            case 't':
            case 'f':
                String ident = readIdentifier();

                if (ident.equals("null"))  return Token.TokenNull;
                if (ident.equals("true"))  return Token.TokenTrue;
                if (ident.equals("false")) return Token.TokenFalse;

                throw new IOException("Unexpected identifier '" + ident + "' " + onLineCol());

            default:
                throw new IOException("Unexpected character '" + (char)lastChar + "' " + onLineCol());

        }

    }
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
762941
def parse_string(self):
        """"""
        aliased_value = LITERAL_ALIASES.get(self.current_token.value.lower())
        if aliased_value is not None:
            return aliased_value
        return String(self.current_token.value)
771013
public final void nextToken() {
        skipIgnoredToken();
        if (isVariableBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
        } else if (isNCharBegin()) {
            currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
        } else if (isIdentifierBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
        } else if (isHexDecimalBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
        } else if (isNumberBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
        } else if (isSymbolBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
        } else if (isCharsBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanChars();
        } else if (isEnd()) {
            currentToken = new Token(Assist.END, "", offset);
        } else {
            throw new SQLParsingException(this, Assist.ERROR);
        }
        offset = currentToken.getEndPosition();
    }
108424
@Override
  public String getNext() {
    try {
      int nextTok = st.nextToken();
      switch (nextTok) {
        case java.io.StreamTokenizer.TT_EOL:
          return eolString;
        case java.io.StreamTokenizer.TT_EOF:
          return null;
        case java.io.StreamTokenizer.TT_WORD:
          return st.sval;
        case java.io.StreamTokenizer.TT_NUMBER:
          return Double.toString(st.nval);
        default:
          char[] t = { (char) nextTok };    // (array initialization)
          return new String(t);
      }
    } catch (IOException ioe) {
      // do nothing, return null
      return null;
    }
  }
405511
@SuppressWarnings("unchecked")
  @Override
  protected T getNext() {
    T token = null;
    if (lexer == null) {
      return token;
    }
    try {
      token = (T) lexer.next();
      while (token != null && token.word().equals(WhitespaceLexer.NEWLINE)) {
        if (eolIsSignificant) {
          return token;
        } else {
          token = (T) lexer.next();
        }
      }
    } catch (IOException e) {
      // do nothing, return null
    }
    return token;
  }
1163510
@Nullable
    public Token next() throws IOException {
        if (unreadToken != null) {
            lastToken = unreadToken;
            unreadToken = null;
            return lastToken;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == Token.kNewLine ||
                lastChar == Token.kCarriageReturn ||
                lastChar == Token.kSpace ||
                lastChar == Token.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == Token.kShellComment) {
                getRestOfLine();
                continue;
            }

            if (lastChar == Token.kLiteralDoubleQuote || lastChar == Token.kLiteralQuote) {
                lastToken = nextStringLiteral((char) lastChar);
                return lastToken;
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                lastToken = nextNumber();
                return lastToken;
            } else if ('_' == lastChar ||
                       ('a' <= lastChar && lastChar <= 'z') ||
                       ('A' <= lastChar && lastChar <= 'Z')) {
                lastToken = nextToken();
                return lastToken;
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // non-ASCII UTF-8 characters are only allowed inside JSON string literals.
                throw failure(lineNo, linePos, 1, "Unknown token initiator '%s'", Strings.escape((char) lastChar));
            } else {
                lastToken = nextSymbol();
                return lastToken;
            }
        }

        return null;
    }
475351
function nextToken(tokens, idx) {
  if (++idx >= tokens.length - 2) {
    return idx;
  }
  if ((tokens[idx].type === 'paragraph_open' && tokens[idx].tight) &&
      (tokens[idx + 1].type === 'inline' && tokens[idx + 1].content.length === 0) &&
      (tokens[idx + 2].type === 'paragraph_close' && tokens[idx + 2].tight)) {
    return nextToken(tokens, idx + 2);
  }
  return idx;
}
952130
function(regex, text){
    var nextIndex = 0;
    var result = [];

    var execResult;
    while ((execResult = regex.exec(text))!== null) {
        result.push({
            start: execResult.index, 
            matchLength: execResult[0].length});
    }

    return {
        next: function(){
            return nextIndex < result.length ?
                result[nextIndex++] :
                null;
        }
    };
}
1198803
def get_next_character(self):
        

        cursor = self.textCursor()
        cursor.movePosition(QTextCursor.NextCharacter, QTextCursor.KeepAnchor)
        return cursor.selectedText()
1161883
def NextToken(self):
    
    # If we don't have enough data - feed ourselves: We assume
    # that we must have at least one sector in our buffer.
    if len(self.buffer) < 512:
      if self.Feed() == 0 and not self.buffer:
        return None

    return Lexer.NextToken(self)
96962
function isLastTokenInCurrentLine(token) {
            const tokenAfter = sourceCode.getTokenAfter(token);

            return !(tokenAfter && astUtils.isTokenOnSameLine(token, tokenAfter));
        }