A {@link Parser} that recognizes a sequence of tokens identified by {@code tokenNames}, as an
atomic step.
764549
public Parser<?> phrase(String... tokenNames) {
    Parser<?>[] wordParsers = new Parser<?>[tokenNames.length];
    for (int i = 0; i < tokenNames.length; i++) {
      wordParsers[i] = token(tokenNames[i]);
    }
    String phrase = Strings.join(" ", tokenNames);
    return Parsers.sequence(wordParsers).atomic().retn(phrase).label(phrase);
  }
889058
public final Parser<T> followedBy(Parser<?> parser) {
    return Parsers.sequence(this, parser, InternalFunctors.<T, Object>firstOfTwo());
  }
857939
public static TokenRegex compile(String pattern) throws ParseException {
      ExpressionIterator p = QueryToPredicate.PARSER.parse(pattern);
      Expression exp;
      TransitionFunction top = null;
      while ((exp = p.next()) != null) {
         if (top == null) {
            top = consumerize(exp);
         } else {
            top = new TransitionFunction.Sequence(top, consumerize(exp));
         }
      }
      return new TokenRegex(top);
   }
1151794
public final <R> Parser<R> next(Parser<R> parser) {
    return Parsers.sequence(this, parser);
  }
771013
public final void nextToken() {
        skipIgnoredToken();
        if (isVariableBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
        } else if (isNCharBegin()) {
            currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
        } else if (isIdentifierBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
        } else if (isHexDecimalBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
        } else if (isNumberBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
        } else if (isSymbolBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
        } else if (isCharsBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanChars();
        } else if (isEnd()) {
            currentToken = new Token(Assist.END, "", offset);
        } else {
            throw new SQLParsingException(this, Assist.ERROR);
        }
        offset = currentToken.getEndPosition();
    }
626611
def token_name(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == expected:
        return TokenMatch(None, token.value, (token,))
1255203
private static String nextToken(String[] tokens, int x, String context)
            throws ParseException {
        if (x < tokens.length) {
            return tokens[x];
        }
        throw new ParseException("missing token at end of '" + context + "'", -1);
    }
60318
private void parseLiteral(String token) {
        Matcher m = Pattern.compile(String.format(ESCAPED_LITERAL_PATTERN_TEMPLATE, escapeCharacter)).matcher(token);
        while (m.find()) {
            cleanedToken = trimSpecialCharacters(m.group(1));
            if (tokenHasContent()) {
                mergedTokens.add(new LiteralToken(cleanedToken));
            }
        }
    }
974301
public void parseReferences(ArrayList<String> name)
  {
    String foreignTable = parseIdentifier();

    Token token = scanToken();

    ArrayList<String> foreignColumns = new ArrayList<String>();

    if (token == Token.LPAREN) {
      _token = token;

      foreignColumns = parseColumnNames();
    }
    else {
      _token = token;
    }
  }
1229171
private static List<Tokenizer.Token> toReversePolishNotation(List<Tokenizer.Token> tokens) {
        LinkedList<Tokenizer.Token> queue = new LinkedList<Tokenizer.Token>();
        Stack<Tokenizer.Token> stack = new Stack<Tokenizer.Token>();

        for (int i = 0; i < tokens.size(); i++) {
            Tokenizer.Token token = tokens.get(i);
            switch (token.type) {
                case VERSION:
                    queue.add(token);
                    break;
                case CLOSING:
                    while (stack.peek().type != Tokenizer.TokenType.OPENING) {
                        queue.add(stack.pop());
                    }
                    stack.pop();
                    if (stack.size() > 0 && stack.peek().type.isUnary()) {
                        queue.add(stack.pop());
                    }
                    break;
                default:
                    if (token.type.isUnary()) {
                        queue.push(token);
                        i++;
                        queue.push(tokens.get(i));
                    } else {
                        stack.push(token);
                    }
                    break;
            }
        }

        while (!stack.isEmpty()) {
            queue.add(stack.pop());
        }

        Collections.reverse(queue);
        return queue;
    }
297835
public Parse parse(Parse tokens) {
    Parse p = parse(tokens,1)[0];
    setParents(p);
    return p;
  }
1199757
function tokenBase(stream, state) {
    var ch = stream.next(), 
        mightBeFunction = false,
        isEQName = isEQNameAhead(stream);
    
    // an XML tag (if not in some sub, chained tokenizer)
    if (ch == "<") {
      if(stream.match("!--", true))
        return chain(stream, state, tokenXMLComment);
        
      if(stream.match("![CDATA", false)) {
        state.tokenize = tokenCDATA;
        return ret("tag", "tag");
      }
      
      if(stream.match("?", false)) {
        return chain(stream, state, tokenPreProcessing);
      }
      
      var isclose = stream.eat("/");
      stream.eatSpace();
      var tagName = "", c;
      while ((c = stream.eat(/[^\s\u00a0=<>\"\'\/?]/))) tagName += c;
      
      return chain(stream, state, tokenTag(tagName, isclose));
    }
    // start code block
    else if(ch == "{") {
      pushStateStack(state,{ type: "codeblock"});
      return ret("", "");
    }
    // end code block
    else if(ch == "}") {
      popStateStack(state);
      return ret("", "");
    }
    // if we're in an XML block
    else if(isInXmlBlock(state)) {
      if(ch == ">")
        return ret("tag", "tag");
      else if(ch == "/" && stream.eat(">")) {
        popStateStack(state);
        return ret("tag", "tag");
      }
      else  
        return ret("word", "variable");
    }
    // if a number
    else if (/\d/.test(ch)) {
      stream.match(/^\d*(?:\.\d*)?(?:E[+\-]?\d+)?/);
      return ret("number", "atom");
    }
    // comment start
    else if (ch === "(" && stream.eat(":")) {
      pushStateStack(state, { type: "comment"});
      return chain(stream, state, tokenComment);
    }
    // quoted string
    else if (  !isEQName && (ch === '"' || ch === "'"))
      return chain(stream, state, tokenString(ch));
    // variable
    else if(ch === "$") {
      return chain(stream, state, tokenVariable);
    }
    // assignment
    else if(ch ===":" && stream.eat("=")) {
      return ret("operator", "keyword");
    }
    // open paren
    else if(ch === "(") {
      pushStateStack(state, { type: "paren"});
      return ret("", "");
    }
    // close paren
    else if(ch === ")") {
      popStateStack(state);
      return ret("", "");
    }
    // open paren
    else if(ch === "[") {
      pushStateStack(state, { type: "bracket"});
      return ret("", "");
    }
    // close paren
    else if(ch === "]") {
      popStateStack(state);
      return ret("", "");
    }
    else {
      var known = keywords.propertyIsEnumerable(ch) && keywords[ch];

      // if there's a EQName ahead, consume the rest of the string portion, it's likely a function
      if(isEQName && ch === '\"') while(stream.next() !== '"'){}
      if(isEQName && ch === '\'') while(stream.next() !== '\''){}
      
      // gobble up a word if the character is not known
      if(!known) stream.eatWhile(/[\w\$_-]/);
      
      // gobble a colon in the case that is a lib func type call fn:doc
      var foundColon = stream.eat(":");
      
      // if there's not a second colon, gobble another word. Otherwise, it's probably an axis specifier
      // which should get matched as a keyword
      if(!stream.eat(":") && foundColon) {
        stream.eatWhile(/[\w\$_-]/);
      }
      // if the next non whitespace character is an open paren, this is probably a function (if not a keyword of other sort)
      if(stream.match(/^[ \t]*\(/, false)) {
        mightBeFunction = true;
      }
      // is the word a keyword?
      var word = stream.current();
      known = keywords.propertyIsEnumerable(word) && keywords[word];
      
      // if we think it's a function call but not yet known, 
      // set style to variable for now for lack of something better
      if(mightBeFunction && !known) known = {type: "function_call", style: "variable def"};
      
      // if the previous word was element, attribute, axis specifier, this word should be the name of that
      if(isInXmlConstructor(state)) {
        popStateStack(state);
        return ret("word", "variable", word);
      }
      // as previously checked, if the word is element,attribute, axis specifier, call it an "xmlconstructor" and 
      // push the stack so we know to look for it on the next word
      if(word == "element" || word == "attribute" || known.type == "axis_specifier") pushStateStack(state, {type: "xmlconstructor"});
      
      // if the word is known, return the details of that else just call this a generic 'word'
      return known ? ret(known.type, known.style, word) :
                     ret("word", "variable", word);
    }
  }
579234
Token match( TokenStream input,
                 int ttype,
                 String text,
                 int[] follow,
                 DroolsEditorType etype ) throws RecognitionException {
        Token matchedSymbol = null;
        matchedSymbol = input.LT(1);
        if (input.LA(1) == ttype && (text == null || text.equals(matchedSymbol.getText()))) {
            input.consume();
            state.errorRecovery = false;
            state.failed = false;
            helper.emit(matchedSymbol,
                    etype);
            return matchedSymbol;
        }
        if (state.backtracking > 0) {
            state.failed = true;
            return matchedSymbol;
        }
        matchedSymbol = recoverFromMismatchedToken(input,
                ttype,
                text,
                follow);
        helper.emit(matchedSymbol,
                etype);
        return matchedSymbol;
    }
108031
public static <T> Parser<T> sequence(Parser<?> p1, Parser<T> p2) {
    return sequence(p1, p2, InternalFunctors.<Object, T>lastOfTwo());
  }
424255
function chain(stream, state, parser) {
        state.tokenize = parser;
        return parser(stream, state);
    }
388221
protected List<String> parseNameList( DdlTokenStream tokens ) throws ParsingException {
        List<String> names = new LinkedList<String>();

        while (true) {
            names.add(parseName(tokens));

            if (!tokens.canConsume(COMMA)) {
                break;
            }
        }

        return names;
    }
691542
protected void parseSequencesWithColons(TokenList tokens , Sequence sequence ) {

        TokenList.Token t = tokens.getFirst();
        if( t == null )
            return;

        int state = 0;

        TokenList.Token start = null;
        TokenList.Token middle = null;
        TokenList.Token prev = t;

        boolean last = false;
        while( true ) {
            if( state == 0 ) {
                if( isVariableInteger(t) && (t.next != null && t.next.getSymbol() == Symbol.COLON) ) {
                    start = t;
                    state = 1;
                    t = t.next;
                } else if( t != null && t.getSymbol() == Symbol.COLON ) {
                    // If it starts with a colon then it must be 'all'  or a type-o
                    IntegerSequence range = new IntegerSequence.Range(null,null);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(range);
                    TokenList.Token n = new TokenList.Token(varSequence);
                    tokens.insert(t.previous, n);
                    tokens.remove(t);
                    t = n;
                }
            } else if( state == 1 ) {
                // var : ?
                if (isVariableInteger(t)) {
                    state = 2;
                } else {
                    // array range
                    IntegerSequence range = new IntegerSequence.Range(start,null);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(range);
                    replaceSequence(tokens, varSequence, start, prev);
                    state = 0;
                }
            } else if ( state == 2 ) {
                // var:var ?
                if( t != null && t.getSymbol() == Symbol.COLON ) {
                    middle = prev;
                    state = 3;
                } else {
                    // create for sequence with start and stop elements only
                    IntegerSequence numbers = new IntegerSequence.For(start,null,prev);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(numbers);
                    replaceSequence(tokens, varSequence, start, prev );
                    if( t != null )
                        t = t.previous;
                    state = 0;
                }
            } else if ( state == 3 ) {
                // var:var: ?
                if( isVariableInteger(t) ) {
                    // create 'for' sequence with three variables
                    IntegerSequence numbers = new IntegerSequence.For(start,middle,t);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(numbers);
                    t = replaceSequence(tokens, varSequence, start, t);
                } else {
                    // array range with 2 elements
                    IntegerSequence numbers = new IntegerSequence.Range(start,middle);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(numbers);
                    replaceSequence(tokens, varSequence, start, prev);
                }
                state = 0;
            }

            if( last ) {
                break;
            } else if( t.next == null ) {
                // handle the case where it is the last token in the sequence
                last = true;
            }
            prev = t;
            t = t.next;
        }
    }