Lexes an expression

@throws RuntimeException Thrown if there was an invalid token
513641
private function lexExpression()
    {
        $statementMethods = $this->getStatementLexingMethods();

        while (!$this->atEof()) {
            reset($statementMethods);
            $matchedStatement = false;

            // This is essentially a foreach loop that can be reset
            while (list($statementOpenDelimiter, $methodName) = each($statementMethods)) {
                if ($this->matches($statementOpenDelimiter)) {
                    // This is an unescaped statement
                    $matchedStatement = true;
                    $this->{$methodName}();

                    // Now that we've matched, we want to reset the loop so that longest delimiters are matched first
                    reset($statementMethods);
                } elseif ($this->getCurrentChar() === '\\') {
                    // Now that we know we're on an escape character, spend the resources to check for a match
                    if ($this->matches("\\$statementOpenDelimiter")) {
                        // This is an escaped statement
                        $this->expressionBuffer .= $statementOpenDelimiter;
                    }
                }
            }

            // Handle any text outside statements
            if (!$matchedStatement && !$this->atEof()) {
                $currentChar = $this->getCurrentChar();
                $this->expressionBuffer .= $currentChar;
                $this->cursor++;

                // Keep on going if we're seeing alphanumeric text
                while (ctype_alnum($this->getCurrentChar())) {
                    $currentChar = $this->getCurrentChar();
                    $this->expressionBuffer .= $currentChar;
                    $this->cursor++;
                }
            } else {
                $this->flushExpressionBuffer();
            }
        }

        // Flush anything left over in the buffer
        $this->flushExpressionBuffer();
    }
851111
@Check
	public void checkBreakKeywordUse(SarlBreakExpression expression) {
		final EObject container = Utils.getFirstContainerForPredicate(expression,
				it -> !(it instanceof XExpression) || it instanceof XAbstractWhileExpression
				|| it instanceof XBasicForLoopExpression || it instanceof XForLoopExpression);
		if (container instanceof XExpression) {
			if (!isIgnored(DISCOURAGED_LOOP_BREAKING_KEYWORD_USE)
					&& container instanceof XBasicForLoopExpression) {
				addIssue(
						MessageFormat.format(Messages.SARLValidator_17, this.grammarAccess.getBreakKeyword()),
						expression,
						null,
						ValidationMessageAcceptor.INSIGNIFICANT_INDEX,
						DISCOURAGED_LOOP_BREAKING_KEYWORD_USE);
			}
		} else {
			error(MessageFormat.format(Messages.SARLValidator_18, this.grammarAccess.getBreakKeyword()),
					expression,
					null,
					ValidationMessageAcceptor.INSIGNIFICANT_INDEX,
					INVALID_USE_OF_LOOP_BREAKING_KEYWORD);
		}
	}
335819
public final void expressionList() throws RecognitionException {
		int expressionList_StartIndex = input.index();

		try {
			if ( state.backtracking>0 && alreadyParsedRule(input, 105) ) { return; }

			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1107:5: ( expression ( ',' expression )* )
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1107:9: expression ( ',' expression )*
			{
			pushFollow(FOLLOW_expression_in_expressionList4817);
			expression();
			state._fsp--;
			if (state.failed) return;
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1107:20: ( ',' expression )*
			loop139:
			while (true) {
				int alt139=2;
				int LA139_0 = input.LA(1);
				if ( (LA139_0==43) ) {
					alt139=1;
				}

				switch (alt139) {
				case 1 :
					// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1107:21: ',' expression
					{
					match(input,43,FOLLOW_43_in_expressionList4820); if (state.failed) return;
					pushFollow(FOLLOW_expression_in_expressionList4822);
					expression();
					state._fsp--;
					if (state.failed) return;
					}
					break;

				default :
					break loop139;
				}
			}

			}

		}
		catch (RecognitionException re) {
			reportError(re);
			recover(input,re);
		}
		finally {
			// do for sure before leaving
			if ( state.backtracking>0 ) { memoize(input, 105, expressionList_StartIndex); }

		}
	}
643722
public final void entryRuleXBasicForLoopExpression() throws RecognitionException {
        try {
            // InternalXbase.g:984:1: ( ruleXBasicForLoopExpression EOF )
            // InternalXbase.g:985:1: ruleXBasicForLoopExpression EOF
            {
            if ( state.backtracking==0 ) {
               before(grammarAccess.getXBasicForLoopExpressionRule()); 
            }
            pushFollow(FOLLOW_1);
            ruleXBasicForLoopExpression();

            state._fsp--;
            if (state.failed) return ;
            if ( state.backtracking==0 ) {
               after(grammarAccess.getXBasicForLoopExpressionRule()); 
            }
            match(input,EOF,FOLLOW_2); if (state.failed) return ;

            }

        }
        catch (RecognitionException re) {
            reportError(re);
            recover(input,re);
        }
        finally {
        }
        return ;
    }
230741
public final void constantExpression() throws RecognitionException {
		int constantExpression_StartIndex = input.index();

		try {
			if ( state.backtracking>0 && alreadyParsedRule(input, 107) ) { return; }

			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1115:5: ( expression )
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1115:7: expression
			{
			pushFollow(FOLLOW_expression_in_constantExpression4858);
			expression();
			state._fsp--;
			if (state.failed) return;
			}

		}
		catch (RecognitionException re) {
			reportError(re);
			recover(input,re);
		}
		finally {
			// do for sure before leaving
			if ( state.backtracking>0 ) { memoize(input, 107, constantExpression_StartIndex); }

		}
	}
75972
private void evaluateExpressions() throws JspException {
        /*
         * Note: we don't check for type mismatches here; we assume
         * the expression evaluator will return the expected type
         * (by virtue of knowledge we give it about what that type is).
         * A ClassCastException here is truly unexpected, so we let it
         * propagate up.
         */

        if (begin_ != null) {
            Object r = ExpressionEvaluatorManager.evaluate(
                    "begin", begin_, Integer.class, this, pageContext);
            if (r == null) {
                throw new NullAttributeException("forTokens", "begin");
            }
            begin = ((Integer) r).intValue();
            validateBegin();
        }

        if (end_ != null) {
            Object r = ExpressionEvaluatorManager.evaluate(
                    "end", end_, Integer.class, this, pageContext);
            if (r == null) {
                throw new NullAttributeException("forTokens", "end");
            }
            end = ((Integer) r).intValue();
            validateEnd();
        }

        if (step_ != null) {
            Object r = ExpressionEvaluatorManager.evaluate(
                    "step", step_, Integer.class, this, pageContext);
            if (r == null) {
                throw new NullAttributeException("forTokens", "step");
            }
            step = ((Integer) r).intValue();
            validateStep();
        }

        if (items_ != null) {
            items = (String) ExpressionEvaluatorManager.evaluate(
                    "items", items_, String.class, this, pageContext);
            // use the empty string to indicate "no iteration"
            if (items == null) {
                items = "";
            }
        }

        if (delims_ != null) {
            delims = (String) ExpressionEvaluatorManager.evaluate(
                    "delims", delims_, String.class, this, pageContext);
            // use the empty string to cause monolithic tokenization
            if (delims == null) {
                delims = "";
            }
        }
    }
407375
private void evaluateExpressions() throws JspException {
        /* 
         * Note: we don't check for type mismatches here; we assume
         * the expression evaluator will return the expected type
         * (by virtue of knowledge we give it about what that type is).
         * A ClassCastException here is truly unexpected, so we let it
         * propagate up.
         */

        value = (String) ExpressionUtil.evalNotNull(
                "url", "value", value_, String.class, this, pageContext);
        context = (String) ExpressionUtil.evalNotNull(
                "url", "context", context_, String.class, this, pageContext);
    }
1031116
public String parse(String expression) throws InvalidExpressionException {

        this.expression = expression;

        try {
            var = check();
        } catch (InvalidExpressionException e) {
            e.printStackTrace();
        }

        this.expression = formatString(expression);
        System.out.println(this.expression);
        tokens = tokenize(this.expression);

        return this.expression;
    }
181223
private function lexCommentStatement()
    {
        $this->lexDelimitedExpressionStatement(
            TokenTypes::T_COMMENT_OPEN,
            $this->commentDelimiters[0],
            TokenTypes::T_COMMENT_CLOSE,
            $this->commentDelimiters[1],
            false
        );
    }
1303761
public final BaseDescr conditionalAndExpression() throws RecognitionException {
		BaseDescr result = null;


		BaseDescr left =null;
		AnnotationDescr args =null;
		BaseDescr right =null;

		try {
			// src/main/resources/org/drools/compiler/lang/DRL5Expressions.g:231:3: (left= inclusiveOrExpression ( DOUBLE_AMPER (args= fullAnnotation[null] )? right= inclusiveOrExpression )* )
			// src/main/resources/org/drools/compiler/lang/DRL5Expressions.g:231:5: left= inclusiveOrExpression ( DOUBLE_AMPER (args= fullAnnotation[null] )? right= inclusiveOrExpression )*
			{
			pushFollow(FOLLOW_inclusiveOrExpression_in_conditionalAndExpression1182);
			left=inclusiveOrExpression();
			state._fsp--;
			if (state.failed) return result;
			if ( state.backtracking==0 ) { if( buildDescr  ) { result = left; } }
			// src/main/resources/org/drools/compiler/lang/DRL5Expressions.g:232:3: ( DOUBLE_AMPER (args= fullAnnotation[null] )? right= inclusiveOrExpression )*
			loop27:
			while (true) {
				int alt27=2;
				int LA27_0 = input.LA(1);
				if ( (LA27_0==DOUBLE_AMPER) ) {
					alt27=1;
				}

				switch (alt27) {
				case 1 :
					// src/main/resources/org/drools/compiler/lang/DRL5Expressions.g:232:5: DOUBLE_AMPER (args= fullAnnotation[null] )? right= inclusiveOrExpression
					{
					match(input,DOUBLE_AMPER,FOLLOW_DOUBLE_AMPER_in_conditionalAndExpression1190); if (state.failed) return result;
					if ( state.backtracking==0 ) { if ( isNotEOF() ) helper.emit( Location.LOCATION_LHS_INSIDE_CONDITION_OPERATOR ); }
					// src/main/resources/org/drools/compiler/lang/DRL5Expressions.g:234:13: (args= fullAnnotation[null] )?
					int alt26=2;
					int LA26_0 = input.LA(1);
					if ( (LA26_0==AT) ) {
						alt26=1;
					}
					switch (alt26) {
						case 1 :
							// src/main/resources/org/drools/compiler/lang/DRL5Expressions.g:234:13: args= fullAnnotation[null]
							{
							pushFollow(FOLLOW_fullAnnotation_in_conditionalAndExpression1213);
							args=fullAnnotation(null);
							state._fsp--;
							if (state.failed) return result;
							}
							break;

					}

					pushFollow(FOLLOW_inclusiveOrExpression_in_conditionalAndExpression1219);
					right=inclusiveOrExpression();
					state._fsp--;
					if (state.failed) return result;
					if ( state.backtracking==0 ) { if( buildDescr  ) {
					               ConstraintConnectiveDescr descr = ConstraintConnectiveDescr.newAnd();
					               descr.addOrMerge( result );
					               descr.addOrMerge( right );
					               if ( args != null ) { descr.addAnnotation( args ); }
					               result = descr;
					           }
					         }
					}
					break;

				default :
					break loop27;
				}
			}

			}

		}

		catch (RecognitionException re) {
		    throw re;
		}

		finally {
			// do for sure before leaving
		}
		return result;
	}
145180
private void evaluateExpressions() throws JspException {
        Integer tempInt = null;

        if (dataSourceEL != null) {
            rawDataSource = (Object) ExpressionEvaluatorManager.evaluate(
                    "dataSource", dataSourceEL, Object.class, this, pageContext);
        }

        if (sqlEL != null) {
            sql = (String) ExpressionEvaluatorManager.evaluate("sql", sqlEL,
                    String.class, this, pageContext);
        }

        if (startRowEL != null) {
            tempInt = (Integer) ExpressionEvaluatorManager.evaluate(
                    "startRow", startRowEL, Integer.class, this, pageContext);
            if (tempInt != null) {
                startRow = tempInt.intValue();
            }
        }

        if (maxRowsEL != null) {
            tempInt = (Integer) ExpressionEvaluatorManager.evaluate(
                    "maxRows", maxRowsEL, Integer.class, this, pageContext);
            if (tempInt != null) {
                maxRows = tempInt.intValue();
            }
        }
    }
365618
private void evaluateExpressions() throws JspException {
        /* 
         * Note: we don't check for type mismatches here; we assume
         * the expression evaluator will return the expected type
         * (by virtue of knowledge we give it about what that type is).
         * A ClassCastException here is truly unexpected, so we let it
         * propagate up.
         */

        name = (String) ExpressionUtil.evalNotNull(
                "import", "name", name_, String.class, this, pageContext);
        value = (String) ExpressionUtil.evalNotNull(
                "import", "value", value_, String.class, this, pageContext);
    }
1018471
static function skipCurly(array $tokens, &$i) {
    $iStart = $i;
    $level = 0;
    for (++$i;; ++$i) {
      if ('}' === $token = $tokens[$i]) {
        if (--$level < 0) {
          ++$i;
          return NULL;
        }
      }
      elseif ('{' === $token || T_CURLY_OPEN === $token[0] || T_DOLLAR_OPEN_CURLY_BRACES === $token[0]) {
        ++$level;
      }
      elseif ('#' === $token) {
        $i = $iStart;
        return FALSE;
      }
    }

    throw new \RuntimeException('Unreachable code.');
  }
538500
public function lex()
    {
        // TODO: Sometimes, static::parse* functions make unnecessary calls to
        // is* functions. For a better performance, some rules can be deduced
        // from context.
        // For example, in `parseBool` there is no need to compare the token
        // every time with `true` and `false`. The first step would be to
        // compare with 'true' only and just after that add another letter from
        // context and compare again with `false`.
        // Another example is `parseComment`.

        $list = new TokensList();

        /**
         * Last processed token.
         *
         * @var Token
         */
        $lastToken = null;

        for ($this->last = 0, $lastIdx = 0; $this->last < $this->len; $lastIdx = ++$this->last) {
            /**
             * The new token.
             *
             * @var Token
             */
            $token = null;

            foreach (static::$PARSER_METHODS as $method) {
                if ($token = $this->$method()) {
                    break;
                }
            }

            if ($token === null) {
                // @assert($this->last === $lastIdx);
                $token = new Token($this->str[$this->last]);
                $this->error(
                    'Unexpected character.',
                    $this->str[$this->last],
                    $this->last
                );
            } elseif ($lastToken !== null
                && $token->type === Token::TYPE_SYMBOL
                && $token->flags & Token::FLAG_SYMBOL_VARIABLE
                && (
                    $lastToken->type === Token::TYPE_STRING
                    || (
                        $lastToken->type === Token::TYPE_SYMBOL
                        && $lastToken->flags & Token::FLAG_SYMBOL_BACKTICK
                    )
                )
            ) {
                // Handles ```... FROM 'user'@'%' ...```.
                $lastToken->token .= $token->token;
                $lastToken->type = Token::TYPE_SYMBOL;
                $lastToken->flags = Token::FLAG_SYMBOL_USER;
                $lastToken->value .= '@' . $token->value;
                continue;
            } elseif ($lastToken !== null
                && $token->type === Token::TYPE_KEYWORD
                && $lastToken->type === Token::TYPE_OPERATOR
                && $lastToken->value === '.'
            ) {
                // Handles ```... tbl.FROM ...```. In this case, FROM is not
                // a reserved word.
                $token->type = Token::TYPE_NONE;
                $token->flags = 0;
                $token->value = $token->token;
            }

            $token->position = $lastIdx;

            $list->tokens[$list->count++] = $token;

            // Handling delimiters.
            if ($token->type === Token::TYPE_NONE && $token->value === 'DELIMITER') {
                if ($this->last + 1 >= $this->len) {
                    $this->error(
                        'Expected whitespace(s) before delimiter.',
                        '',
                        $this->last + 1
                    );
                    continue;
                }

                // Skipping last R (from `delimiteR`) and whitespaces between
                // the keyword `DELIMITER` and the actual delimiter.
                $pos = ++$this->last;
                if (($token = $this->parseWhitespace()) !== null) {
                    $token->position = $pos;
                    $list->tokens[$list->count++] = $token;
                }

                // Preparing the token that holds the new delimiter.
                if ($this->last + 1 >= $this->len) {
                    $this->error(
                        'Expected delimiter.',
                        '',
                        $this->last + 1
                    );
                    continue;
                }
                $pos = $this->last + 1;

                // Parsing the delimiter.
                $this->delimiter = null;
                $delimiterLen = 0;
                while (++$this->last < $this->len && ! Context::isWhitespace($this->str[$this->last]) && $delimiterLen < 15) {
                    $this->delimiter .= $this->str[$this->last];
                    ++$delimiterLen;
                }

                if (empty($this->delimiter)) {
                    $this->error(
                        'Expected delimiter.',
                        '',
                        $this->last
                    );
                    $this->delimiter = ';';
                }

                --$this->last;

                // Saving the delimiter and its token.
                $this->delimiterLen = strlen($this->delimiter);
                $token = new Token($this->delimiter, Token::TYPE_DELIMITER);
                $token->position = $pos;
                $list->tokens[$list->count++] = $token;
            }

            $lastToken = $token;
        }

        // Adding a final delimiter to mark the ending.
        $list->tokens[$list->count++] = new Token(null, Token::TYPE_DELIMITER);

        // Saving the tokens list.
        $this->list = $list;
    }
1235605
private function parseTypes(\ArrayIterator $tokens, Context $context, $parserContext)
    {
        $types = [];
        $token = '';
        while ($tokens->valid()) {
            $token = $tokens->current();

            if ($token === '|') {
                if (count($types) === 0) {
                    throw new \RuntimeException(
                        'A type is missing before a type separator'
                    );
                }

                if ($parserContext !== self::PARSER_IN_COMPOUND
                    && $parserContext !== self::PARSER_IN_ARRAY_EXPRESSION
                    && $parserContext !== self::PARSER_IN_COLLECTION_EXPRESSION
                ) {
                    throw new \RuntimeException(
                        'Unexpected type separator'
                    );
                }

                $tokens->next();
            } elseif ($token === '?') {
                if ($parserContext !== self::PARSER_IN_COMPOUND
                    && $parserContext !== self::PARSER_IN_ARRAY_EXPRESSION
                    && $parserContext !== self::PARSER_IN_COLLECTION_EXPRESSION
                ) {
                    throw new \RuntimeException(
                        'Unexpected nullable character'
                    );
                }

                $tokens->next();
                $type = $this->parseTypes($tokens, $context, self::PARSER_IN_NULLABLE);
                $types[] = new Nullable($type);
            } elseif ($token === '(') {
                $tokens->next();
                $type = $this->parseTypes($tokens, $context, self::PARSER_IN_ARRAY_EXPRESSION);

                $resolvedType = new Array_($type);

                $token = $tokens->current();
                // Someone did not properly close their array expression ..
                if ($token === null) {
                    break;
                }

                // we generate arrays corresponding to the number of '[]' after the ')'
                $numberOfArrays = (strlen($token) - 1) / 2;
                for ($i = 0; $i < $numberOfArrays - 1; ++$i) {
                    $resolvedType = new Array_($resolvedType);
                }

                $types[] = $resolvedType;
                $tokens->next();
            } elseif ($parserContext === self::PARSER_IN_ARRAY_EXPRESSION && $token[0] === ')') {
                break;
            } elseif ($token === '<') {
                if (count($types) === 0) {
                    throw new \RuntimeException(
                        'Unexpected collection operator "<", class name is missing'
                    );
                }

                $classType = array_pop($types);
                if ($classType !== null) {
                    $types[] = $this->resolveCollection($tokens, $classType, $context);
                }

                $tokens->next();
            } elseif ($parserContext === self::PARSER_IN_COLLECTION_EXPRESSION
                && ($token === '>' || trim($token) === ',')
                ) {
                break;
            } else {
                $type = $this->resolveSingleType($token, $context);
                $tokens->next();
                if ($parserContext === self::PARSER_IN_NULLABLE) {
                    return $type;
                }

                $types[] = $type;
            }
        }

        if ($token === '|') {
            throw new \RuntimeException(
                'A type is missing after a type separator'
            );
        }

        if (count($types) === 0) {
            if ($parserContext === self::PARSER_IN_NULLABLE) {
                throw new \RuntimeException(
                    'A type is missing after a nullable character'
                );
            }

            if ($parserContext === self::PARSER_IN_ARRAY_EXPRESSION) {
                throw new \RuntimeException(
                    'A type is missing in an array expression'
                );
            }

            if ($parserContext === self::PARSER_IN_COLLECTION_EXPRESSION) {
                throw new \RuntimeException(
                    'A type is missing in a collection expression'
                );
            }
        } elseif (count($types) === 1) {
            return $types[0];
        }

        return new Compound($types);
    }
866636
public final void ruleXBasicForLoopExpression() throws RecognitionException {

        		int stackSize = keepStackSize();
        	
        try {
            // InternalXbase.g:996:2: ( ( ( rule__XBasicForLoopExpression__Group__0 ) ) )
            // InternalXbase.g:997:2: ( ( rule__XBasicForLoopExpression__Group__0 ) )
            {
            // InternalXbase.g:997:2: ( ( rule__XBasicForLoopExpression__Group__0 ) )
            // InternalXbase.g:998:3: ( rule__XBasicForLoopExpression__Group__0 )
            {
            if ( state.backtracking==0 ) {
               before(grammarAccess.getXBasicForLoopExpressionAccess().getGroup()); 
            }
            // InternalXbase.g:999:3: ( rule__XBasicForLoopExpression__Group__0 )
            // InternalXbase.g:999:4: rule__XBasicForLoopExpression__Group__0
            {
            pushFollow(FOLLOW_2);
            rule__XBasicForLoopExpression__Group__0();

            state._fsp--;
            if (state.failed) return ;

            }

            if ( state.backtracking==0 ) {
               after(grammarAccess.getXBasicForLoopExpressionAccess().getGroup()); 
            }

            }


            }

        }
        catch (RecognitionException re) {
            reportError(re);
            recover(input,re);
        }
        finally {

            	restoreStackSize(stackSize);

        }
        return ;
    }
792135
protected function saveRegex()
	{
		echo $this->a . $this->b;

		while (($this->a = $this->getChar()) !== false)
		{
			if ($this->a === '/')
			{
				break;
			}

			if ($this->a === '\\')
			{
				echo $this->a;
				$this->a = $this->getChar();
			}

			if ($this->a === "\n")
			{
				throw new \RuntimeException('Unclosed regex pattern at position: ' . $this->index);
			}

			echo $this->a;
		}

		$this->b = $this->getReal();
	}