@param string $string
@param array  $tokens

@return string
169231
protected function replaceTokens($string, array $tokens)
    {
        foreach ($tokens as $key => $value) {
            $string = str_replace('{{ '.$key.' }}', $value, $string);
        }

        return $string;
    }
1072698
protected function getNamespace($key, array $tokens)
    {
        $namespace = null;
        $tokenCount = count($tokens);
        for ($i = $key; $i < $tokenCount; $i++) {
            if ($this->isPartOfNamespace($tokens[$i])) {
                $namespace .= $tokens[$i][1];
            } elseif ($tokens[$i] == ';') {
                return $namespace;
            }
        }
    }
546583
public function process($tokens) {
        $prev_category = "";
        $token_category = "";
        $skip_next = 0;
        $out = false;

	// $tokens may come as a numeric indexed array starting with an index greater than 0 (or as a boolean)
	$tokenCount = count($tokens);
        if ( is_array($tokens) ){
          $tokens = array_values($tokens);
        }
        for ($tokenNumber = 0; $tokenNumber < $tokenCount; ++$tokenNumber) {

            // https://github.com/greenlion/PHP-SQL-Parser/issues/279
            // https://github.com/sinri/PHP-SQL-Parser/commit/eac592a0e19f1df6f420af3777a6d5504837faa7
            // as there is no pull request for 279 by the user. His solution works and tested.
            if (!isset($tokens[$tokenNumber])) continue;// as a fix by Sinri 20180528
            $token = $tokens[$tokenNumber];
            $trim = trim($token); // this removes also \n and \t!

            // if it starts with an "(", it should follow a SELECT
            if ($trim !== "" && $trim[0] === "(" && $token_category === "") {
                $token_category = 'BRACKET';
                $prev_category = $token_category;
            }

            /*
             * If it isn't obvious, when $skip_next is set, then we ignore the next real token, that is we ignore whitespace.
             */
            if ($skip_next > 0) {
                if ($trim === "") {
                    if ($token_category !== "") { // is this correct??
                        $out[$token_category][] = $token;
                    }
                    continue;
                }
                // to skip the token we replace it with whitespace
                $trim = "";
                $token = "";
                $skip_next--;
                if ($skip_next > 0) {
                    continue;
                }
            }

            $upper = strtoupper($trim);
            switch ($upper) {

            /* Tokens that get their own sections. These keywords have subclauses. */
            case 'SELECT':
            case 'ORDER':
            case 'VALUES':
            case 'GROUP':
            case 'HAVING':
            case 'WHERE':
            case 'CALL':
            case 'PROCEDURE':
            case 'FUNCTION':
            case 'SERVER':
            case 'LOGFILE':
            case 'DEFINER':
            case 'RETURNS':
            case 'TABLESPACE':
            case 'TRIGGER':
            case 'DO':
            case 'FLUSH':
            case 'KILL':
            case 'RESET':
            case 'STOP':
            case 'PURGE':
            case 'EXECUTE':
            case 'PREPARE':
                $token_category = $upper;
                break;

            case 'DEALLOCATE':
                if ($trim === 'DEALLOCATE') {
                    $skip_next = 1;
                }
                $token_category = $upper;
                break;

            case 'DUPLICATE':
                if ($token_category !== 'VALUES') {
                    $token_category = $upper;
                }
                break;

            case 'SET':
                if ($token_category !== 'TABLE') {
                    $token_category = $upper;
                }
                break;

            case 'LIMIT':
            case 'PLUGIN':
            // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'FROM':
            // this FROM is different from FROM in other DML (not join related)
                if ($token_category === 'PREPARE') {
                    continue 2;
                }
                // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EXPLAIN':
            case 'DESCRIBE':
            case 'SHOW':
                $token_category = $upper;
                break;

            case 'DESC':
                if ($token_category === '') {
                    // short version of DESCRIBE
                    $token_category = $upper;
                }
                // else direction of ORDER-BY
                break;

            case 'RENAME':
                $token_category = $upper;
                break;

            case 'DATABASE':
            case 'SCHEMA':
                if ($prev_category === 'DROP') {
                    continue;
                }
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EVENT':
            // issue 71
                if ($prev_category === 'DROP' || $prev_category === 'ALTER' || $prev_category === 'CREATE') {
                    $token_category = $upper;
                }
                break;

            case 'DATA':
            // prevent wrong handling of DATA as keyword
                if ($prev_category === 'LOAD') {
                    $token_category = $upper;
                }
                break;

            case 'INTO':
            // prevent wrong handling of CACHE within LOAD INDEX INTO CACHE...
                if ($prev_category === 'LOAD') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $prev_category = $upper;
                break;

            case 'USER':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'RENAME' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            case 'VIEW':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'ALTER' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            /*
             * These tokens get their own section, but have no subclauses. These tokens identify the statement but have no specific subclauses of their own.
             */
            case 'DELETE':
            case 'ALTER':
            case 'INSERT':
            case 'TRUNCATE':
            case 'OPTIMIZE':
            case 'GRANT':
            case 'REVOKE':
            case 'HANDLER':
            case 'LOAD':
            case 'ROLLBACK':
            case 'SAVEPOINT':
            case 'UNLOCK':
            case 'INSTALL':
            case 'UNINSTALL':
            case 'ANALZYE':
            case 'BACKUP':
            case 'CHECKSUM':
            case 'REPAIR':
            case 'RESTORE':
            case 'HELP':
                $token_category = $upper;
                // set the category in case these get subclauses in a future version of MySQL
                $out[$upper][0] = $trim;
                continue 2;

            case 'REPLACE':
            	if ($prev_category === '') {
            		// set the category in case these get subclauses in a future version of MySQL
            		$token_category = $upper;
            		$out[$upper][0] = $trim;
            		continue 2;
            	}
                // part of the CREATE TABLE statement or a function
                $out[$prev_category][] = $trim;
                continue 2;

            case 'IGNORE':
                if ($prev_category === 'TABLE') {
                    // part of the CREATE TABLE statement
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                if ($token_category === 'FROM') {
                    // part of the FROM statement (index hint)
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $upper;
                continue 2;

            case 'CHECK':
                if ($prev_category === 'TABLE') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'CREATE':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'INDEX':
	            if ( in_array( $prev_category, array( 'CREATE', 'DROP' ) ) ) {
		            $out[ $prev_category ][] = $trim;
		            $token_category          = $upper;
	            }
	            break;

            case 'TABLE':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                if ($prev_category === 'TRUNCATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                break;

            case 'TEMPORARY':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $prev_category;
                    continue 2;
                }
                break;

            case 'IF':
                if ($prev_category === 'TABLE') {
                    $token_category = 'CREATE';
                    $out[$token_category] = array_merge($out[$token_category], $out[$prev_category]);
                    $out[$prev_category] = array();
                    $out[$token_category][] = $trim;
                    $prev_category = $token_category;
                    continue 2;
                }
                break;

            case 'NOT':
                if ($prev_category === 'CREATE') {
                    $token_category = $prev_category;
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                break;

            case 'EXISTS':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $prev_category = $token_category = 'TABLE';
                    continue 2;
                }
                break;

            case 'CACHE':
                if ($prev_category === "" || $prev_category === 'RESET' || $prev_category === 'FLUSH'
                    || $prev_category === 'LOAD') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* This is either LOCK TABLES or SELECT ... LOCK IN SHARE MODE */
            case 'LOCK':
                if ($token_category === "") {
                    $token_category = $upper;
                    $out[$upper][0] = $trim;
                } elseif ($token_category === 'INDEX') {
                    break;
                } else {
                    $trim = 'LOCK IN SHARE MODE';
                    $skip_next = 3;
                    $out['OPTIONS'][] = $trim;
                }
                continue 2;

            case 'USING': /* USING in FROM clause is different from USING w/ prepared statement*/
                if ($token_category === 'EXECUTE') {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'FROM' && !empty($out['DELETE'])) {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* DROP TABLE is different from ALTER TABLE DROP ... */
            case 'DROP':
                if ($token_category !== 'ALTER') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            case 'FOR':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $skip_next = 1;
                $out['OPTIONS'][] = 'FOR UPDATE'; // TODO: this could be generate problems within the position calculator
                continue 2;

            case 'UPDATE':
                if ($token_category === "") {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            case 'START':
                $trim = "BEGIN";
                $out[$upper][0] = $upper; // TODO: this could be generate problems within the position calculator
                $skip_next = 1;
                break;

            // This token is ignored, except within RENAME
            case 'TO':
                if ($token_category === 'RENAME') {
                    break;
                }
                continue 2;

            // This token is ignored, except within CREATE TABLE
            case 'BY':
                if ($prev_category === 'TABLE') {
                    break;
                }
                continue 2;

            // These tokens are ignored.
            case 'ALL':
            case 'SHARE':
            case 'MODE':
            case ';':
                continue 2;

            case 'KEY':
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            /* These tokens set particular options for the statement. */
            case 'LOW_PRIORITY':
            case 'DELAYED':
            case 'QUICK':
            case 'HIGH_PRIORITY':
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'USE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                // set the category in case these get subclauses in a future version of MySQL
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'FORCE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'WITH':
                if ($token_category === 'GROUP') {
                    $skip_next = 1;
                    $out['OPTIONS'][] = 'WITH ROLLUP'; // TODO: this could be generate problems within the position calculator
                    continue 2;
                }
                if ($token_category === '') {
                	$token_category = $upper;
                }
                break;

            case 'AS':
                break;

            case '':
            case ',':
            case ';':
                break;

            default:
                break;
            }

            // remove obsolete category after union (empty category because of
            // empty token before select)
            if ($token_category !== "" && ($prev_category === $token_category)) {
                $out[$token_category][] = $token;
            }

            $prev_category = $token_category;
        }

        return parent::process($out);
    }
672553
protected function parseTokensContent(array $tokens): string
    {
        $statement = '';
        foreach ($tokens as $token) {
            $statement .= $token['content'];
        }

        return $statement;
    }
503731
public function parse(array $tokens)
    {
        $parser = new Parser($this->symbolContainer);

        if ($this->customGrammarChecker !== null) {
            $parser->setCustomGrammarChecker($this->customGrammarChecker);
        }

        $rootNode = $parser->parse($tokens);

        return $rootNode;
    }
801227
protected static function _tokenize(array $tokens) {
		$inString = false;
		$results = array();
		$cpt = 0;
		$previousTokenId = null;
		foreach ($tokens as $tokenId => $token) {
			if ($token['content'] === '"') {
				if (!$inString) {
					$token['id'] = T_START_DOUBLE_QUOTE;
					$token['name'] = 'T_START_DOUBLE_QUOTE';
				} else {
					$token['id'] = T_END_DOUBLE_QUOTE;
					$token['name'] = 'T_END_DOUBLE_QUOTE';
				}
				$inString = !$inString;
			}

			$isCurlyBrace = (
				$token['content'] === '$' &&
				isset($tokens[$tokenId + 1]) &&
				$tokens[$tokenId + 1]['content'] === '{'
			);
			if ($isCurlyBrace) {
				$token['id'] = T_DOLLAR_CURLY_BRACES;
				$token['name'] = 'T_DOLLAR_CURLY_BRACES';
			}

			$isArray = (
				!$token['id'] &&
				$token['content'] === '(' &&
				$previousTokenId !== null &&
				$tokens[$previousTokenId]['id'] === T_ARRAY
			);
			if ($isArray) {
				$token['id'] = T_ARRAY_OPEN;
				$token['name'] = 'T_ARRAY_OPEN';
			}

			$isShortArray = (
				!$token['id'] &&
				$token['content'] === '[' &&
				$previousTokenId !== null && (
					$tokens[$previousTokenId]['id'] !== T_VARIABLE &&
					$tokens[$previousTokenId]['id'] !== T_ENCAPSED_AND_WHITESPACE
				)
			);
			if ($isShortArray) {
				$token['id'] = T_SHORT_ARRAY_OPEN;
				$token['name'] = 'T_SHORT_ARRAY_OPEN';
			}

			if ($token['content'] === '(' && !$token['id']) {
				$token['id'] = T_START_BRACKET;
				$token['name'] = 'T_START_BRACKET';
			}

			if ($token['id'] !== T_WHITESPACE) {
				$previousTokenId = $tokenId;
			}

			if ($token['id'] === T_IF) {
				$i = $cpt;
				$spaces = '';
				while ($i-- > 0 && $results[$i]['id'] === T_WHITESPACE) {
					$spaces = $results[$i]['content'] . $spaces;
				}
				if ($i >= 0 && $results[$i]['id'] === T_ELSE) {
					$token['id'] = T_ELSEIF;
					$token['name'] = 'T_ELSEIF';
					$token['content'] = $results[$i]['content'] . $spaces . $token['content'];
					$cpt = $i;
				}
			}

			$results[$cpt++] = $token;
		}

		return $results;
	}
1072066
function getTokens (string) {
  let tokens = null
  try { tokens = esprima.tokenize('`'+ string +'`') }
  catch (e) { tokens = [] }
  return tokens
}
703210
public static function offset(array $tokens, $index)
    {
        if (count($tokens) < 1) {
            return 0;
        }

        if (null === $index) {
            $index = count($tokens) - 1;
        }

        $source = '';

        for ($i = 0; $i < $index; ++$i) {
            $source .= $tokens[$i]->content();
        }

        return strlen($source);
    }
1114783
public function resend($tokenString)
    {
        $token = $this->findToken($tokenString);

        if (! is_null($token)) {
            session()->forget('inactive_email');

            $token->user->resendInvite();
        }
    }
348029
private static function getSegmentFromToken($token, $key, array $tokens)
    {
        $relation_tokens = array('>', '~', '+');

        if (\in_array($token, $relation_tokens, true)) { // not a segment
            return false;
        }

        $segment = (object) array(
            'selector' => '',
            'relation_token' => false,
            'attribute_filters' => array(),
            'pseudo_filters' => array()
        );

        if (isset($tokens[$key-1]) && \in_array($tokens[$key-1], $relation_tokens, true)) { // get relationship token
            $segment->relation_token = $tokens[$key-1];
        }

        if (ctype_alpha($token)) { // simple element selector
            $segment->selector = $token;
            return $segment;
        }

        $char_tmp_replaced = false;
        if (strpos($token, '\\') !== false) {
            $token = preg_replace_callback( // temporary replace escaped characters
                '#(\\\\)(.{1})#',
                function ($matches) {
                    return 'ESCAPED'. \ord($matches[2]);
                },
                $token
            );
            $char_tmp_replaced = true;
        }

        $segment->selector = $token;

        while (preg_match('/(.*)\:(not|contains|has)\((.+)\)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2].'('.$matches[3].')';
        }

        while (preg_match('/(.*)\:([a-z][a-z\-]+)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\[([^]]+)\]$/', $segment->selector, $matches)) { // attribute selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\.([^\.\#]+)$/i', $segment->selector, $matches)) { // class selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'class~="'.$matches[2].'"';
        }

        while (preg_match('/(.*)\#([^\.\#]+)$/i', $segment->selector, $matches)) { // id selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'id="'.$matches[2].'"';
        }

        if ($char_tmp_replaced) { // restore temporary replaced characters
            $set_escape_back = function (string $str) {
                return preg_replace_callback(
                    '#(ESCAPED)([0-9]{1,3})#',
                    function ($matches) {
                        return \chr($matches[2]);
                    },
                    $str
                );
            };

            $segment->selector = $set_escape_back($segment->selector);

            foreach ($segment->attribute_filters as &$attr_filter) {
                $attr_filter = $set_escape_back($attr_filter);
            }
        }

        return $segment;
    }
453074
protected function _parseTag($string, array $keys = []) {
		$parts = explode(' ', $string, count($keys));
		$result = array_fill_keys($keys, null);

		foreach ($keys as $i => $key) {
			if (isset($parts[$i])) {
				if ($key === 'description') {
					$result[$key] = $this->_clean($parts[$i]);
				} else {
					$result[$key] = $parts[$i];
				}
			}
		}
		return $result;
	}
41935
static function parseUseTrait(array $tokens, &$i) {
    ++$i;
    $names = ParserUtil::parseIdentifierList($tokens, $i);
    if (FALSE === $names) {
      return FALSE;
    }
    $id = ParserUtil::nextSubstantialIncl($tokens, $i);
    if (';' === $id) {
      ++$i;
      return new AstUseTrait($names);
    }
    elseif ('{' === $id) {
      ParserUtil::skipCurly($tokens, $i);
      return new AstUseTrait($names);
    }
    else {
      return FALSE;
    }
  }
1132206
public function generateUrl(array $tokens = [])
    {
        $path = $this->path;

        foreach ($tokens as $key => $value) {
            $path = str_replace(":{$key}", $value, $path);
        }

        return $path;
    }
413131
public static function create(array $tokens, array $options) {
        HashMap::assert($tokens, 'tokens');

        $format    = Format::create($options);
        $operation = SqlOperation::create($tokens);

        if ($operation === SqlOperations::INSERT) return $format->encode(InsertChangeSet::create($tokens));
        if ($operation === SqlOperations::UPDATE) return $format->encode(UpdateChangeSet::create($tokens));

        return null;
    }
936238
public static function create(array $tokens, $content) {
        if (empty($content) || !is_array($content)) return [];
        return empty($content[0]) ? SelectSingleResult::create($tokens, $content) : SelectAllResult::create($tokens, $content);
    }
49025
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        $columns = array_map(function($token) {
            $segments = explode('=', $token['base_expr']);
            return trim($segments[0]);
        }, $tokens['SET']);

        $values = array_map(function($token) {
            $segments = explode('=', $token['base_expr']);
            return Value::create(trim($segments[1]));
        }, $tokens['SET']);

        return array_combine($columns, $values);
    }
921646
protected function strTokenWrapper($string)
    {
        $token = strtok($string, $this->tokenExpression);

        $tokens = array();
        while ($token !== false) {
            // avoid tokenizing white spaces
            if(!empty(trim($token))) { 
                $tokens[] = $token;
            }
            $token = strtok($this->tokenExpression);        
        }
        return $tokens;
    }