Serialize an array of AVT tokens back into an attribute value

@param  array  $tokens
@return string
176040
public static function serialize(array $tokens)
	{
		$attrValue = '';
		foreach ($tokens as $token)
		{
			if ($token[0] === 'literal')
			{
				$attrValue .= preg_replace('([{}])', '$0$0', $token[1]);
			}
			elseif ($token[0] === 'expression')
			{
				$attrValue .= '{' . $token[1] . '}';
			}
			else
			{
				throw new RuntimeException('Unknown token type');
			}
		}

		return $attrValue;
	}
143757
protected function serializeCharacterClass(array $values)
	{
		$expr = '[';
		foreach ($this->getRanges($values) as list($start, $end))
		{
			$expr .= $this->serializeCharacterClassUnit($start);
			if ($end > $start)
			{
				if ($end > $start + 1)
				{
					$expr .= '-';
				}
				$expr .= $this->serializeCharacterClassUnit($end);
			}
		}
		$expr .= ']';

		return $expr;
	}
731137
private function parseToken(array $token): string
    {
        list($id, $content) = $token;

        if ($id == T_INLINE_HTML) {
            $content = $this->compileVerbatim($content);

            foreach ($this->compilers as $callable) {
                if (!is_callable($callable) and is_scalar($callable)) {
                    $callable = [$this, $callable];
                }

                $content = (string) call_user_func_array($callable, [$content]);
            }

            $content = $this->restoreVerbatim($content);
        }

        return $content;
    }
898852
private function extractUseStatement(\ArrayIterator $tokens)
    {
        $result = [''];
        while ($tokens->valid()
            && (self::T_LITERAL_USE_SEPARATOR !== $tokens->current()[0])
            && (self::T_LITERAL_END_OF_USE !== $tokens->current()[0])
        ) {
            if (T_AS === $tokens->current()[0]) {
                $result[] = '';
            }
            if (T_STRING === $tokens->current()[0] || T_NS_SEPARATOR === $tokens->current()[0]) {
                $result[\count($result) - 1] .= $tokens->current()[1];
            }
            $tokens->next();
        }

        if (1 == \count($result)) {
            $backslashPos = strrpos($result[0], '\\');

            if (false !== $backslashPos) {
                $result[] = substr($result[0], $backslashPos + 1);
            } else {
                $result[] = $result[0];
            }
        }

        return array_reverse($result);
    }
266593
private function grabAttribsBeforePropertyToken(Tokens $tokens, $index)
    {
        static $tokenAttribsMap = [
            T_VAR       => null, // destroy T_VAR token!
            T_PRIVATE   => 'visibility',
            T_PROTECTED => 'visibility',
            T_PUBLIC    => 'visibility',
            T_STATIC    => 'static',
        ];

        $result = $this->grabAttribsBeforeToken(
            $tokens,
            $index,
            $tokenAttribsMap,
            [
                'visibility' => new Token([T_PUBLIC, 'public']),
                'static'     => null,
            ]
        );
        if ($result['visibility'] && 'public' === $result['visibility']->getContent()) {
            // If visibility is public and static is set, we remove visibility.
            if ($result['static']) {
                $result['visibility'] = null;
            }
        }
        return $result;
    }
269949
function translate($str, $tokens = [])
    {
        if (array_key_exists($str, $this->translation)) {
            $str = $this->translation[$str];
        }
        if (is_array($tokens) && sizeof($tokens) > 0) {
            $str = vsprintf($str, $tokens);
        }

        return $str;
    }
166650
public function attributesToString(array $attributes = []): string
    {
        $html = [];
        foreach ($attributes as $key => $value) {
            if (is_array($value)) {
                $value = json_encode($value);
            }

            if (in_array($key, $this->booleanAttributes, true)) {
                $value = $this->prepareBooleanAttribute($key, $value);
            }

            if ($value === '') {
                continue;
            }

            $html[] = sprintf(
                '%s="%s"',
                $this->escapeAttribute($key),
                $this->escapeAttribute($value)
            );
        }

        return implode(' ', $html);
    }
908437
private function getLicenseTokenIndex(array $tokens)
    {
        $licenseTokenIndex = null;

        foreach ($tokens as $index => $token) {
            if ($token[0] === T_COMMENT) {
                $licenseTokenIndex = $index;
            }

            // if we reach the class declaration then it does not have a license
            if ($token[0] === T_CLASS) {
                break;
            }
        }

        return $licenseTokenIndex;
    }
1181837
public function apply(array &$tokens)
		{
		$r = 0;
		$transformations = &$this->transformations;
		for ($i = 0, $l = count($tokens); $i < $l; $i++)
			{
			if (get_class($tokens[$i]) === "CssRulesetDeclarationToken")
				{
				$tProperty = $tokens[$i]->Property;
				if (isset($transformations[$tProperty]))
					{
					$result = array();
					if (is_callable($transformations[$tProperty]))
						{
						$result = call_user_func_array($transformations[$tProperty], array($tokens[$i]));
						if (!is_array($result) && is_object($result))
							{
							$result = array($result);
							}
						}
					else
						{
						$tValue			= $tokens[$i]->Value;
						$tMediaTypes	= $tokens[$i]->MediaTypes;
						foreach ($transformations[$tProperty] as $property)
							{
							if ($property !== null)
								{
								$result[] = new CssRulesetDeclarationToken($property, $tValue, $tMediaTypes);
								}
							}
						}
					if (count($result) > 0)
						{
						array_splice($tokens, $i + 1, 0, $result);
						$i += count($result);
						$l += count($result);
						}
					}
				}
			}
		return $r;
		}
1367179
private function cleanupGroupDelimiters(array &$tokens)
    {
        $indexes = $this->getUnmatchedGroupDelimiterIndexes($tokens);

        while (!empty($indexes)) {
            $lastIndex = array_pop($indexes);
            $token = $tokens[$lastIndex];
            unset($tokens[$lastIndex]);

            if ($token->type === Tokenizer::TOKEN_GROUP_BEGIN) {
                $this->addCorrection(
                    self::CORRECTION_UNMATCHED_GROUP_LEFT_DELIMITER_IGNORED,
                    $token
                );
            } else {
                $this->addCorrection(
                    self::CORRECTION_UNMATCHED_GROUP_RIGHT_DELIMITER_IGNORED,
                    $token
                );
            }
        }
    }
403468
private function getQueryNorm($tokens)
    {
        $db = $this->getSearchDbHandle();
        $db->/** @scrutinizer ignore-call */
        sqliteCreateFunction('sqrt', 'sqrt', 1);
        foreach ($tokens as $key => $token) {
            $tokens[$key] = $db->quote($token);
        }
        $terms = implode(',', $tokens);
        $sql = '
			SELECT (1 / sqrt(SUM(inverseDocumentFrequency))) AS queryNorm
			  FROM inverse_document_frequency
			 WHERE term IN (' . $terms . ') 
		';
        if (!$stmt = $db->prepare($sql)) {
            throw new \RuntimeException('SQLite exception: <pre>' . print_r($db->errorInfo(),
                    true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
        }
        if (!$stmt->execute()) {
            throw new \RuntimeException('SQLite exception: <pre>' . print_r($db->errorInfo(),
                    true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
        }
        $result = $stmt->fetch(\PDO::FETCH_OBJ);
        return $result->queryNorm == null ? 1 : $result->queryNorm;
    }
412841
public static function columns(array $tokens) {
        $columns = array_filter($tokens['INSERT'], function($token) {
            return $token['expr_type'] === 'column-list';
        });

        return array_map(function($column) {
            return end($column['no_quotes']['parts']);
        }, end($columns)['sub_tree']);
    }
594549
protected function getNamedCapturesExpressions(array $tokens)
	{
		$exprs = [];
		foreach ($tokens as $token)
		{
			if ($token['type'] !== 'capturingSubpatternStart' || !isset($token['name']))
			{
				continue;
			}

			$expr = $token['content'];
			if (strpos($expr, '|') !== false)
			{
				$expr = '(?:' . $expr . ')';
			}

			$exprs[$token['name']] = $expr;
		}

		return $exprs;
	}
659508
public function encode(array $headers): string {
        // @TODO implementation is deliberately primitive... [doesn't use any dynamic table...]
        $output = "";

        foreach ($headers as $name => $values) {
            foreach ((array) $values as $value) {
                if (isset(self::$indexMap[$name])) {
                    $index = self::$indexMap[$name];
                    if ($index < 0x10) {
                        $output .= \chr($index);
                    } else {
                        $output .= "\x0f" . \chr($index - 0x0f);
                    }
                } elseif (\strlen($name) < 0x7f) {
                    $output .= "\0" . \chr(\strlen($name)) . $name;
                } else {
                    $output .= "\0\x7f" . self::encodeDynamicInteger(\strlen($name) - 0x7f) . $name;
                }

                if (\strlen($value) < 0x7f) {
                    $output .= \chr(\strlen($value)) . $value;
                } else {
                    $output .= "\x7f" . self::encodeDynamicInteger(\strlen($value) - 0x7f) . $value;
                }
            }
        }

        return $output;
    }
49025
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        $columns = array_map(function($token) {
            $segments = explode('=', $token['base_expr']);
            return trim($segments[0]);
        }, $tokens['SET']);

        $values = array_map(function($token) {
            $segments = explode('=', $token['base_expr']);
            return Value::create(trim($segments[1]));
        }, $tokens['SET']);

        return array_combine($columns, $values);
    }
431597
private function getCurrentConstantKeyFromClassTokens($tokens)
    {
        $parsingStateConstDeclarationFound = false;
        $constDeclarationKey = 0;

        foreach ($tokens as $key => $token) {
            if (is_array($token) === true && $token[0] === T_CONST) {
                $parsingStateConstDeclarationFound = true;
                $constDeclarationKey = $key;
            }

            if ($parsingStateConstDeclarationFound === true && $token[0] === T_STRING) {
                if ($token[1] === $this->getName()) {
                    break;
                } else {
                    $parsingStateConstDeclarationFound = false;
                }
            }
        }

        return $constDeclarationKey;
    }
947853
protected function recurseArrayForTokens($arr, $tokens)
    {
        $out = [];
        foreach ($arr as $name => $value) {
            if (is_array($value)) {
                $out[$name] = $this->recurseArrayForTokens($value, $tokens);
                continue;
            }
            if (is_string($value)) {
                foreach ($tokens as $token => $tval) {
                    $value = str_replace($token, $tval, $value);
                }
                $out[$name] = $value;
                continue;
            }
            $out[$name] = $value;
        }

        return $out;
    }