Tokenizes a string into a TokenStream

@param $input
@return TokenStream
761110
public function tokenize(string $input) : TokenStream
    {
        $this->prepare($input);

        // Loop each character
        while ($this->cursor < $this->end) {
            $this->currentChar = $this->input[$this->cursor];

            if (preg_match('@'.Token::REGEX_T_EOL.'@', $this->currentChar)) {
                if ($this->mode !== self::MODE_IDENT) {
                    ++$this->line;
                    $this->lineOffset = $this->cursor + 1;
                }
            }

            switch ($this->mode) {

                case self::MODE_ALL:
                    $this->lexAll();
                    break;
                case self::MODE_INSIDE_TAG:
                    $this->lexInsideTag();
                    break;
                case self::MODE_IDENT:
                    $this->lexIdent();
                    break;
                case self::MODE_VAR:
                    $this->lexVar();
                    break;
                case self::MODE_STRING:
                    $this->lexString();
                    break;
                case self::MODE_NUMBER:
                    $this->lexNumber();
                    break;
                case self::MODE_OP:
                    $this->lexOperator();
                    break;
            }
        }

        return $this->stream;
    }
1140730
public static function fromString(string $input): StreamInterface
    {
        /** @var resource $stream */
        $stream = \fopen('php://temp', 'w+');
        if (!\is_resource($stream)) {
            throw new \Error('Could not create stream');
        }
        \fwrite($stream, $input);
        \rewind($stream);
        return new static($stream);
    }
202701
public function encode($input)
    {
        $chars = [
            '&#x102;' => 'a',
            '&#x103;' => 'a',
            '&#xC2;' => 'A',
            '&#xE2;' => 'a',
            '&#xCE;' => 'I',
            '&#xEE;' => 'i',
            '&#x218;' => 'S',
            '&#x219;' => 's',
            '&#x15E;' => 'S',
            '&#x15F;' => 's',
            '&#x21A;' => 'T',
            '&#x21B;' => 't',
            '&#354;' => 'T',
            '&#355;' => 't'
        ];

        $change = $with = [];
        foreach ($chars as $i => $v) {
            $change[] = html_entity_decode($i, ENT_QUOTES, 'UTF-8');
            $with[] = $v;
        }
        $input = str_replace($change, $with, $input);

        preg_match_all("/[a-z0-9]+/i", $input, $sections);
        return strtolower(implode("-", $sections[0]));
    }
957944
protected function toBuffer($input)
    {
        if (is_string($input)) {
            if (strpos($input, '0x') === 0) {
                // hex string
                // $input = str_replace('0x', '', $input);
                return new Buffer($input, 'hex');
            }
            return new Buffer(str_split($input, 1));
        } elseif (is_numeric($input)) {
            if (!$input || $input < 0) {
                return new Buffer([]);
            }
            if (is_float($input)) {
                $input = number_format($input, 0, '', '');
                var_dump($input);
            }
            $gmpInput = gmp_init($input, 10);
            return new Buffer('0x' . gmp_strval($gmpInput, 16), 'hex');
        } elseif ($input === null) {
            return new Buffer([]);
        } elseif (is_array($input)) {
            return new Buffer($input);
        } elseif ($input instanceof Buffer) {
            return $input;
        }
        throw new InvalidArgumentException('The input type didn\'t support.');
    }
1305240
public function canonicalize($input)
    {
        $strippedInput = str_replace(' ', '', $input);
        if (!preg_match('/^\d{4}[A-Za-z]{2}$/', $strippedInput)) {
            return $input;
        }

        return implode(' ', [substr($strippedInput, 0, 4), mb_strtoupper(substr($strippedInput, -2, 2))]);
    }
131121
public function stringToStream(string $input): StreamInterface
    {
        /** @var StreamInterface|null $stream */
        $stream = stream_for($input);
        if (!($stream instanceof StreamInterface)) {
            throw new \TypeError('Could not convert string to a stream');
        }
        return $stream;
    }
885022
protected static function decodePart($input)
    {
        $n = static::INITIAL_N;
        $i = 0;
        $bias = static::INITIAL_BIAS;
        $output = '';

        $pos = strrpos($input, static::DELIMITER);

        if ($pos !== false) {
            $output = substr($input, 0, $pos++);
        } else {
            $pos = 0;
        }

        $outputLength = strlen($output);
        $inputLength = strlen($input);
        $res = array();

        while ($pos < $inputLength) {
            $oldi = $i;
            $w = 1;

            for ($k = static::BASE;; $k += static::BASE) {
                $digit = static::$decodeTable[$input[$pos++]];
                $i = $i + ($digit * $w);
                $t = static::calculateThreshold($k, $bias);

                if ($digit < $t) {
                    break;
                }

                $w = $w * (static::BASE - $t);
            }

            $bias = static::adapt($i - $oldi, ++$outputLength, ($oldi === 0));
            $n = $n + (int) ($i / $outputLength);
            $i = $i % ($outputLength);

            if ($n < 0x80) {
                $cp = chr($n);
            } elseif ($n <= 0x7FF) {
                $cp = chr(($n >> 6) + 0xC0) . chr(($n & 0x3F) + 0x80);
            } elseif ($n <= 0xFFFF) {
                $cp = chr(($n >> 12) + 0xE0) . chr((($n >> 6) & 0x3F) + 0x80) . chr(($n & 0x3F) + 0x80);
            } elseif ($n <= 0x10FFFF) {
                $cp = chr(($n >> 18) + 0xF0) . chr((($n >> 12) & 0x3F) + 0x80)
                    . chr((($n >> 6) & 0x3F) + 0x80) . chr(($n & 0x3F) + 0x80);
            } else {
                throw new Exception("Invalid UTF-8");
            }

            $output = substr($output, 0, $i) . $cp . substr($output, $i, $outputLength - 1);

            $i += strlen($cp);
        }

        return $output;
    }
1098762
private function prepare(string $input)
    {
        $this->input = str_replace(["\n\r", "\r"], "\n", $input);

        // Create new token stream
        $this->stream = new TokenStream();

        $this->cursor = 0;
        $this->line = 1;
        $this->lineOffset = 0;
        $this->end = strlen($this->input);
        $this->lastCharPos = $this->end - 1;

        $this->currentChar = '';
        $this->currentValue = '';
        $this->modeStartChar = '';
        $this->modeStartPosition = 0;
        $this->modeStartLine = 0;

        // Set the mode to "ALL"
        $this->setMode(self::MODE_ALL);
    }
400623
private function inputInOpenStringOrComment($input)
    {
        if (!$this->hasCode()) {
            return;
        }

        $code = $this->codeBuffer;
        \array_push($code, $input);
        $tokens = @\token_get_all('<?php ' . \implode("\n", $code));
        $last = \array_pop($tokens);

        return $last === '"' || $last === '`' ||
            (\is_array($last) && \in_array($last[0], [T_ENCAPSED_AND_WHITESPACE, T_START_HEREDOC, T_COMMENT]));
    }
22750
private function fromCompactSerializationToSerialization($input)
    {
        $parts = explode('.', $input);
        switch (count($parts)) {
            case 3:
                $this->log(LogLevel::DEBUG, 'The input seems to be a compact JWS.');

                return $this->fromCompactSerializationSignatureToSerialization($parts);
            case 5:
                $this->log(LogLevel::DEBUG, 'The input seems to be a compact JWE.');

                return $this->fromCompactSerializationRecipientToSerialization($parts);
            default:
                throw new \InvalidArgumentException('Unsupported input');
        }
    }
1074278
public function setInput($input)
    {
        if ($this->linebuffer === $input) {
            return $this;
        }

        // remember old input length if echo replacement is used
        $oldlen = (is_string($this->echo)) ? $this->strlen($this->linebuffer) : null;

        $this->linebuffer = $input;
        $this->linepos = $this->strlen($this->linebuffer);

        // only redraw if input should be echo'ed (i.e. is not hidden anyway)
        // and echo replacement is used, make sure the input length changes
        if ($this->echo !== false && $this->linepos !== $oldlen) {
            $this->redraw();
        }

        return $this;
    }
1242402
public function setInput($input)
    {
        // Resetting the parser
        $this->lineIndex = 0;
        $this->startLine = 0;

        if (is_string($input)) {
            // Convering to a stream.
            $stream = fopen('php://temp', 'r+');
            fwrite($stream, $input);
            rewind($stream);
            $this->input = $stream;
        } elseif (is_resource($input)) {
            $this->input = $input;
        } else {
            throw new \InvalidArgumentException('This parser can only read from strings or streams.');
        }
    }
371248
public function parse(string $input)
    {
        $ts = $this->lexer->tokenize($input);
        $parseResult = $this->parseImplementation($ts);

        if ($ts->hasPendingTokens()) {
            throw new SyntaxErrorException('There are tokens not processed.');
        }

        return $parseResult;
    }
222024
public static function fromString(string $input): self
    {
        $input = preg_replace('/[^\d]/', '', $input);

        if (!is_numeric($input)) {
            throw new \InvalidArgumentException("Expects \$input to be a number, \"{$input}\" given.");
        }

        // Get the last digit.
        $checkDigit = (int) $input[strlen($input) - 1];

        // Remove the last digit.
        $number = substr($input, 0, strlen($input) - 1);

        return new self($number, $checkDigit);
    }
233136
public function decode($input)
    {
        $parts = explode('.', $input);

        foreach ($parts as &$part) {
            if (strpos($part, static::PREFIX) === 0) {
                $part = $this->_decodePart(
                    substr($part, strlen(static::PREFIX))
                );
            }
        }

        return implode('.', $parts);
    }
375242
public function matched($input)
    {
        if ($this->token instanceof Token) {
            return $this->token->matched($input);
        }
        
        return $input;
    }
379392
public function import($input)
    {
        if (is_array($input)) {
            $this->fromArray($input);
        }
        else {
            $this->fromString($input);
        }

        return $this;
    }