import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY





@is_pipeline_test
@require_torch_or_tf
class      lowerCAmelCase						(					unittest.TestCase			):
     _SCREAMING_SNAKE_CASE     :     int		=MODEL_FOR_CAUSAL_LM_MAPPING
     _SCREAMING_SNAKE_CASE     :     List[str]		=TF_MODEL_FOR_CAUSAL_LM_MAPPING



     @require_torch
     def 			a__  (					self       ):
            _A=	pipeline(task='text-generation'				,      model='sshleifer/tiny-ctrl'				,      framework='pt'       )
            # Using `do_sample=False` to force deterministic output
            _A=	text_generator('This is a test'				,      do_sample=lowerCAmelCase__       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    {
                        'generated_text': (
                            'This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'
                            ' oscope. FiliFili@@'
                        )
                    }
                ]				,      )

            _A=	text_generator(['This is a test', 'This is a second test']       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    [
                        {
                            'generated_text': (
                                'This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'
                                ' oscope. FiliFili@@'
                            )
                        }
                    ],
                    [
                        {
                            'generated_text': (
                                'This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy'
                                ' oscope. oscope. FiliFili@@'
                            )
                        }
                    ],
                ]				,      )

            _A=	text_generator('This is a test'				,      do_sample=lowerCAmelCase__				,      num_return_sequences=2				,      return_tensors=lowerCAmelCase__       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    {'generated_token_ids': ANY(lowerCAmelCase__       )},
                    {'generated_token_ids': ANY(lowerCAmelCase__       )},
                ]				,      )
            _A=	text_generator.model.config.eos_token_id
            _A=	'<pad>'
            _A=	text_generator(
                ['This is a test', 'This is a second test']				,      do_sample=lowerCAmelCase__				,      num_return_sequences=2				,      batch_size=2				,      return_tensors=lowerCAmelCase__				,      )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    [
                        {'generated_token_ids': ANY(lowerCAmelCase__       )},
                        {'generated_token_ids': ANY(lowerCAmelCase__       )},
                    ],
                    [
                        {'generated_token_ids': ANY(lowerCAmelCase__       )},
                        {'generated_token_ids': ANY(lowerCAmelCase__       )},
                    ],
                ]				,      )



     @require_tf
     def 			a__  (					self       ):
            _A=	pipeline(task='text-generation'				,      model='sshleifer/tiny-ctrl'				,      framework='tf'       )

            # Using `do_sample=False` to force deterministic output
            _A=	text_generator('This is a test'				,      do_sample=lowerCAmelCase__       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    {
                        'generated_text': (
                            'This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'
                            ' please,'
                        )
                    }
                ]				,      )

            _A=	text_generator(['This is a test', 'This is a second test']				,      do_sample=lowerCAmelCase__       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    [
                        {
                            'generated_text': (
                                'This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'
                                ' please,'
                            )
                        }
                    ],
                    [
                        {
                            'generated_text': (
                                'This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes'
                                ' Cannes 閲閲Cannes Cannes Cannes 攵 please,'
                            )
                        }
                    ],
                ]				,      )



     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__       ):
            _A=	TextGenerationPipeline(model=lowerCAmelCase__				,      tokenizer=lowerCAmelCase__       )
            return text_generator, ["This is a test", "Another test"]



     def 			a__  (					self       ):
            _A=	'Hello I believe in'
            _A=	pipeline('text-generation'				,      model='hf-internal-testing/tiny-random-gpt2'       )
            _A=	text_generator(lowerCAmelCase__       )
            self.assertEqual(
                lowerCAmelCase__				,      [{'generated_text': 'Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe'}]				,      )

            _A=	text_generator(lowerCAmelCase__				,      stop_sequence=' fe'       )
            self.assertEqual(lowerCAmelCase__				,      [{'generated_text': 'Hello I believe in fe'}]       )



     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__       ):
            _A=	text_generator.model
            _A=	text_generator.tokenizer

            _A=	text_generator('This is a test'       )
            self.assertEqual(lowerCAmelCase__				,      [{'generated_text': ANY(lowerCAmelCase__       )}]       )
            self.assertTrue(outputs[0]['generated_text'].startswith('This is a test'       )       )

            _A=	text_generator('This is a test'				,      return_full_text=lowerCAmelCase__       )
            self.assertEqual(lowerCAmelCase__				,      [{'generated_text': ANY(lowerCAmelCase__       )}]       )
            self.assertNotIn('This is a test'				,      outputs[0]['generated_text']       )

            _A=	pipeline(task='text-generation'				,      model=lowerCAmelCase__				,      tokenizer=lowerCAmelCase__				,      return_full_text=lowerCAmelCase__       )
            _A=	text_generator('This is a test'       )
            self.assertEqual(lowerCAmelCase__				,      [{'generated_text': ANY(lowerCAmelCase__       )}]       )
            self.assertNotIn('This is a test'				,      outputs[0]['generated_text']       )

            _A=	text_generator('This is a test'				,      return_full_text=lowerCAmelCase__       )
            self.assertEqual(lowerCAmelCase__				,      [{'generated_text': ANY(lowerCAmelCase__       )}]       )
            self.assertTrue(outputs[0]['generated_text'].startswith('This is a test'       )       )

            _A=	text_generator(['This is great !', 'Something else']				,      num_return_sequences=2				,      do_sample=lowerCAmelCase__       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    [{'generated_text': ANY(lowerCAmelCase__       )}, {'generated_text': ANY(lowerCAmelCase__       )}],
                    [{'generated_text': ANY(lowerCAmelCase__       )}, {'generated_text': ANY(lowerCAmelCase__       )}],
                ]				,      )

            if text_generator.tokenizer.pad_token is not None:
                   _A=	text_generator(
                       ['This is great !', 'Something else']				,      num_return_sequences=2				,      batch_size=2				,      do_sample=lowerCAmelCase__       )
                   self.assertEqual(
                       lowerCAmelCase__				,      [
                           [{'generated_text': ANY(lowerCAmelCase__       )}, {'generated_text': ANY(lowerCAmelCase__       )}],
                           [{'generated_text': ANY(lowerCAmelCase__       )}, {'generated_text': ANY(lowerCAmelCase__       )}],
                       ]				,      )

            with self.assertRaises(lowerCAmelCase__       ):
                   _A=	text_generator('test'				,      return_full_text=lowerCAmelCase__				,      return_text=lowerCAmelCase__       )
            with self.assertRaises(lowerCAmelCase__       ):
                   _A=	text_generator('test'				,      return_full_text=lowerCAmelCase__				,      return_tensors=lowerCAmelCase__       )
            with self.assertRaises(lowerCAmelCase__       ):
                   _A=	text_generator('test'				,      return_text=lowerCAmelCase__				,      return_tensors=lowerCAmelCase__       )

            # Empty prompt is slighly special
            # it requires BOS token to exist.
            # Special case for Pegasus which will always append EOS so will
            # work even without BOS.
            if (
                text_generator.tokenizer.bos_token_id is not None
                or "Pegasus" in tokenizer.__class__.__name__
                or "Git" in model.__class__.__name__
            ):
                   _A=	text_generator(''       )
                   self.assertEqual(lowerCAmelCase__				,      [{'generated_text': ANY(lowerCAmelCase__       )}]       )
            else:
                   with self.assertRaises((ValueError, AssertionError)       ):
                          _A=	text_generator(''       )

            if text_generator.framework == "tf":
                   # TF generation does not support max_new_tokens, and it's impossible
                   # to control long generation with only max_length without
                   # fancy calculation, dismissing tests for now.
                   return
            # We don't care about infinite range models.
            # They already work.
            # Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
            _A=	['RwkvForCausalLM', 'XGLMForCausalLM', 'GPTNeoXForCausalLM']
            if (
                tokenizer.model_max_length < 10000
                and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
            ):
                   # Handling of large generations
                   with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)       ):
                          text_generator('This is a test' * 500				,      max_new_tokens=20       )

                   _A=	text_generator('This is a test' * 500				,      handle_long_generation='hole'				,      max_new_tokens=20       )
                   # Hole strategy cannot work
                   with self.assertRaises(lowerCAmelCase__       ):
                          text_generator(
                              'This is a test' * 500				,      handle_long_generation='hole'				,      max_new_tokens=tokenizer.model_max_length + 10				,      )



     @require_torch
     @require_accelerate
     @require_torch_gpu
     def 			a__  (					self       ):
            import torch

            # Classic `model_kwargs`
            _A=	pipeline(
                model='hf-internal-testing/tiny-random-bloom'				,      model_kwargs={'device_map': 'auto', 'torch_dtype': torch.bfloataa}				,      )
            self.assertEqual(pipe.model.device				,      torch.device(0       )       )
            self.assertEqual(pipe.model.lm_head.weight.dtype				,      torch.bfloataa       )
            _A=	pipe('This is a test'       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    {
                        'generated_text': (
                            'This is a test test test test test test test test test test test test test test test test'
                            ' test'
                        )
                    }
                ]				,      )

            # Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
            _A=	pipeline(model='hf-internal-testing/tiny-random-bloom'				,      device_map='auto'				,      torch_dtype=torch.bfloataa       )
            self.assertEqual(pipe.model.device				,      torch.device(0       )       )
            self.assertEqual(pipe.model.lm_head.weight.dtype				,      torch.bfloataa       )
            _A=	pipe('This is a test'       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    {
                        'generated_text': (
                            'This is a test test test test test test test test test test test test test test test test'
                            ' test'
                        )
                    }
                ]				,      )

            # torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
            _A=	pipeline(model='hf-internal-testing/tiny-random-bloom'				,      device_map='auto'       )
            self.assertEqual(pipe.model.device				,      torch.device(0       )       )
            self.assertEqual(pipe.model.lm_head.weight.dtype				,      torch.floataa       )
            _A=	pipe('This is a test'       )
            self.assertEqual(
                lowerCAmelCase__				,      [
                    {
                        'generated_text': (
                            'This is a test test test test test test test test test test test test test test test test'
                            ' test'
                        )
                    }
                ]				,      )



     @require_torch
     @require_torch_gpu
     def 			a__  (					self       ):
            import torch

            _A=	pipeline(model='hf-internal-testing/tiny-random-bloom'				,      device=0				,      torch_dtype=torch.floataa       )
            pipe('This is a test'       )



     @require_torch
     @require_accelerate
     @require_torch_gpu
     def 			a__  (					self       ):
            import torch

            _A=	pipeline(model='hf-internal-testing/tiny-random-bloom'				,      device_map='auto'				,      torch_dtype=torch.floataa       )
            pipe('This is a test'				,      do_sample=lowerCAmelCase__				,      top_p=0.5       )





     def 			a__  (					self       ):
            _A=	'Hello world'
            _A=	pipeline('text-generation'				,      model='hf-internal-testing/tiny-random-gpt2'       )
            if text_generator.model.framework == "tf":
                   _A=	logging.get_logger('transformers.generation.tf_utils'       )
            else:
                   _A=	logging.get_logger('transformers.generation.utils'       )
            _A=	'Both `max_new_tokens`'  # The beggining of the message to be checked in this test

            # Both are set by the user -> log warning
            with CaptureLogger(lowerCAmelCase__       ) as cl:
                   _A=	text_generator(lowerCAmelCase__				,      max_length=10				,      max_new_tokens=1       )
            self.assertIn(lowerCAmelCase__				,      cl.out       )

            # The user only sets one -> no warning
            with CaptureLogger(lowerCAmelCase__       ) as cl:
                   _A=	text_generator(lowerCAmelCase__				,      max_new_tokens=1       )
            self.assertNotIn(lowerCAmelCase__				,      cl.out       )

            with CaptureLogger(lowerCAmelCase__       ) as cl:
                   _A=	text_generator(lowerCAmelCase__				,      max_length=10       )
            self.assertNotIn(lowerCAmelCase__				,      cl.out       )
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
      UpperCAmelCase_								=							'''pt'''
elif is_tf_available():
      UpperCAmelCase_								=							'''tf'''
else:
      UpperCAmelCase_								=							'''jax'''





class      lowerCAmelCase						(					_a			,		unittest.TestCase			):
     _SCREAMING_SNAKE_CASE     :     Optional[int]		=ByTaTokenizer
     _SCREAMING_SNAKE_CASE     :     Dict		=False



     def 			a__  (					self       ):
            super().setUp()
            _A=	ByTaTokenizer()
            tokenizer.save_pretrained(self.tmpdirname       )



     @cached_property
     def 			a__  (					self       ):
            return ByTaTokenizer.from_pretrained('google/byt5-small'       )



     def 			a__  (					self				,      **lowerCAmelCase__       ):
            return self.tokenizer_class.from_pretrained(self.tmpdirname				,      **lowerCAmelCase__       )



     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__=False				,      lowerCAmelCase__=20				,      lowerCAmelCase__=5       ):
            # XXX The default common tokenizer tests assume that every ID is decodable on its own.
            # This assumption is invalid for ByT5 because single bytes might not be
            # valid utf-8 (byte 128 for instance).
            # Here we're overriding the smallest possible method to provide
            # a clean sequence without making the same assumption.

            _A=	[]
            for i in range(len(lowerCAmelCase__       )       ):
                   try:
                          _A=	tokenizer.decode([i]				,      clean_up_tokenization_spaces=lowerCAmelCase__       )
                   except UnicodeDecodeError:
                          pass
                   toks.append((i, tok)       )

            _A=	list(filter(lambda lowerCAmelCase__       : re.match(r'^[ a-zA-Z]+$'				,      t[1]       )				,      lowerCAmelCase__       )       )
            _A=	list(filter(lambda lowerCAmelCase__       : [t[0]] == tokenizer.encode(t[1]				,      add_special_tokens=lowerCAmelCase__       )				,      lowerCAmelCase__       )       )
            if max_length is not None and len(lowerCAmelCase__       ) > max_length:
                   _A=	toks[:max_length]
            if min_length is not None and len(lowerCAmelCase__       ) < min_length and len(lowerCAmelCase__       ) > 0:
                   while len(lowerCAmelCase__       ) < min_length:
                          _A=	toks + toks
        # toks_str = [t[1] for t in toks]
            _A=	[t[0] for t in toks]

            # Ensure consistency
            _A=	tokenizer.decode(lowerCAmelCase__				,      clean_up_tokenization_spaces=lowerCAmelCase__       )
            if " " not in output_txt and len(lowerCAmelCase__       ) > 1:
                   _A=	(
                       tokenizer.decode([toks_ids[0]]				,      clean_up_tokenization_spaces=lowerCAmelCase__       )
                       + ' '
                       + tokenizer.decode(toks_ids[1:]				,      clean_up_tokenization_spaces=lowerCAmelCase__       )
                   )
            if with_prefix_space:
                   _A=	' ' + output_txt
            _A=	tokenizer.encode(lowerCAmelCase__				,      add_special_tokens=lowerCAmelCase__       )
            return output_txt, output_ids



     def 			a__  (					self       ):
            _A=	self.ta_base_tokenizer
            _A=	tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']       )
            _A=	tokenizer(['hi', 'I went to the gym', '']       )
            self.assertListEqual(batch_with_eos_added['input_ids']				,      batch_without_eos_added['input_ids']       )



     def 			a__  (					self       ):
            _A=	self.ta_base_tokenizer
            _A=	'Unicode €.'
            _A=	tokenizer(lowerCAmelCase__       )
            _A=	[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
            self.assertEqual(encoded['input_ids']				,      lowerCAmelCase__       )

            # decoding
            _A=	tokenizer.decode(lowerCAmelCase__       )
            self.assertEqual(lowerCAmelCase__				,      'Unicode €.</s>'       )

            _A=	tokenizer('e è é ê ë'       )
            _A=	[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
            self.assertEqual(encoded['input_ids']				,      lowerCAmelCase__       )
            # decoding
            _A=	tokenizer.decode(lowerCAmelCase__       )
            self.assertEqual(lowerCAmelCase__				,      'e è é ê ë</s>'       )

            # encode/decode, but with `encode` instead of `__call__`
            self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'       )       )				,      'e è é ê ë</s>'       )



     def 			a__  (					self       ):
            _A=	self.ta_base_tokenizer
            _A=	['A long paragraph for summarization.', 'Another paragraph for summarization.']
            # fmt: off
            _A=	[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
            # fmt: on
            _A=	tokenizer(lowerCAmelCase__				,      padding=lowerCAmelCase__				,      return_tensors=lowerCAmelCase__       )
            self.assertIsInstance(lowerCAmelCase__				,      lowerCAmelCase__       )

            if FRAMEWORK != "jax":
                   _A=	list(batch.input_ids.numpy()[0]       )
            else:
                   _A=	list(batch.input_ids.tolist()[0]       )

            self.assertListEqual(lowerCAmelCase__				,      lowerCAmelCase__       )

            self.assertEqual((2, 37)				,      batch.input_ids.shape       )
            self.assertEqual((2, 37)				,      batch.attention_mask.shape       )



     def 			a__  (					self       ):
            _A=	self.ta_base_tokenizer
            _A=	['A long paragraph for summarization.', 'Another paragraph for summarization.']
            _A=	tokenizer(lowerCAmelCase__				,      padding=lowerCAmelCase__				,      return_tensors=lowerCAmelCase__       )
            # check if input_ids are returned and no decoder_input_ids
            self.assertIn('input_ids'				,      lowerCAmelCase__       )
            self.assertIn('attention_mask'				,      lowerCAmelCase__       )
            self.assertNotIn('decoder_input_ids'				,      lowerCAmelCase__       )
            self.assertNotIn('decoder_attention_mask'				,      lowerCAmelCase__       )



     def 			a__  (					self       ):
            _A=	self.ta_base_tokenizer
            _A=	[
                'Summary of the text.',
                'Another summary.',
            ]
            _A=	tokenizer(
                text_target=lowerCAmelCase__				,      max_length=32				,      padding='max_length'				,      truncation=lowerCAmelCase__				,      return_tensors=lowerCAmelCase__       )
            self.assertEqual(32				,      targets['input_ids'].shape[1]       )



     def 			a__  (					self       ):
            _A=	self.ta_base_tokenizer
            _A=	['A long paragraph for summarization. </s>']
            _A=	['Summary of the text. </s>']
            # fmt: off
            _A=	[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
            _A=	[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
            # fmt: on

            _A=	tokenizer(lowerCAmelCase__				,      text_target=lowerCAmelCase__       )

            self.assertEqual(lowerCAmelCase__				,      batch['input_ids'][0]       )
            self.assertEqual(lowerCAmelCase__				,      batch['labels'][0]       )



     def 			a__  (					self       ):
            # safety check on max_len default value so we are sure the test works
            _A=	self.get_tokenizers()
            for tokenizer in tokenizers:
                   with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                          self.assertNotEqual(tokenizer.model_max_length				,      42       )

        # Now let's start the test
            _A=	self.get_tokenizers()
            for tokenizer in tokenizers:
                   with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                          # Isolate this from the other tests because we save additional tokens/etc
                          _A=	tempfile.mkdtemp()

                          _A=	' He is very happy, UNwant\u00E9d,running'
                          _A=	tokenizer.encode(lowerCAmelCase__				,      add_special_tokens=lowerCAmelCase__       )
                          tokenizer.save_pretrained(lowerCAmelCase__       )

                          _A=	tokenizer.__class__.from_pretrained(lowerCAmelCase__       )
                          _A=	after_tokenizer.encode(lowerCAmelCase__				,      add_special_tokens=lowerCAmelCase__       )
                          self.assertListEqual(lowerCAmelCase__				,      lowerCAmelCase__       )

                          shutil.rmtree(lowerCAmelCase__       )

            _A=	self.get_tokenizers(model_max_length=42       )
            for tokenizer in tokenizers:
                   with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                          # Isolate this from the other tests because we save additional tokens/etc
                          _A=	tempfile.mkdtemp()

                          _A=	' He is very happy, UNwant\u00E9d,running'
                          tokenizer.add_tokens(['bim', 'bambam']       )
                          _A=	tokenizer.additional_special_tokens
                          additional_special_tokens.append('new_additional_special_token'       )
                          tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}       )
                          _A=	tokenizer.encode(lowerCAmelCase__				,      add_special_tokens=lowerCAmelCase__       )
                          tokenizer.save_pretrained(lowerCAmelCase__       )

                          _A=	tokenizer.__class__.from_pretrained(lowerCAmelCase__       )
                          _A=	after_tokenizer.encode(lowerCAmelCase__				,      add_special_tokens=lowerCAmelCase__       )
                          self.assertListEqual(lowerCAmelCase__				,      lowerCAmelCase__       )
                          self.assertIn('new_additional_special_token'				,      after_tokenizer.additional_special_tokens       )
                          self.assertEqual(after_tokenizer.model_max_length				,      42       )

                          _A=	tokenizer.__class__.from_pretrained(lowerCAmelCase__				,      model_max_length=43       )
                          self.assertEqual(tokenizer.model_max_length				,      43       )

                          shutil.rmtree(lowerCAmelCase__       )



     def 			a__  (					self       ):
            _A=	[]
            if self.test_slow_tokenizer:
                   tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

            if self.test_rust_tokenizer:
                   tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

            for tokenizer_class, tokenizer_utils in tokenizer_list:
                   with tempfile.TemporaryDirectory() as tmp_dir:
                          tokenizer_utils.save_pretrained(lowerCAmelCase__       )

                          with open(os.path.join(lowerCAmelCase__				,      'special_tokens_map.json'       )				,      encoding='utf-8'       ) as json_file:
                                 _A=	json.load(lowerCAmelCase__       )

                          with open(os.path.join(lowerCAmelCase__				,      'tokenizer_config.json'       )				,      encoding='utf-8'       ) as json_file:
                                 _A=	json.load(lowerCAmelCase__       )

                          _A=	[f"<extra_id_{i}>" for i in range(125       )]

                          _A=	added_tokens_extra_ids + [
                              'an_additional_special_token'
                          ]
                          _A=	added_tokens_extra_ids + [
                              'an_additional_special_token'
                          ]

                          with open(os.path.join(lowerCAmelCase__				,      'special_tokens_map.json'       )				,      'w'				,      encoding='utf-8'       ) as outfile:
                                 json.dump(lowerCAmelCase__				,      lowerCAmelCase__       )
                          with open(os.path.join(lowerCAmelCase__				,      'tokenizer_config.json'       )				,      'w'				,      encoding='utf-8'       ) as outfile:
                                 json.dump(lowerCAmelCase__				,      lowerCAmelCase__       )

                          # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                          # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                          # "special_tokens_map.json" files
                          _A=	tokenizer_class.from_pretrained(
                              lowerCAmelCase__				,      )
                          self.assertIn(
                              'an_additional_special_token'				,      tokenizer_without_change_in_init.additional_special_tokens       )
                          # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                          self.assertEqual(
                              ['an_additional_special_token']				,      tokenizer_without_change_in_init.convert_ids_to_tokens(
                                  tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']       )       )				,      )

                          # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                          _A=	added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'				,      lstrip=lowerCAmelCase__       )]
                          _A=	tokenizer_class.from_pretrained(
                              lowerCAmelCase__				,      additional_special_tokens=lowerCAmelCase__				,      )

                          self.assertIn('a_new_additional_special_token'				,      tokenizer.additional_special_tokens       )
                          self.assertEqual(
                              ['a_new_additional_special_token']				,      tokenizer.convert_ids_to_tokens(
                                  tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']       )       )				,      )



     def 			a__  (					self       ):
            _A=	[]
            if self.test_slow_tokenizer:
                   tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

            if self.test_rust_tokenizer:
                   tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

            for tokenizer_class, tokenizer_utils in tokenizer_list:
                   with tempfile.TemporaryDirectory() as tmp_dir:
                          tokenizer_utils.save_pretrained(lowerCAmelCase__       )

                          _A=	tokenizer_class.from_pretrained(lowerCAmelCase__       )

                          self.assertTrue(tokenizer.decode([255]       ) == ''       )



     def 			a__  (					self       ):
            pass



     def 			a__  (					self       ):
            pass



     def 			a__  (					self       ):
            pass



     def 			a__  (					self       ):
            pass



     def 			a__  (					self       ):
            # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
            # and special added tokens as tokens
            _A=	self.get_tokenizers(fast=lowerCAmelCase__				,      do_lower_case=lowerCAmelCase__       )
            for tokenizer in tokenizers:
                   with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                          _A=	['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                          _A=	tokenizer.convert_tokens_to_string(lowerCAmelCase__       )

                          self.assertIsInstance(lowerCAmelCase__				,      lowerCAmelCase__       )





     def 			a__  (					self       ):
            _A=	self.get_tokenizers()
            for tokenizer in tokenizers:
                   with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                          _A=	[
                              'bos_token',
                              'eos_token',
                              'unk_token',
                              'sep_token',
                              'pad_token',
                              'cls_token',
                              'mask_token',
                          ]

                          _A=	0
                          _A=	tokenizer.convert_ids_to_tokens(
                              lowerCAmelCase__				,      skip_special_tokens=lowerCAmelCase__       )

                          for attr in attributes_list:
                                 setattr(lowerCAmelCase__				,      attr + '_id'				,      lowerCAmelCase__       )
                                 self.assertEqual(getattr(lowerCAmelCase__				,      lowerCAmelCase__       )				,      lowerCAmelCase__       )
                                 self.assertEqual(getattr(lowerCAmelCase__				,      attr + '_id'       )				,      lowerCAmelCase__       )

                                 setattr(lowerCAmelCase__				,      attr + '_id'				,      lowerCAmelCase__       )
                                 self.assertEqual(getattr(lowerCAmelCase__				,      lowerCAmelCase__       )				,      lowerCAmelCase__       )
                                 self.assertEqual(getattr(lowerCAmelCase__				,      attr + '_id'       )				,      lowerCAmelCase__       )

                          setattr(lowerCAmelCase__				,      'additional_special_tokens_ids'				,      []       )
                          self.assertListEqual(getattr(lowerCAmelCase__				,      'additional_special_tokens'       )				,      []       )
                          self.assertListEqual(getattr(lowerCAmelCase__				,      'additional_special_tokens_ids'       )				,      []       )

                          setattr(lowerCAmelCase__				,      'additional_special_tokens_ids'				,      [token_id_to_test_setters]       )
                          self.assertListEqual(getattr(lowerCAmelCase__				,      'additional_special_tokens'       )				,      [token_to_test_setters]       )
                          self.assertListEqual(getattr(lowerCAmelCase__				,      'additional_special_tokens_ids'       )				,      [token_id_to_test_setters]       )