


import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionPanoramaPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, skip_mps

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()







@skip_mps
class 							_UpperCamelCase (						__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,unittest.TestCase						):

   '''simple docstring'''







   a_				:							Optional[int]    = StableDiffusionPanoramaPipeline
   a_				:							List[str]    = TEXT_TO_IMAGE_PARAMS
   a_				:							Dict    = TEXT_TO_IMAGE_BATCH_PARAMS
   a_				:							Any    = TEXT_TO_IMAGE_IMAGE_PARAMS
   a_				:							str    = TEXT_TO_IMAGE_IMAGE_PARAMS

   def   _snake_case    (  self  : Tuple							):






         '''simple docstring'''



         torch.manual_seed(0							)
         __lowerCamelCase     :      Tuple       						=						UNetaDConditionModel(
             block_out_channels=(3_2, 6_4)       ,  layers_per_block=1       ,  sample_size=3_2       ,  in_channels=4       ,  out_channels=4       ,  down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")       ,  up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")       ,  cross_attention_dim=3_2       ,  )
         __lowerCamelCase     :      int       						=						DDIMScheduler()
         torch.manual_seed(0							)
         __lowerCamelCase     :      Optional[Any]       						=						AutoencoderKL(
             block_out_channels=[3_2, 6_4]       ,  in_channels=3       ,  out_channels=3       ,  down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]       ,  up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]       ,  latent_channels=4       ,  )
         torch.manual_seed(0							)
         __lowerCamelCase     :      List[Any]       						=						CLIPTextConfig(
             bos_token_id=0       ,  eos_token_id=2       ,  hidden_size=3_2       ,  intermediate_size=3_7       ,  layer_norm_eps=1E-05       ,  num_attention_heads=4       ,  num_hidden_layers=5       ,  pad_token_id=1       ,  vocab_size=1_0_0_0       ,  )
         __lowerCamelCase     :      List[Any]       						=						CLIPTextModel(_a							)
         __lowerCamelCase     :      Any       						=						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""							)

         __lowerCamelCase     :      str       						=						{
             """unet""": unet,
             """scheduler""": scheduler,
             """vae""": vae,
             """text_encoder""": text_encoder,
             """tokenizer""": tokenizer,
             """safety_checker""": None,
             """feature_extractor""": None,
         }
         return components

   def   _snake_case    (  self  : Optional[int]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : str=0							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[int]       						=						torch.manual_seed(_a							)
         __lowerCamelCase     :      List[str]       						=						{
             """prompt""": """a photo of the dolomites""",
             """generator""": generator,
             # Setting height and width to None to prevent OOMs on CPU.
             """height""": None,
             """width""": None,
             """num_inference_steps""": 1,
             """guidance_scale""": 6.0,
             """output_type""": """numpy""",
         }
         return inputs

   def   _snake_case    (  self  : Tuple							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
         __lowerCamelCase     :      List[Any]       						=						self.get_dummy_components()
         __lowerCamelCase     :      List[str]       						=						StableDiffusionPanoramaPipeline(**_a							)
         __lowerCamelCase     :      Dict       						=						sd_pipe.to(_a							)
         sd_pipe.set_progress_bar_config(disable=_a							)

         __lowerCamelCase     :      Optional[Any]       						=						self.get_dummy_inputs(_a							)
         __lowerCamelCase     :      str       						=						sd_pipe(**_a							).images
         __lowerCamelCase     :      int       						=						image[0, -3:, -3:, -1]
         assert image.shape == (1, 6_4, 6_4, 3)

         __lowerCamelCase     :      str       						=						np.array([0.6_186, 0.5_374, 0.4_915, 0.4_135, 0.4_114, 0.4_563, 0.5_128, 0.4_977, 0.4_757]							)

         assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2

   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         super().test_inference_batch_consistent(batch_sizes=[1, 2]							)

   def   _snake_case    (  self  : Any							):






         '''simple docstring'''



         super().test_inference_batch_single_identical(batch_size=2       ,  expected_max_diff=3.25E-3							)

   def   _snake_case    (  self  : Dict							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
         __lowerCamelCase     :      List[Any]       						=						self.get_dummy_components()
         __lowerCamelCase     :      List[Any]       						=						StableDiffusionPanoramaPipeline(**_a							)
         __lowerCamelCase     :      Dict       						=						sd_pipe.to(_a							)
         sd_pipe.set_progress_bar_config(disable=_a							)

         __lowerCamelCase     :      Dict       						=						self.get_dummy_inputs(_a							)
         __lowerCamelCase     :      Union[str, Any]       						=						"""french fries"""
         __lowerCamelCase     :      Tuple       						=						sd_pipe(**_a       ,  negative_prompt=_a							)
         __lowerCamelCase     :      Optional[int]       						=						output.images
         __lowerCamelCase     :      int       						=						image[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)

         __lowerCamelCase     :      Any       						=						np.array([0.6_187, 0.5_375, 0.4_915, 0.4_136, 0.4_114, 0.4_563, 0.5_128, 0.4_976, 0.4_757]							)

         assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2

   def   _snake_case    (  self  : List[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      int       						=						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
         __lowerCamelCase     :      Any       						=						self.get_dummy_components()
         __lowerCamelCase     :      Tuple       						=						StableDiffusionPanoramaPipeline(**_a							)
         __lowerCamelCase     :      List[Any]       						=						sd_pipe.to(_a							)
         sd_pipe.set_progress_bar_config(disable=_a							)

         __lowerCamelCase     :      Optional[Any]       						=						self.get_dummy_inputs(_a							)
         __lowerCamelCase     :      List[str]       						=						sd_pipe(**_a       ,  view_batch_size=2							)
         __lowerCamelCase     :      Optional[int]       						=						output.images
         __lowerCamelCase     :      List[Any]       						=						image[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)

         __lowerCamelCase     :      Optional[int]       						=						np.array([0.6_187, 0.5_375, 0.4_915, 0.4_136, 0.4_114, 0.4_563, 0.5_128, 0.4_976, 0.4_757]							)

         assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2

   def   _snake_case    (  self  : Optional[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      str       						=						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
         __lowerCamelCase     :      Optional[Any]       						=						self.get_dummy_components()
         __lowerCamelCase     :      Tuple       						=						EulerAncestralDiscreteScheduler(
             beta_start=0.00_085       ,  beta_end=0.012       ,  beta_schedule="""scaled_linear"""							)
         __lowerCamelCase     :      int       						=						StableDiffusionPanoramaPipeline(**_a							)
         __lowerCamelCase     :      List[str]       						=						sd_pipe.to(_a							)
         sd_pipe.set_progress_bar_config(disable=_a							)

         __lowerCamelCase     :      Tuple       						=						self.get_dummy_inputs(_a							)
         __lowerCamelCase     :      List[str]       						=						sd_pipe(**_a							).images
         __lowerCamelCase     :      List[Any]       						=						image[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)

         __lowerCamelCase     :      Dict       						=						np.array([0.4_024, 0.6_510, 0.4_901, 0.5_378, 0.5_813, 0.5_622, 0.4_795, 0.4_467, 0.4_952]							)

         assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


   def   _snake_case    (  self  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[int]       						=						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
         __lowerCamelCase     :      List[str]       						=						self.get_dummy_components()
         __lowerCamelCase     :      List[Any]       						=						PNDMScheduler(
             beta_start=0.00_085       ,  beta_end=0.012       ,  beta_schedule="""scaled_linear"""       ,  skip_prk_steps=_a							)
         __lowerCamelCase     :      List[str]       						=						StableDiffusionPanoramaPipeline(**_a							)
         __lowerCamelCase     :      Optional[int]       						=						sd_pipe.to(_a							)
         sd_pipe.set_progress_bar_config(disable=_a							)

         __lowerCamelCase     :      str       						=						self.get_dummy_inputs(_a							)
         __lowerCamelCase     :      Optional[int]       						=						sd_pipe(**_a							).images
         __lowerCamelCase     :      List[Any]       						=						image[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)

         __lowerCamelCase     :      Any       						=						np.array([0.6_391, 0.6_291, 0.4_861, 0.5_134, 0.5_552, 0.4_578, 0.5_032, 0.5_023, 0.4_539]							)

         assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2







@slow
@require_torch_gpu
class 							_UpperCamelCase (						unittest.TestCase						):

   '''simple docstring'''







   def   _snake_case    (  self  : Dict							):






         '''simple docstring'''



         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()

   def   _snake_case    (  self  : Any       ,  _lowerCamelCase  : str=0							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						torch.manual_seed(_a							)
         __lowerCamelCase     :      Optional[int]       						=						{
             """prompt""": """a photo of the dolomites""",
             """generator""": generator,
             """num_inference_steps""": 3,
             """guidance_scale""": 7.5,
             """output_type""": """numpy""",
         }
         return inputs

   def   _snake_case    (  self  : Optional[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						"""stabilityai/stable-diffusion-2-base"""
         __lowerCamelCase     :      int       						=						DDIMScheduler.from_pretrained(_a       ,  subfolder="""scheduler"""							)
         __lowerCamelCase     :      int       						=						StableDiffusionPanoramaPipeline.from_pretrained(_a       ,  scheduler=_a       ,  safety_checker=_a							)
         pipe.to(_a							)
         pipe.set_progress_bar_config(disable=_a							)
         pipe.enable_attention_slicing()

         __lowerCamelCase     :      Any       						=						self.get_inputs()
         __lowerCamelCase     :      Optional[int]       						=						pipe(**_a							).images
         __lowerCamelCase     :      Optional[Any]       						=						image[0, -3:, -3:, -1].flatten()

         assert image.shape == (1, 5_1_2, 2_0_4_8, 3)

         __lowerCamelCase     :      Any       						=						np.array(
             [
                 0.36_968_392,
                 0.27_025_372,
                 0.32_446_766,
                 0.28_379_387,
                 0.36_363_274,
                 0.30_733_347,
                 0.27_100_027,
                 0.27_054_125,
                 0.25_536_096,
             ]							)

         assert np.abs(expected_slice - image_slice							).max() < 1E-2

   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						StableDiffusionPanoramaPipeline.from_pretrained(
             """stabilityai/stable-diffusion-2-base"""       ,  safety_checker=_a							)
         __lowerCamelCase     :      int       						=						LMSDiscreteScheduler.from_config(pipe.scheduler.config							)
         pipe.to(_a							)
         pipe.set_progress_bar_config(disable=_a							)
         pipe.enable_attention_slicing()

         __lowerCamelCase     :      Dict       						=						self.get_inputs()
         __lowerCamelCase     :      Any       						=						pipe(**_a							).images
         __lowerCamelCase     :      int       						=						image[0, -3:, -3:, -1].flatten()

         assert image.shape == (1, 5_1_2, 2_0_4_8, 3)

         __lowerCamelCase     :      Union[str, Any]       						=						np.array(
             [
                 [
                     0.0,
                     0.0,
                     0.0,
                     0.0,
                     0.0,
                     0.0,
                     0.0,
                     0.0,
                     0.0,
                 ]
             ]							)

         assert np.abs(expected_slice - image_slice							).max() < 1E-3

   def   _snake_case    (  self  : str							):






         '''simple docstring'''



         __lowerCamelCase     :      str       						=						0

         def callback_fn(_lowerCamelCase  : List[Any]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Optional[int]							) -> None:
               __lowerCamelCase     :      List[Any]       						=						True
               nonlocal number_of_steps
               number_of_steps += 1
               if step == 1:
                     __lowerCamelCase     :      Dict       						=						latents.detach().cpu().numpy()
                     assert latents.shape == (1, 4, 6_4, 2_5_6)
                     __lowerCamelCase     :      Any       						=						latents[0, -3:, -3:, -1]

                     __lowerCamelCase     :      Any       						=						np.array(
                         [
                             0.18_681_869,
                             0.33_907_816,
                             0.5_361_276,
                             0.14_432_865,
                             -0.02_856_611,
                             -0.73_941_123,
                             0.23_397_987,
                             0.47_322_682,
                             -0.37_823_164,
                         ]							)
                     assert np.abs(latents_slice.flatten() - expected_slice							).max() < 5E-2
               elif step == 2:
                     __lowerCamelCase     :      int       						=						latents.detach().cpu().numpy()
                     assert latents.shape == (1, 4, 6_4, 2_5_6)
                     __lowerCamelCase     :      List[Any]       						=						latents[0, -3:, -3:, -1]

                     __lowerCamelCase     :      Optional[int]       						=						np.array(
                         [
                             0.18_539_645,
                             0.33_987_248,
                             0.5_378_559,
                             0.14_437_142,
                             -0.02_455_261,
                             -0.7_338_317,
                             0.23_990_755,
                             0.47_356_272,
                             -0.3_786_505,
                         ]							)

                     assert np.abs(latents_slice.flatten() - expected_slice							).max() < 5E-2

         __lowerCamelCase     :      Optional[Any]       						=						False

         __lowerCamelCase     :      Tuple       						=						"""stabilityai/stable-diffusion-2-base"""
         __lowerCamelCase     :      List[str]       						=						DDIMScheduler.from_pretrained(_a       ,  subfolder="""scheduler"""							)
         __lowerCamelCase     :      Optional[int]       						=						StableDiffusionPanoramaPipeline.from_pretrained(_a       ,  scheduler=_a       ,  safety_checker=_a							)
         __lowerCamelCase     :      Tuple       						=						pipe.to(_a							)
         pipe.set_progress_bar_config(disable=_a							)
         pipe.enable_attention_slicing()

         __lowerCamelCase     :      Optional[Any]       						=						self.get_inputs()
         pipe(**_a       ,  callback=_a       ,  callback_steps=1							)
         assert callback_fn.has_been_called
         assert number_of_steps == 3


   def   _snake_case    (  self  : Optional[int]							):






         '''simple docstring'''



         torch.cuda.empty_cache()
         torch.cuda.reset_max_memory_allocated()
         torch.cuda.reset_peak_memory_stats()

         __lowerCamelCase     :      Any       						=						"""stabilityai/stable-diffusion-2-base"""
         __lowerCamelCase     :      Any       						=						DDIMScheduler.from_pretrained(_a       ,  subfolder="""scheduler"""							)
         __lowerCamelCase     :      List[str]       						=						StableDiffusionPanoramaPipeline.from_pretrained(_a       ,  scheduler=_a       ,  safety_checker=_a							)
         __lowerCamelCase     :      List[Any]       						=						pipe.to(_a							)
         pipe.set_progress_bar_config(disable=_a							)
         pipe.enable_attention_slicing(1							)
         pipe.enable_sequential_cpu_offload()

         __lowerCamelCase     :      Dict       						=						self.get_inputs()
         __lowerCamelCase     :      Optional[int]       						=						pipe(**_a							)

         __lowerCamelCase     :      int       						=						torch.cuda.max_memory_allocated()
         # make sure that less than 5.2 GB is allocated
         assert mem_bytes < 5.5 * 1_0**9







import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase    :						str     =							get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
@require_tokenizers
class 							_UpperCamelCase (						A,unittest.TestCase						):

   '''simple docstring'''







   a_				:							List[Any]    = ReformerTokenizer
   a_				:							Optional[int]    = ReformerTokenizerFast
   a_				:							Optional[int]    = True
   a_				:							List[Any]    = False
   a_				:							Any    = True

   def   _snake_case    (  self  : Optional[int]							):






         '''simple docstring'''



         super().setUp()

         __lowerCamelCase     :      str       						=						ReformerTokenizer(_lowerCamelCase       ,  keep_accents=_lowerCamelCase							)
         tokenizer.save_pretrained(self.tmpdirname							)

   def   _snake_case    (  self  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						"""<s>"""
         __lowerCamelCase     :      Any       						=						1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase							)       ,  _lowerCamelCase							)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase							)       ,  _lowerCamelCase							)

   def   _snake_case    (  self  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						list(self.get_tokenizer().get_vocab().keys()							)

         self.assertEqual(vocab_keys[0]       ,  """<unk>"""							)
         self.assertEqual(vocab_keys[1]       ,  """<s>"""							)
         self.assertEqual(vocab_keys[-1]       ,  """j"""							)
         self.assertEqual(len(_lowerCamelCase							)       ,  1_0_0_0							)

   def   _snake_case    (  self  : str							):






         '''simple docstring'''



         self.assertEqual(self.get_tokenizer().vocab_size       ,  1_0_0_0							)

   def   _snake_case    (  self  : Tuple							):






         '''simple docstring'''



         if not self.test_rust_tokenizer:
               return

         __lowerCamelCase     :      Dict       						=						self.get_tokenizer()
         __lowerCamelCase     :      Optional[int]       						=						self.get_rust_tokenizer()

         __lowerCamelCase     :      Optional[Any]       						=						"""I was born in 92000, and this is falsé."""

         __lowerCamelCase     :      Union[str, Any]       						=						tokenizer.tokenize(_lowerCamelCase							)
         __lowerCamelCase     :      Optional[int]       						=						rust_tokenizer.tokenize(_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase       ,  _lowerCamelCase							)

         __lowerCamelCase     :      str       						=						tokenizer.encode(_lowerCamelCase       ,  add_special_tokens=_lowerCamelCase							)
         __lowerCamelCase     :      Tuple       						=						rust_tokenizer.encode(_lowerCamelCase       ,  add_special_tokens=_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase       ,  _lowerCamelCase							)

         __lowerCamelCase     :      str       						=						self.get_rust_tokenizer()
         __lowerCamelCase     :      Dict       						=						tokenizer.encode(_lowerCamelCase							)
         __lowerCamelCase     :      Dict       						=						rust_tokenizer.encode(_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase       ,  _lowerCamelCase							)

   def   _snake_case    (  self  : Any       ,  _lowerCamelCase  : int=1_5							):






         '''simple docstring'''



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
                     __lowerCamelCase     :      Union[str, Any]       						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase       ,  **_lowerCamelCase							)

                     # Simple input
                     __lowerCamelCase     :      Any       						=						"""This is a simple input"""
                     __lowerCamelCase     :      Optional[Any]       						=						["""This is a simple input 1""", """This is a simple input 2"""]
                     __lowerCamelCase     :      Union[str, Any]       						=						("""This is a simple input""", """This is a pair""")
                     __lowerCamelCase     :      Optional[int]       						=						[
                         ("""This is a simple input 1""", """This is a simple input 2"""),
                         ("""This is a simple pair 1""", """This is a simple pair 2"""),
                     ]

                     # Simple input tests
                     self.assertRaises(_lowerCamelCase       ,  tokenizer_r.encode       ,  _lowerCamelCase       ,  max_length=_lowerCamelCase       ,  padding="""max_length"""							)

                     # Simple input
                     self.assertRaises(_lowerCamelCase       ,  tokenizer_r.encode_plus       ,  _lowerCamelCase       ,  max_length=_lowerCamelCase       ,  padding="""max_length"""							)

                     # Simple input
                     self.assertRaises(
                         _lowerCamelCase       ,  tokenizer_r.batch_encode_plus       ,  _lowerCamelCase       ,  max_length=_lowerCamelCase       ,  padding="""max_length"""       ,  )

                     # Pair input
                     self.assertRaises(_lowerCamelCase       ,  tokenizer_r.encode       ,  _lowerCamelCase       ,  max_length=_lowerCamelCase       ,  padding="""max_length"""							)

                     # Pair input
                     self.assertRaises(_lowerCamelCase       ,  tokenizer_r.encode_plus       ,  _lowerCamelCase       ,  max_length=_lowerCamelCase       ,  padding="""max_length"""							)

                     # Pair input
                     self.assertRaises(
                         _lowerCamelCase       ,  tokenizer_r.batch_encode_plus       ,  _lowerCamelCase       ,  max_length=_lowerCamelCase       ,  padding="""max_length"""       ,  )

   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         pass

   def   _snake_case    (  self  : Tuple							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						ReformerTokenizer(_lowerCamelCase       ,  keep_accents=_lowerCamelCase							)

         __lowerCamelCase     :      List[Any]       						=						tokenizer.tokenize("""This is a test"""							)
         self.assertListEqual(_lowerCamelCase       ,  ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(_lowerCamelCase							)       ,  [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]       ,  )

         __lowerCamelCase     :      List[Any]       						=						tokenizer.tokenize("""I was born in 92000, and this is falsé."""							)
         self.assertListEqual(
             _lowerCamelCase       ,  [
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ]       ,  )
         __lowerCamelCase     :      Union[str, Any]       						=						tokenizer.convert_tokens_to_ids(_lowerCamelCase							)
         self.assertListEqual(
             _lowerCamelCase       ,  [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]       ,  )

         __lowerCamelCase     :      List[str]       						=						tokenizer.convert_ids_to_tokens(_lowerCamelCase							)
         self.assertListEqual(
             _lowerCamelCase       ,  [
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ]       ,  )

   @cached_property
   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""							)

   @slow
   def   _snake_case    (  self  : str							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						"""Hello World!"""
         __lowerCamelCase     :      Any       						=						[1_2_6, 3_2, 2_6_2, 1_5_2, 3_8, 7_2, 2_8_7]

         self.assertListEqual(_lowerCamelCase       ,  self.big_tokenizer.encode(_lowerCamelCase							)							)

   @slow
   def   _snake_case    (  self  : Tuple							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						(
             """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
             """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
         )
         __lowerCamelCase     :      List[str]       						=						[
             1_0_8,
             2_6_5,
             2_4,
             1_1_1,
             4,
             2_5_8,
             1_5_6,
             3_5,
             2_8,
             2_7_5,
             3,
             2_5_9,
             2_9_7,
             2_6_0,
             8_4,
             4,
             3_5,
             1_1_0,
             4_4,
             8,
             2_5_9,
             9_1,
             2_6_8,
             2_1,
             1_1,
             2_0_9,
             2_7_4,
             1_0_9,
             2_6_6,
             2_7_7,
             1_1_7,
             8_6,
             9_3,
             3_1_5,
             2_5_8,
             2_7_8,
             2_5_8,
             2_7_7,
             2_5_8,
             0,
             2_5_8,
             2_8_8,
             2_5_8,
             3_1_9,
             2_5_8,
             0,
             2_5_8,
             0,
             2_5_8,
             0,
             2_5_8,
             0,
             2_5_8,
             2_8_7,
             2_5_8,
             3_1_5,
             2_5_8,
             2_8_9,
             2_5_8,
             2_7_8,
             9_9,
             2_6_9,
             2_6_6,
             2_6_2,
             8,
             2_5_9,
             2_4_1,
             4,
             2_1_7,
             2_3_0,
             2_6_8,
             2_6_6,
             5_5,
             1_6_8,
             1_0_6,
             7_5,
             1_9_3,
             2_6_6,
             2_2_3,
             2_7,
             4_9,
             2_6,
             2_8_2,
             2_5,
             2_6_4,
             2_9_9,
             1_9,
             2_6,
             0,
             2_5_8,
             2_7_7,
             1_1_7,
             8_6,
             9_3,
             1_7_6,
             1_8_3,
             2_7_0,
             1_1,
             2_6_2,
             4_2,
             6_1,
             2_6_5,
         ]

         self.assertListEqual(_lowerCamelCase       ,  self.big_tokenizer.encode(_lowerCamelCase							)							)

   @require_torch
   @slow
   def   _snake_case    (  self  : Optional[int]							):






         '''simple docstring'''



         import torch

         from transformers import ReformerConfig, ReformerModel

         # Build sequence
         __lowerCamelCase     :      int       						=						list(self.big_tokenizer.get_vocab().keys()							)[:1_0]
         __lowerCamelCase     :      Union[str, Any]       						=						""" """.join(_lowerCamelCase							)
         __lowerCamelCase     :      Union[str, Any]       						=						self.big_tokenizer.encode_plus(_lowerCamelCase       ,  return_tensors="""pt"""							)
         __lowerCamelCase     :      Union[str, Any]       						=						self.big_tokenizer.batch_encode_plus([sequence, sequence]       ,  return_tensors="""pt"""							)

         __lowerCamelCase     :      List[Any]       						=						ReformerConfig()
         # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
         __lowerCamelCase     :      List[str]       						=						encoded_sequence["""input_ids"""].shape
         __lowerCamelCase     :      Optional[Any]       						=						ReformerModel(_lowerCamelCase							)

         # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
               model(**_lowerCamelCase							)
               model(**_lowerCamelCase							)


   @slow
   def   _snake_case    (  self  : List[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Tuple       						=						{"""input_ids""": [[1_0_8, 2_6_5, 2_4, 1_1_1, 4, 2_5_8, 1_5_6, 7, 5_1, 2_7_9, 5_8, 7, 7_6, 2_5, 6_9, 2_7_8], [1_4_0, 2_4_3, 2_6_4, 1_3_4, 1_7, 2_6_7, 7_7, 2_6_3, 2_2, 2_6_2, 2_9_7, 2_5_8, 3_0_4, 1_7_7, 2_7_9, 2_6_6, 1_4, 8_9, 1_3, 3_5, 2_6_1, 2_9_9, 2_7_2, 1_3_7, 2_7_5, 2_7_8]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # This tokenizer does not know some characters like ")".
         # That is the reason why we use very simple texts here.
         # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
         __lowerCamelCase     :      int       						=						[
             """This is a very simple sentence.""",
             """The quick brown fox jumps over the lazy dog.""",
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=_lowerCamelCase       ,  model_name="""google/reformer-crime-and-punishment"""       ,  revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""       ,  padding=_lowerCamelCase       ,  sequences=_lowerCamelCase       ,  )



