





import gc
import unittest

import torch
from parameterized import parameterized

from diffusers import AutoencoderKL
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()




class   UpperCamelCase_       (     UpperCamelCase__				,  UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					AutoencoderKL
							lowerCamelCase_           =					"sample"
							lowerCamelCase_           =					1e-2


							@property
							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      4
									SCREAMING_SNAKE_CASE__			=      3
									SCREAMING_SNAKE_CASE__			=      (32, 32)

									SCREAMING_SNAKE_CASE__			=      floats_tensor((batch_size, num_channels) + sizes		).to(__A		)

									return {"sample": image}


							@property
							def      _snake_case  (							self    :Any		)   ->   List[Any]:



									"""simple docstring"""



									return (3, 32, 32)


							@property
							def      _snake_case  (							self    :Optional[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									return (3, 32, 32)


							def      _snake_case  (							self    :str		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      {
									    """block_out_channels""": [32, 64],
									    """in_channels""": 3,
									    """out_channels""": 3,
									    """down_block_types""": ["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],
									    """up_block_types""": ["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],
									    """latent_channels""": 4,
									}
									SCREAMING_SNAKE_CASE__			=      self.dummy_input
									return init_dict, inputs_dict


							def      _snake_case  (							self    :List[Any]		)   ->   Optional[int]:



									"""simple docstring"""



									pass


							def      _snake_case  (							self    :Optional[Any]		)   ->   List[Any]:



									"""simple docstring"""



									pass


							@unittest.skipIf(torch_device == """mps"""							,				"""Gradient checkpointing skipped on MPS"""		)
							def      _snake_case  (							self    :Any		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.prepare_init_args_and_inputs_for_common()
									SCREAMING_SNAKE_CASE__			=      self.model_class(**__A		)
									model.to(__A		)

									assert not model.is_gradient_checkpointing and model.training

									SCREAMING_SNAKE_CASE__			=      model(**__A		).sample
									# run the backwards pass on the model. For backwards pass, for simplicity purpose,
									# we won't calculate the loss and rather backprop on out.sum()
									model.zero_grad()

									SCREAMING_SNAKE_CASE__			=      torch.randn_like(__A		)
									SCREAMING_SNAKE_CASE__			=      (out - labels).mean()
									loss.backward()

									# re-instantiate the model now enabling gradient checkpointing
									SCREAMING_SNAKE_CASE__			=      self.model_class(**__A		)
									# clone model
									model_a.load_state_dict(model.state_dict()		)
									model_a.to(__A		)
									model_a.enable_gradient_checkpointing()

									assert model_a.is_gradient_checkpointing and model_a.training

									SCREAMING_SNAKE_CASE__			=      model_a(**__A		).sample
									# run the backwards pass on the model. For backwards pass, for simplicity purpose,
									# we won't calculate the loss and rather backprop on out.sum()
									model_a.zero_grad()
									SCREAMING_SNAKE_CASE__			=      (out_a - labels).mean()
									loss_a.backward()

									# compare the output and parameters gradients
									self.assertTrue((loss - loss_a).abs() < 1E-5		)
									SCREAMING_SNAKE_CASE__			=      dict(model.named_parameters()		)
									SCREAMING_SNAKE_CASE__			=      dict(model_a.named_parameters()		)
									for name, param in named_params.items():
											self.assertTrue(torch_all_close(param.grad.data							,				named_params_a[name].grad.data							,				atol=5E-5		)		)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      AutoencoderKL.from_pretrained("""fusing/autoencoder-kl-dummy"""							,				output_loading_info=__A		)
									self.assertIsNotNone(__A		)
									self.assertEqual(len(loading_info["""missing_keys"""]		)							,				0		)

									model.to(__A		)
									SCREAMING_SNAKE_CASE__			=      model(**self.dummy_input		)

									assert image is not None, "Make sure output is not None"


							def      _snake_case  (							self    :List[Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      AutoencoderKL.from_pretrained("""fusing/autoencoder-kl-dummy"""		)
									SCREAMING_SNAKE_CASE__			=      model.to(__A		)
									model.eval()

									if torch_device == "mps":
											SCREAMING_SNAKE_CASE__			=      torch.manual_seed(0		)
									else:
											SCREAMING_SNAKE_CASE__			=      torch.Generator(device=__A		).manual_seed(0		)

									SCREAMING_SNAKE_CASE__			=      torch.randn(
									    1							,				model.config.in_channels							,				model.config.sample_size							,				model.config.sample_size							,				generator=torch.manual_seed(0		)							,				)
									SCREAMING_SNAKE_CASE__			=      image.to(__A		)
									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model(__A							,				sample_posterior=__A							,				generator=__A		).sample

									SCREAMING_SNAKE_CASE__			=      output[0, -1, -3:, -3:].flatten().cpu()

									# Since the VAE Gaussian prior's generator is seeded on the appropriate device,
									# the expected output slices are not the same for CPU and GPU.
									if torch_device == "mps":
											SCREAMING_SNAKE_CASE__			=      torch.tensor(
											    [
											        -4.0_078E-01,
											        -3.8_323E-04,
											        -1.2_681E-01,
											        -1.1_462E-01,
											        2.0_095E-01,
											        1.0_893E-01,
											        -8.8_247E-02,
											        -3.0_361E-01,
											        -9.8_644E-03,
											    ]		)
									elif torch_device == "cpu":
											SCREAMING_SNAKE_CASE__			=      torch.tensor(
											    [-0.1_3_5_2, 0.0_8_7_8, 0.0_4_1_9, -0.0_8_1_8, -0.1_0_6_9, 0.0_6_8_8, -0.1_4_5_8, -0.4_4_4_6, -0.0_0_2_6]		)
									else:
											SCREAMING_SNAKE_CASE__			=      torch.tensor(
											    [-0.2_4_2_1, 0.4_6_4_2, 0.2_5_0_7, -0.0_4_3_8, 0.0_6_8_2, 0.3_1_6_0, -0.2_0_1_8, -0.0_7_2_7, 0.2_4_8_5]		)

									self.assertTrue(torch_all_close(__A							,				__A							,				rtol=1E-2		)		)




@slow
class   UpperCamelCase_       (     unittest.TestCase    ):


							def      _snake_case  (							self    :Optional[int]							,				__A    :int							,				__A    :Optional[Any]		)   ->   Tuple:



									"""simple docstring"""



									return f'''gaussian_noise_s={seed}_shape={'_'.join([str(__A		) for s in shape]		)}.npy'''


							def      _snake_case  (							self    :List[str]		)   ->   int:



									"""simple docstring"""



									super().tearDown()
									gc.collect()
									torch.cuda.empty_cache()


							def      _snake_case  (							self    :int							,				__A    :Any=0							,				__A    :List[str]=(4, 3, 512, 512)							,				__A    :int=False		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      torch.floataa if fpaa else torch.floataa
									SCREAMING_SNAKE_CASE__			=      torch.from_numpy(load_hf_numpy(self.get_file_format(__A							,				__A		)		)		).to(__A		).to(__A		)
									return image


							def      _snake_case  (							self    :Tuple							,				__A    :int="CompVis/stable-diffusion-v1-4"							,				__A    :Any=False		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      """fp16""" if fpaa else None
									SCREAMING_SNAKE_CASE__			=      torch.floataa if fpaa else torch.floataa

									SCREAMING_SNAKE_CASE__			=      AutoencoderKL.from_pretrained(
									    __A							,				subfolder="""vae"""							,				torch_dtype=__A							,				revision=__A							,				)
									model.to(__A		).eval()

									return model


							def      _snake_case  (							self    :Optional[int]							,				__A    :Optional[Any]=0		)   ->   Optional[int]:



									"""simple docstring"""



									if torch_device == "mps":
											return torch.manual_seed(__A		)
									return torch.Generator(device=__A		).manual_seed(__A		)


							@parameterized.expand(
							    [
							        # fmt: off
							        [33, [-0.1_6_0_3, 0.9_8_7_8, -0.0_4_9_5, -0.0_7_9_0, -0.2_7_0_9, 0.8_3_7_5, -0.2_0_6_0, -0.0_8_2_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
							        [47, [-0.2_3_7_6, 0.1_1_6_8, 0.1_3_3_2, -0.4_8_4_0, -0.2_5_0_8, -0.0_7_9_1, -0.0_4_9_3, -0.4_0_8_9], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
							        # fmt: on
							    ]		)
							def      _snake_case  (							self    :List[Any]							,				__A    :int							,				__A    :str							,				__A    :Optional[int]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model()
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A		)
									SCREAMING_SNAKE_CASE__			=      self.get_generator(__A		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model(__A							,				generator=__A							,				sample_posterior=__A		).sample

									assert sample.shape == image.shape

									SCREAMING_SNAKE_CASE__			=      sample[-1, -2:, -2:, :2].flatten().float().cpu()
									SCREAMING_SNAKE_CASE__			=      torch.tensor(expected_slice_mps if torch_device == """mps""" else expected_slice		)

									assert torch_all_close(__A							,				__A							,				atol=3E-3		)


							@parameterized.expand(
							    [
							        # fmt: off
							        [33, [-0.0_5_1_3, 0.0_2_8_9, 1.3_7_9_9, 0.2_1_6_6, -0.2_5_7_3, -0.0_8_7_1, 0.5_1_0_3, -0.0_9_9_9]],
							        [47, [-0.4_1_2_8, -0.1_3_2_0, -0.3_7_0_4, 0.1_9_6_5, -0.4_1_1_6, -0.2_3_3_2, -0.3_3_4_0, 0.2_2_4_7]],
							        # fmt: on
							    ]		)
							@require_torch_gpu
							def      _snake_case  (							self    :str							,				__A    :Optional[Any]							,				__A    :Any		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model(fpaa=__A		)
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A							,				fpaa=__A		)
									SCREAMING_SNAKE_CASE__			=      self.get_generator(__A		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model(__A							,				generator=__A							,				sample_posterior=__A		).sample

									assert sample.shape == image.shape

									SCREAMING_SNAKE_CASE__			=      sample[-1, -2:, :2, -2:].flatten().float().cpu()
									SCREAMING_SNAKE_CASE__			=      torch.tensor(__A		)

									assert torch_all_close(__A							,				__A							,				atol=1E-2		)


							@parameterized.expand(
							    [
							        # fmt: off
							        [33, [-0.1_6_0_9, 0.9_8_6_6, -0.0_4_8_7, -0.0_7_7_7, -0.2_7_1_6, 0.8_3_6_8, -0.2_0_5_5, -0.0_8_1_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
							        [47, [-0.2_3_7_7, 0.1_1_4_7, 0.1_3_3_3, -0.4_8_4_1, -0.2_5_0_6, -0.0_8_0_5, -0.0_4_9_1, -0.4_0_8_5], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
							        # fmt: on
							    ]		)
							def      _snake_case  (							self    :Any							,				__A    :Tuple							,				__A    :List[str]							,				__A    :str		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model()
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model(__A		).sample

									assert sample.shape == image.shape

									SCREAMING_SNAKE_CASE__			=      sample[-1, -2:, -2:, :2].flatten().float().cpu()
									SCREAMING_SNAKE_CASE__			=      torch.tensor(expected_slice_mps if torch_device == """mps""" else expected_slice		)

									assert torch_all_close(__A							,				__A							,				atol=3E-3		)


							@parameterized.expand(
							    [
							        # fmt: off
							        [13, [-0.2_0_5_1, -0.1_8_0_3, -0.2_3_1_1, -0.2_1_1_4, -0.3_2_9_2, -0.3_5_7_4, -0.2_9_5_3, -0.3_3_2_3]],
							        [37, [-0.2_6_3_2, -0.2_6_2_5, -0.2_1_9_9, -0.2_7_4_1, -0.4_5_3_9, -0.4_9_9_0, -0.3_7_2_0, -0.4_9_2_5]],
							        # fmt: on
							    ]		)
							@require_torch_gpu
							def      _snake_case  (							self    :Dict							,				__A    :Any							,				__A    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model()
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A							,				shape=(3, 4, 64, 64)		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.decode(__A		).sample

									assert list(sample.shape		) == [3, 3, 512, 512]

									SCREAMING_SNAKE_CASE__			=      sample[-1, -2:, :2, -2:].flatten().cpu()
									SCREAMING_SNAKE_CASE__			=      torch.tensor(__A		)

									assert torch_all_close(__A							,				__A							,				atol=1E-3		)


							@parameterized.expand(
							    [
							        # fmt: off
							        [27, [-0.0_3_6_9, 0.0_2_0_7, -0.0_7_7_6, -0.0_6_8_2, -0.1_7_4_7, -0.1_9_3_0, -0.1_4_6_5, -0.2_0_3_9]],
							        [16, [-0.1_6_2_8, -0.2_1_3_4, -0.2_7_4_7, -0.2_6_4_2, -0.3_7_7_4, -0.4_4_0_4, -0.3_6_8_7, -0.4_2_7_7]],
							        # fmt: on
							    ]		)
							@require_torch_gpu
							def      _snake_case  (							self    :int							,				__A    :Any							,				__A    :Optional[int]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model(fpaa=__A		)
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A							,				shape=(3, 4, 64, 64)							,				fpaa=__A		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.decode(__A		).sample

									assert list(sample.shape		) == [3, 3, 512, 512]

									SCREAMING_SNAKE_CASE__			=      sample[-1, -2:, :2, -2:].flatten().float().cpu()
									SCREAMING_SNAKE_CASE__			=      torch.tensor(__A		)

									assert torch_all_close(__A							,				__A							,				atol=5E-3		)


							@parameterized.expand([(13,), (16,), (27,)]		)
							@require_torch_gpu
							@unittest.skipIf(not is_xformers_available()							,				reason="""xformers is not required when using PyTorch 2.0."""		)
							def      _snake_case  (							self    :Optional[int]							,				__A    :Any		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model(fpaa=__A		)
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A							,				shape=(3, 4, 64, 64)							,				fpaa=__A		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.decode(__A		).sample

									model.enable_xformers_memory_efficient_attention()
									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.decode(__A		).sample

									assert list(sample.shape		) == [3, 3, 512, 512]

									assert torch_all_close(__A							,				__A							,				atol=1E-1		)


							@parameterized.expand([(13,), (16,), (37,)]		)
							@require_torch_gpu
							@unittest.skipIf(not is_xformers_available()							,				reason="""xformers is not required when using PyTorch 2.0."""		)
							def      _snake_case  (							self    :str							,				__A    :int		)   ->   Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model()
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A							,				shape=(3, 4, 64, 64)		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.decode(__A		).sample

									model.enable_xformers_memory_efficient_attention()
									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.decode(__A		).sample

									assert list(sample.shape		) == [3, 3, 512, 512]

									assert torch_all_close(__A							,				__A							,				atol=1E-2		)


							@parameterized.expand(
							    [
							        # fmt: off
							        [33, [-0.3_0_0_1, 0.0_9_1_8, -2.6_9_8_4, -3.9_7_2_0, -3.2_0_9_9, -5.0_3_5_3, 1.7_3_3_8, -0.2_0_6_5, 3.4_2_6_7]],
							        [47, [-1.5_0_3_0, -4.3_8_7_1, -6.0_3_5_5, -9.1_1_5_7, -1.6_6_6_1, -2.7_8_5_3, 2.1_6_0_7, -5.0_8_2_3, 2.5_6_3_3]],
							        # fmt: on
							    ]		)
							def      _snake_case  (							self    :Tuple							,				__A    :Union[str, Any]							,				__A    :Any		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_sd_vae_model()
									SCREAMING_SNAKE_CASE__			=      self.get_sd_image(__A		)
									SCREAMING_SNAKE_CASE__			=      self.get_generator(__A		)

									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model.encode(__A		).latent_dist
											SCREAMING_SNAKE_CASE__			=      dist.sample(generator=__A		)

									assert list(sample.shape		) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]]

									SCREAMING_SNAKE_CASE__			=      sample[0, -1, -3:, -3:].flatten().cpu()
									SCREAMING_SNAKE_CASE__			=      torch.tensor(__A		)

									SCREAMING_SNAKE_CASE__			=      3E-3 if torch_device != """mps""" else 1E-2
									assert torch_all_close(__A							,				__A							,				atol=__A		)






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					RoCBertTokenizer
							lowerCamelCase_           =					None
							lowerCamelCase_           =					False
							lowerCamelCase_           =					True
							lowerCamelCase_           =					filter_non_english


							def      _snake_case  (							self    :List[Any]		)   ->   List[Any]:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									for i, value in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_shape_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_pronunciation_file"""]		)
									with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
									with open(self.word_shape_file							,				"""w"""							,				encoding="""utf-8"""		) as word_shape_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)
									with open(self.word_pronunciation_file							,				"""w"""							,				encoding="""utf-8"""		) as word_pronunciation_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""你好[SEP]你是谁"""		)
									self.assertListEqual(__A							,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)


							def      _snake_case  (							self    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""		)							,				["""ah""", """\u535A""", """\u63A8""", """zz"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""hello""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""h\u00E9llo"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :int		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				never_split=["""[UNK]"""]		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									SCREAMING_SNAKE_CASE__			=      {}
									for i, token in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      RoCBertWordpieceTokenizer(vocab=__A							,				unk_token="""[UNK]"""		)

									self.assertListEqual(tokenizer.tokenize(""""""		)							,				[]		)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""		)							,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]		)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""		)							,				["""[UNK]""", """runn""", """##ing"""]		)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_whitespace(""" """		)		)
									self.assertTrue(_is_whitespace("""\t"""		)		)
									self.assertTrue(_is_whitespace("""\r"""		)		)
									self.assertTrue(_is_whitespace("""\n"""		)		)
									self.assertTrue(_is_whitespace("""\u00A0"""		)		)

									self.assertFalse(_is_whitespace("""A"""		)		)
									self.assertFalse(_is_whitespace("""-"""		)		)


							def      _snake_case  (							self    :int		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_control("""\u0005"""		)		)

									self.assertFalse(_is_control("""A"""		)		)
									self.assertFalse(_is_control(""" """		)		)
									self.assertFalse(_is_control("""\t"""		)		)
									self.assertFalse(_is_control("""\r"""		)		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									self.assertTrue(_is_punctuation("""-"""		)		)
									self.assertTrue(_is_punctuation("""$"""		)		)
									self.assertTrue(_is_punctuation("""`"""		)		)
									self.assertTrue(_is_punctuation("""."""		)		)

									self.assertFalse(_is_punctuation("""A"""		)		)
									self.assertFalse(_is_punctuation(""" """		)		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)

									if self.test_rust_tokenizer:
											SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)


							def      _snake_case  (							self    :int		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode_plus(
													    __A							,				return_attention_mask=__A							,				return_token_type_ids=__A							,				return_offsets_mapping=__A							,				add_special_tokens=__A							,				)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.do_lower_case if hasattr(__A							,				"""do_lower_case"""		) else False
													SCREAMING_SNAKE_CASE__			=      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """A"""),
													        ((1, 2), ""","""),
													        ((3, 5), """na"""),
													        ((5, 6), """##ï"""),
													        ((6, 8), """##ve"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """Allen"""),
													        ((21, 23), """##NL"""),
													        ((23, 24), """##P"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """a"""),
													        ((1, 2), ""","""),
													        ((3, 8), """naive"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """allen"""),
													        ((21, 23), """##nl"""),
													        ((23, 24), """##p"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]		)		)
													self.assertEqual([e[0] for e in expected_results]							,				tokens["""offset_mapping"""]		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""的""", """人""", """有"""]
									SCREAMING_SNAKE_CASE__			=      """""".join(__A		)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      True
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)

													SCREAMING_SNAKE_CASE__			=      False
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE__			=      [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__A		)
													]
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)


							@slow
							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你好"""							,				add_special_tokens=__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你是谁"""							,				add_special_tokens=__A		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A							,				__A		)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizers(do_lower_case=__A		)
									for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
													SCREAMING_SNAKE_CASE__			=      """你好，你是谁"""
													SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_shape_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_pronunciation_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.prepare_for_model(
													    __A							,				__A							,				__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer.encode_plus(__A							,				add_special_tokens=__A		)

													self.assertEqual(__A							,				__A		)