






import gc
import unittest

import numpy as np
import torch
from torch.backends.cuda import sdp_kernel

from diffusers import (
    CMStochasticIterativeScheduler,
    ConsistencyModelPipeline,
    UNetaDModel,
)
from diffusers.utils import randn_tensor, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_a, require_torch_gpu

from ..pipeline_params import UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS, UNCONDITIONAL_IMAGE_GENERATION_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()


class 					_snake_case						(			_A       ,    unittest.TestCase ):
	_A          =   ConsistencyModelPipeline
	_A          =   UNCONDITIONAL_IMAGE_GENERATION_PARAMS
	_A          =   UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS

	# Override required_optional_params to remove num_images_per_prompt
	_A          =   frozenset(
	    [
	        'num_inference_steps',
	        'generator',
	        'latents',
	        'output_type',
	        'return_dict',
	        'callback',
	        'callback_steps',
	    ] )






	@property
	def    lowerCAmelCase_							(		self       )						->    Dict:
					snake_case__      :Tuple         =   UNetaDModel.from_pretrained(
					    "diffusers/consistency-models-test"					,subfolder="test_unet"					,)
					return unet






	@property
	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Tuple         =   UNetaDModel.from_pretrained(
					    "diffusers/consistency-models-test"					,subfolder="test_unet_class_cond"					,)
					return unet






	def    lowerCAmelCase_							(		self					,UpperCamelCase=False       )						->    Tuple:
					if class_cond:
									snake_case__      :Tuple         =   self.dummy_cond_unet
					else:
									snake_case__      :Optional[int]         =   self.dummy_uncond_unet

					# Default to CM multistep sampler
					snake_case__      :List[str]         =   CMStochasticIterativeScheduler(
					    num_train_timesteps=40					,sigma_min=0.002					,sigma_max=80.0					,)

					snake_case__      :Tuple         =   {
					    "unet": unet,
					    "scheduler": scheduler,
					}

					return components






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase=0       )						->    str:
					if str(UpperCamelCase       ).startswith("mps"       ):
									snake_case__      :List[str]         =   torch.manual_seed(UpperCamelCase       )
					else:
									snake_case__      :Any         =   torch.Generator(device=UpperCamelCase       ).manual_seed(UpperCamelCase       )

					snake_case__      :Any         =   {
					    "batch_size": 1,
					    "num_inference_steps": None,
					    "timesteps": [22, 0],
					    "generator": generator,
					    "output_type": "np",
					}

					return inputs






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :Optional[int]         =   "cpu"  # ensure determinism for the device-dependent torch.Generator
					snake_case__      :Optional[Any]         =   self.get_dummy_components()
					snake_case__      :List[Any]         =   ConsistencyModelPipeline(**UpperCamelCase       )
					snake_case__      :Tuple         =   pipe.to(UpperCamelCase       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :Dict         =   self.get_dummy_inputs(UpperCamelCase       )
					snake_case__      :List[Any]         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 32, 32, 3)

					snake_case__      :Optional[int]         =   image[0, -3:, -3:, -1]
					snake_case__      :List[str]         =   np.array([0.3572, 0.6273, 0.4031, 0.3961, 0.4321, 0.5730, 0.5266, 0.4780, 0.5004]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3






	def    lowerCAmelCase_							(		self       )						->    Any:
					snake_case__      :Dict         =   "cpu"  # ensure determinism for the device-dependent torch.Generator
					snake_case__      :Tuple         =   self.get_dummy_components(class_cond=UpperCamelCase       )
					snake_case__      :Tuple         =   ConsistencyModelPipeline(**UpperCamelCase       )
					snake_case__      :Dict         =   pipe.to(UpperCamelCase       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :List[Any]         =   self.get_dummy_inputs(UpperCamelCase       )
					snake_case__      :Optional[Any]         =   0
					snake_case__      :int         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 32, 32, 3)

					snake_case__      :int         =   image[0, -3:, -3:, -1]
					snake_case__      :str         =   np.array([0.3572, 0.6273, 0.4031, 0.3961, 0.4321, 0.5730, 0.5266, 0.4780, 0.5004]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :List[str]         =   "cpu"  # ensure determinism for the device-dependent torch.Generator
					snake_case__      :List[str]         =   self.get_dummy_components()
					snake_case__      :Any         =   ConsistencyModelPipeline(**UpperCamelCase       )
					snake_case__      :Tuple         =   pipe.to(UpperCamelCase       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :int         =   self.get_dummy_inputs(UpperCamelCase       )
					snake_case__      :Optional[Any]         =   1
					snake_case__      :str         =   None
					snake_case__      :Dict         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 32, 32, 3)

					snake_case__      :Dict         =   image[0, -3:, -3:, -1]
					snake_case__      :List[Any]         =   np.array([0.5004, 0.5004, 0.4994, 0.5008, 0.4976, 0.5018, 0.4990, 0.4982, 0.4987]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :List[str]         =   "cpu"  # ensure determinism for the device-dependent torch.Generator
					snake_case__      :Union[str, Any]         =   self.get_dummy_components(class_cond=UpperCamelCase       )
					snake_case__      :Dict         =   ConsistencyModelPipeline(**UpperCamelCase       )
					snake_case__      :Any         =   pipe.to(UpperCamelCase       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :Tuple         =   self.get_dummy_inputs(UpperCamelCase       )
					snake_case__      :Optional[Any]         =   1
					snake_case__      :List[Any]         =   None
					snake_case__      :Dict         =   0
					snake_case__      :Dict         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 32, 32, 3)

					snake_case__      :Any         =   image[0, -3:, -3:, -1]
					snake_case__      :Optional[int]         =   np.array([0.5004, 0.5004, 0.4994, 0.5008, 0.4976, 0.5018, 0.4990, 0.4982, 0.4987]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3




@slow
@require_torch_gpu
class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					super().tearDown()
					gc.collect()
					torch.cuda.empty_cache()






	def    lowerCAmelCase_							(		self					,UpperCamelCase=0					,UpperCamelCase=False					,UpperCamelCase="cpu"					,UpperCamelCase=torch.floataa					,UpperCamelCase=(1, 3, 64, 64)       )						->    List[Any]:
					snake_case__      :str         =   torch.manual_seed(UpperCamelCase       )

					snake_case__      :str         =   {
					    "num_inference_steps": None,
					    "timesteps": [22, 0],
					    "class_labels": 0,
					    "generator": generator,
					    "output_type": "np",
					}

					if get_fixed_latents:
									snake_case__      :Any         =   self.get_fixed_latents(seed=UpperCamelCase					,device=UpperCamelCase					,dtype=UpperCamelCase					,shape=UpperCamelCase       )
									snake_case__      :Union[str, Any]         =   latents

					return inputs






	def    lowerCAmelCase_							(		self					,UpperCamelCase=0					,UpperCamelCase="cpu"					,UpperCamelCase=torch.floataa					,UpperCamelCase=(1, 3, 64, 64)       )						->    Any:
					if type(UpperCamelCase       ) == str:
									snake_case__      :int         =   torch.device(UpperCamelCase       )
					snake_case__      :List[Any]         =   torch.Generator(device=UpperCamelCase       ).manual_seed(UpperCamelCase       )
					snake_case__      :Tuple         =   randn_tensor(UpperCamelCase					,generator=UpperCamelCase					,device=UpperCamelCase					,dtype=UpperCamelCase       )
					return latents






	def    lowerCAmelCase_							(		self       )						->    Any:
					snake_case__      :List[str]         =   UNetaDModel.from_pretrained("diffusers/consistency_models"					,subfolder="diffusers_cd_imagenet64_l2"       )
					snake_case__      :Any         =   CMStochasticIterativeScheduler(
					    num_train_timesteps=40					,sigma_min=0.002					,sigma_max=80.0					,)
					snake_case__      :int         =   ConsistencyModelPipeline(unet=UpperCamelCase					,scheduler=UpperCamelCase       )
					pipe.to(torch_device=UpperCamelCase       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :List[str]         =   self.get_inputs()
					snake_case__      :List[Any]         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 64, 64, 3)

					snake_case__      :Tuple         =   image[0, -3:, -3:, -1]

					snake_case__      :List[str]         =   np.array([0.0888, 0.0881, 0.0666, 0.0479, 0.0292, 0.0195, 0.0201, 0.0163, 0.0254]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 2E-2






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :Optional[int]         =   UNetaDModel.from_pretrained("diffusers/consistency_models"					,subfolder="diffusers_cd_imagenet64_l2"       )
					snake_case__      :Optional[Any]         =   CMStochasticIterativeScheduler(
					    num_train_timesteps=40					,sigma_min=0.002					,sigma_max=80.0					,)
					snake_case__      :Union[str, Any]         =   ConsistencyModelPipeline(unet=UpperCamelCase					,scheduler=UpperCamelCase       )
					pipe.to(torch_device=UpperCamelCase       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :Union[str, Any]         =   self.get_inputs()
					snake_case__      :str         =   1
					snake_case__      :Tuple         =   None
					snake_case__      :str         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 64, 64, 3)

					snake_case__      :List[str]         =   image[0, -3:, -3:, -1]

					snake_case__      :Union[str, Any]         =   np.array([0.0340, 0.0152, 0.0063, 0.0267, 0.0221, 0.0107, 0.0416, 0.0186, 0.0217]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 2E-2






	@require_torch_a
	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :List[str]         =   UNetaDModel.from_pretrained("diffusers/consistency_models"					,subfolder="diffusers_cd_imagenet64_l2"       )
					snake_case__      :Union[str, Any]         =   CMStochasticIterativeScheduler(
					    num_train_timesteps=40					,sigma_min=0.002					,sigma_max=80.0					,)
					snake_case__      :Union[str, Any]         =   ConsistencyModelPipeline(unet=UpperCamelCase					,scheduler=UpperCamelCase       )
					pipe.to(torch_device=UpperCamelCase					,torch_dtype=torch.floataa       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :Any         =   self.get_inputs(get_fixed_latents=UpperCamelCase					,device=UpperCamelCase       )
					# Ensure usage of flash attention in torch 2.0
					with sdp_kernel(enable_flash=UpperCamelCase					,enable_math=UpperCamelCase					,enable_mem_efficient=UpperCamelCase       ):
									snake_case__      :Any         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 64, 64, 3)

					snake_case__      :Tuple         =   image[0, -3:, -3:, -1]

					snake_case__      :List[str]         =   np.array([0.1875, 0.1428, 0.1289, 0.2151, 0.2092, 0.1477, 0.1877, 0.1641, 0.1353]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3






	@require_torch_a
	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :Optional[int]         =   UNetaDModel.from_pretrained("diffusers/consistency_models"					,subfolder="diffusers_cd_imagenet64_l2"       )
					snake_case__      :int         =   CMStochasticIterativeScheduler(
					    num_train_timesteps=40					,sigma_min=0.002					,sigma_max=80.0					,)
					snake_case__      :Dict         =   ConsistencyModelPipeline(unet=UpperCamelCase					,scheduler=UpperCamelCase       )
					pipe.to(torch_device=UpperCamelCase					,torch_dtype=torch.floataa       )
					pipe.set_progress_bar_config(disable=UpperCamelCase       )

					snake_case__      :List[Any]         =   self.get_inputs(get_fixed_latents=UpperCamelCase					,device=UpperCamelCase       )
					snake_case__      :Tuple         =   1
					snake_case__      :int         =   None
					# Ensure usage of flash attention in torch 2.0
					with sdp_kernel(enable_flash=UpperCamelCase					,enable_math=UpperCamelCase					,enable_mem_efficient=UpperCamelCase       ):
									snake_case__      :Any         =   pipe(**UpperCamelCase       ).images
					assert image.shape == (1, 64, 64, 3)

					snake_case__      :Dict         =   image[0, -3:, -3:, -1]

					snake_case__      :Tuple         =   np.array([0.1663, 0.1948, 0.2275, 0.1680, 0.1204, 0.1245, 0.1858, 0.1338, 0.2095]       )

					assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast


class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					# A mock response for an HTTP head request to emulate server down
					snake_case__      :Tuple         =   mock.Mock()
					snake_case__      :List[str]         =   500
					snake_case__      :Any         =   {}
					snake_case__      :Union[str, Any]         =   HTTPError
					snake_case__      :Tuple         =   {}

					# Download this model to make sure it's in the cache.
					snake_case__      :Any         =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch("requests.Session.request"					,return_value=UpperCamelCase       ) as mock_head:
									snake_case__      :Dict         =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
									# This check we did call the fake head request
									mock_head.assert_called()






	@require_tokenizers
	def    lowerCAmelCase_							(		self       )						->    Dict:
					# A mock response for an HTTP head request to emulate server down
					snake_case__      :Union[str, Any]         =   mock.Mock()
					snake_case__      :int         =   500
					snake_case__      :Any         =   {}
					snake_case__      :Dict         =   HTTPError
					snake_case__      :List[Any]         =   {}

					# Download this model to make sure it's in the cache.
					snake_case__      :Optional[int]         =   GPTaTokenizerFast.from_pretrained("gpt2"       )

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch("requests.Session.request"					,return_value=UpperCamelCase       ) as mock_head:
									snake_case__      :Any         =   GPTaTokenizerFast.from_pretrained("gpt2"       )
									# This check we did call the fake head request
									mock_head.assert_called()






	def    lowerCAmelCase_							(		self       )						->    int:
					# This test is for deprecated behavior and can be removed in v5
					try:
									snake_case__      :Union[str, Any]         =   tempfile.mktemp()
									with open(UpperCamelCase					,"wb"       ) as f:
													http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"					,UpperCamelCase       )

									snake_case__      :Tuple         =   AlbertTokenizer.from_pretrained(UpperCamelCase       )
					finally:
									os.remove(UpperCamelCase       )

					# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
					# the current folder and have the right name.
					if os.path.isfile("tokenizer.json"       ):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
					try:
									with open("tokenizer.json"					,"wb"       ) as f:
													http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"					,UpperCamelCase       )
									snake_case__      :Dict         =   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size					,1_000       )
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

					finally:
									os.remove("tokenizer.json"       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					# This test is for deprecated behavior and can be removed in v5
					snake_case__      :Union[str, Any]         =   AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )




@is_staging_test
class 					_snake_case						(			unittest.TestCase ):
	_A          =   ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






	@classmethod
	def    lowerCAmelCase_							(		cls       )						->    Optional[int]:
					snake_case__      :List[str]         =   TOKEN
					HfFolder.save_token(UpperCamelCase       )






	@classmethod
	def    lowerCAmelCase_							(		cls       )						->    Union[str, Any]:
					try:
									delete_repo(token=cls._token					,repo_id="test-tokenizer"       )
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token					,repo_id="valid_org/test-tokenizer-org"       )
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token					,repo_id="test-dynamic-tokenizer"       )
					except HTTPError:
									pass






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :List[str]         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :str         =   BertTokenizer(UpperCamelCase       )

					tokenizer.push_to_hub("test-tokenizer"					,use_auth_token=self._token       )
					snake_case__      :Dict         =   BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )

					# Reset repo
					delete_repo(token=self._token					,repo_id="test-tokenizer"       )

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(UpperCamelCase					,repo_id="test-tokenizer"					,push_to_hub=UpperCamelCase					,use_auth_token=self._token       )

					snake_case__      :List[str]         =   BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :List[Any]         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :Any         =   BertTokenizer(UpperCamelCase       )

					tokenizer.push_to_hub("valid_org/test-tokenizer-org"					,use_auth_token=self._token       )
					snake_case__      :Any         =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )

					# Reset repo
					delete_repo(token=self._token					,repo_id="valid_org/test-tokenizer-org"       )

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    UpperCamelCase					,repo_id="valid_org/test-tokenizer-org"					,push_to_hub=UpperCamelCase					,use_auth_token=self._token       )

					snake_case__      :Union[str, Any]         =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )






	@require_tokenizers
	def    lowerCAmelCase_							(		self       )						->    Any:
					CustomTokenizer.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :str         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :Optional[int]         =   CustomTokenizer(UpperCamelCase       )

					# No fast custom tokenizer
					tokenizer.push_to_hub("test-dynamic-tokenizer"					,use_auth_token=self._token       )

					snake_case__      :Union[str, Any]         =   AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizer"       )

					# Fast and slow custom tokenizer
					CustomTokenizerFast.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :int         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

									snake_case__      :Tuple         =   BertTokenizerFast.from_pretrained(UpperCamelCase       )
									bert_tokenizer.save_pretrained(UpperCamelCase       )
									snake_case__      :List[Any]         =   CustomTokenizerFast.from_pretrained(UpperCamelCase       )

					tokenizer.push_to_hub("test-dynamic-tokenizer"					,use_auth_token=self._token       )

					snake_case__      :List[Any]         =   AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizerFast"       )
					snake_case__      :List[str]         =   AutoTokenizer.from_pretrained(
					    f'{USER}/test-dynamic-tokenizer'					,use_fast=UpperCamelCase					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizer"       )




class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :int         =   Trie()
					trie.add("Hello 友達"       )
					self.assertEqual(trie.data					,{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
					trie.add("Hello"       )
					trie.data
					self.assertEqual(trie.data					,{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )






	def    lowerCAmelCase_							(		self       )						->    int:
					snake_case__      :List[str]         =   Trie()
					self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )					,["[CLS] This is a extra_id_100"]       )
					trie.add("[CLS]"       )
					trie.add("extra_id_1"       )
					trie.add("extra_id_100"       )
					self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )					,["[CLS]", " This is a ", "extra_id_100"]       )






	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Optional[Any]         =   Trie()
					trie.add("A"       )
					self.assertEqual(trie.split("ABC"       )					,["A", "BC"]       )
					self.assertEqual(trie.split("BCA"       )					,["BC", "A"]       )






	def    lowerCAmelCase_							(		self       )						->    Dict:
					snake_case__      :Any         =   Trie()
					trie.add("TOKEN]"       )
					trie.add("[SPECIAL_TOKEN]"       )
					self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )					,["This is something ", "[SPECIAL_TOKEN]"]       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :List[Any]         =   Trie()
					trie.add("A"       )
					trie.add("P"       )
					trie.add("[SPECIAL_TOKEN]"       )
					self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )					,["This is something ", "[SPECIAL_TOKEN]"]       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :str         =   Trie()
					trie.add("AB"       )
					trie.add("B"       )
					trie.add("C"       )
					self.assertEqual(trie.split("ABC"       )					,["AB", "C"]       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :Dict         =   Trie()
					trie.add("ABC"       )
					trie.add("B"       )
					trie.add("CD"       )
					self.assertEqual(trie.split("ABCD"       )					,["ABC", "D"]       )






	def    lowerCAmelCase_							(		self       )						->    int:
					# Even if the offsets are wrong, we necessarily output correct string
					# parts.
					snake_case__      :Optional[int]         =   Trie()
					snake_case__      :Union[str, Any]         =   trie.cut_text("ABC"					,[0, 0, 2, 1, 2, 3]       )
					self.assertEqual(UpperCamelCase					,["AB", "C"]       )