






import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotConfig, is_flax_available
from transformers.testing_utils import jax_device, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
	import os

	# The slow tests are often failing with OOM error on GPU
	# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
	# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
	__UpperCAmelCase							:   Union[str, Any]							=  "platform"

	import jax
	import jax.numpy as jnp

	from transformers import BlenderbotTokenizer
	from transformers.models.blenderbot.modeling_flax_blenderbot import (
	    FlaxBlenderbotForConditionalGeneration,
	    FlaxBlenderbotModel,
	    shift_tokens_right,
	)





def 					lowercase_ (    __snake_case     :							Dict	,	__snake_case     :							Dict	,	__snake_case     :							Union[str, Any]=None	,	__snake_case     :							List[str]=None	,	__snake_case     :							Tuple=None	,	__snake_case     :							Optional[Any]=None	,	__snake_case     :							List[str]=None	,	__snake_case     :							Optional[Any]=None	,	)     -> Optional[Any]:





				'''simple docstring'''


				if attention_mask is None:
								snake_case__      :Union[str, Any]         =   np.where(input_ids != config.pad_token_id	,	1	,	0      )
				if decoder_attention_mask is None:
								snake_case__      :Optional[int]         =   np.where(decoder_input_ids != config.pad_token_id	,	1	,	0      )
				if head_mask is None:
								snake_case__      :Union[str, Any]         =   np.ones((config.encoder_layers, config.encoder_attention_heads)      )
				if decoder_head_mask is None:
								snake_case__      :str         =   np.ones((config.decoder_layers, config.decoder_attention_heads)      )
				if cross_attn_head_mask is None:
								snake_case__      :str         =   np.ones((config.decoder_layers, config.decoder_attention_heads)      )
				return {
				    "input_ids": input_ids,
				    "decoder_input_ids": decoder_input_ids,
				    "attention_mask": attention_mask,
				    "decoder_attention_mask": attention_mask,
				}


class 					_snake_case						:






	def __init__(		self					,UpperCamelCase					,UpperCamelCase=13					,UpperCamelCase=7					,UpperCamelCase=True					,UpperCamelCase=False					,UpperCamelCase=99					,UpperCamelCase=16					,UpperCamelCase=2					,UpperCamelCase=4					,UpperCamelCase=4					,UpperCamelCase="gelu"					,UpperCamelCase=0.1					,UpperCamelCase=0.1					,UpperCamelCase=32					,UpperCamelCase=2					,UpperCamelCase=1					,UpperCamelCase=0					,UpperCamelCase=0.02					,)						->    Optional[Any]:
					snake_case__      :Dict         =   parent
					snake_case__      :int         =   batch_size
					snake_case__      :Optional[Any]         =   seq_length
					snake_case__      :Optional[Any]         =   is_training
					snake_case__      :Optional[Any]         =   use_labels
					snake_case__      :Union[str, Any]         =   vocab_size
					snake_case__      :Optional[int]         =   hidden_size
					snake_case__      :List[Any]         =   num_hidden_layers
					snake_case__      :List[Any]         =   num_attention_heads
					snake_case__      :Any         =   intermediate_size
					snake_case__      :Any         =   hidden_act
					snake_case__      :Dict         =   hidden_dropout_prob
					snake_case__      :str         =   attention_probs_dropout_prob
					snake_case__      :str         =   max_position_embeddings
					snake_case__      :Optional[int]         =   eos_token_id
					snake_case__      :Optional[int]         =   pad_token_id
					snake_case__      :Dict         =   bos_token_id
					snake_case__      :List[str]         =   initializer_range






	def    lowerCAmelCase_							(		self       )						->    Any:
					snake_case__      :Optional[Any]         =   np.clip(ids_tensor([self.batch_size, self.seq_length - 1]					,self.vocab_size       )					,3					,self.vocab_size       )
					snake_case__      :str         =   np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)					,dtype=np.intaa       ))					,-1       )

					snake_case__      :Union[str, Any]         =   shift_tokens_right(UpperCamelCase					,1					,2       )

					snake_case__      :List[Any]         =   BlenderbotConfig(
					    vocab_size=self.vocab_size					,d_model=self.hidden_size					,encoder_layers=self.num_hidden_layers					,decoder_layers=self.num_hidden_layers					,encoder_attention_heads=self.num_attention_heads					,decoder_attention_heads=self.num_attention_heads					,encoder_ffn_dim=self.intermediate_size					,decoder_ffn_dim=self.intermediate_size					,dropout=self.hidden_dropout_prob					,attention_dropout=self.attention_probs_dropout_prob					,max_position_embeddings=self.max_position_embeddings					,eos_token_id=self.eos_token_id					,bos_token_id=self.bos_token_id					,pad_token_id=self.pad_token_id					,initializer_range=self.initializer_range					,use_cache=UpperCamelCase					,)
					snake_case__      :Union[str, Any]         =   prepare_blenderbot_inputs_dict(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )
					return config, inputs_dict






	def    lowerCAmelCase_							(		self       )						->    int:
					snake_case__    ,						snake_case__      :Any         =   self.prepare_config_and_inputs()
					return config, inputs_dict






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    Tuple:
					snake_case__      :List[Any]         =   20
					snake_case__      :str         =   model_class_name(UpperCamelCase       )

					snake_case__      :Union[str, Any]         =   model.encode(inputs_dict["input_ids"]       )

					snake_case__    ,						snake_case__      :Any         =   (
					    inputs_dict["decoder_input_ids"],
					    inputs_dict["decoder_attention_mask"],
					)

					snake_case__      :Union[str, Any]         =   model.init_cache(decoder_input_ids.shape[0]					,UpperCamelCase					,UpperCamelCase       )
					snake_case__      :Union[str, Any]         =   jnp.ones((decoder_input_ids.shape[0], max_decoder_length)					,dtype="i4"       )

					snake_case__      :Any         =   jnp.broadcast_to(
					    jnp.arange(decoder_input_ids.shape[-1] - 1       )[None, :]					,(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)					,)
					snake_case__      :Optional[Any]         =   model.decode(
					    decoder_input_ids[:, :-1]					,UpperCamelCase					,decoder_attention_mask=UpperCamelCase					,past_key_values=UpperCamelCase					,decoder_position_ids=UpperCamelCase					,)

					snake_case__      :Optional[Any]         =   jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]					,dtype="i4"       )
					snake_case__      :Any         =   model.decode(
					    decoder_input_ids[:, -1:]					,UpperCamelCase					,decoder_attention_mask=UpperCamelCase					,past_key_values=outputs_cache.past_key_values					,decoder_position_ids=UpperCamelCase					,)

					snake_case__      :str         =   model.decode(UpperCamelCase					,UpperCamelCase       )

					snake_case__      :Tuple         =   np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
					self.parent.assertTrue(diff < 1E-3					,msg=f'Max diff is {diff}'       )






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    Optional[int]:
					snake_case__      :Tuple         =   20
					snake_case__      :Dict         =   model_class_name(UpperCamelCase       )

					snake_case__      :Optional[Any]         =   model.encode(inputs_dict["input_ids"]       )

					snake_case__    ,						snake_case__      :Tuple         =   (
					    inputs_dict["decoder_input_ids"],
					    inputs_dict["decoder_attention_mask"],
					)

					snake_case__      :Tuple         =   jnp.concatenate(
					    [
					        decoder_attention_mask,
					        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])       ),
					    ]					,axis=-1					,)

					snake_case__      :str         =   model.init_cache(decoder_input_ids.shape[0]					,UpperCamelCase					,UpperCamelCase       )
					snake_case__      :Optional[Any]         =   jnp.broadcast_to(
					    jnp.arange(decoder_input_ids.shape[-1] - 1       )[None, :]					,(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)					,)

					snake_case__      :Optional[int]         =   model.decode(
					    decoder_input_ids[:, :-1]					,UpperCamelCase					,decoder_attention_mask=UpperCamelCase					,past_key_values=UpperCamelCase					,decoder_position_ids=UpperCamelCase					,)
					snake_case__      :int         =   jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]					,dtype="i4"       )
					snake_case__      :Optional[Any]         =   model.decode(
					    decoder_input_ids[:, -1:]					,UpperCamelCase					,past_key_values=outputs_cache.past_key_values					,decoder_attention_mask=UpperCamelCase					,decoder_position_ids=UpperCamelCase					,)

					snake_case__      :List[str]         =   model.decode(UpperCamelCase					,UpperCamelCase					,decoder_attention_mask=UpperCamelCase       )

					snake_case__      :Union[str, Any]         =   np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
					self.parent.assertTrue(diff < 1E-3					,msg=f'Max diff is {diff}'       )




@require_flax
class 					_snake_case						(			unittest.TestCase ):
	_A          =   99






	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Tuple         =   np.array(
					    [
					        [71, 82, 18, 33, 46, 91, 2],
					        [68, 34, 26, 58, 30, 82, 2],
					        [5, 97, 17, 39, 94, 40, 2],
					        [76, 83, 94, 25, 70, 78, 2],
					        [87, 59, 41, 35, 48, 66, 2],
					        [55, 13, 16, 58, 5, 2, 1],  # note padding
					        [64, 27, 31, 51, 12, 75, 2],
					        [52, 64, 86, 17, 83, 39, 2],
					        [48, 61, 9, 24, 71, 82, 2],
					        [26, 1, 60, 48, 22, 13, 2],
					        [21, 5, 62, 28, 14, 76, 2],
					        [45, 98, 37, 86, 59, 48, 2],
					        [70, 70, 50, 9, 28, 0, 2],
					    ]					,dtype=np.intaa					,)

					snake_case__      :List[Any]         =   input_ids.shape[0]
					snake_case__      :Optional[Any]         =   BlenderbotConfig(
					    vocab_size=self.vocab_size					,d_model=24					,encoder_layers=2					,decoder_layers=2					,encoder_attention_heads=2					,decoder_attention_heads=2					,encoder_ffn_dim=32					,decoder_ffn_dim=32					,max_position_embeddings=48					,eos_token_id=2					,pad_token_id=1					,bos_token_id=0					,)
					return config, input_ids, batch_size






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__    ,						snake_case__    ,						snake_case__      :List[str]         =   self._get_config_and_data()
					snake_case__      :str         =   FlaxBlenderbotForConditionalGeneration(UpperCamelCase       )
					snake_case__      :Optional[Any]         =   lm_model(input_ids=UpperCamelCase       )
					snake_case__      :Optional[int]         =   (batch_size, input_ids.shape[1], config.vocab_size)
					self.assertEqual(outputs["logits"].shape					,UpperCamelCase       )






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :Tuple         =   BlenderbotConfig(
					    vocab_size=self.vocab_size					,d_model=14					,encoder_layers=2					,decoder_layers=2					,encoder_attention_heads=2					,decoder_attention_heads=2					,encoder_ffn_dim=8					,decoder_ffn_dim=8					,max_position_embeddings=48					,)
					snake_case__      :str         =   FlaxBlenderbotForConditionalGeneration(UpperCamelCase       )
					snake_case__      :Optional[Any]         =   np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]					,dtype=np.intaa       )
					snake_case__      :Optional[Any]         =   np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]					,dtype=np.intaa       )
					snake_case__      :Any         =   lm_model(input_ids=UpperCamelCase					,decoder_input_ids=UpperCamelCase       )
					snake_case__      :str         =   (*summary.shape, config.vocab_size)
					self.assertEqual(outputs["logits"].shape					,UpperCamelCase       )






	def    lowerCAmelCase_							(		self       )						->    int:
					snake_case__      :Union[str, Any]         =   np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]					,dtype=np.intaa       )
					snake_case__      :str         =   shift_tokens_right(UpperCamelCase					,1					,2       )
					snake_case__      :int         =   np.equal(UpperCamelCase					,1       ).astype(np.floataa       ).sum()
					snake_case__      :Dict         =   np.equal(UpperCamelCase					,1       ).astype(np.floataa       ).sum()
					self.assertEqual(shifted.shape					,input_ids.shape       )
					self.assertEqual(UpperCamelCase					,n_pad_before - 1       )
					self.assertTrue(np.equal(shifted[:, 0]					,2       ).all()       )




@require_flax
class 					_snake_case						(			_A       ,    unittest.TestCase       ,    _A ):
	_A          =   True
	_A          =   (
	    (
	        FlaxBlenderbotModel,
	        FlaxBlenderbotForConditionalGeneration,
	    )
	    if is_flax_available()
	    else ()
	)
	_A          =   (FlaxBlenderbotForConditionalGeneration,) if is_flax_available() else ()






	def    lowerCAmelCase_							(		self       )						->    int:
					snake_case__      :List[str]         =   FlaxBlenderbotModelTester(self       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					snake_case__    ,						snake_case__      :Tuple         =   self.model_tester.prepare_config_and_inputs()
					for model_class in self.all_model_classes:
									self.model_tester.check_use_cache_forward(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__    ,						snake_case__      :int         =   self.model_tester.prepare_config_and_inputs()
					for model_class in self.all_model_classes:
									self.model_tester.check_use_cache_forward_with_attn_mask(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )






	def    lowerCAmelCase_							(		self       )						->    List[str]:
					snake_case__    ,						snake_case__      :Dict         =   self.model_tester.prepare_config_and_inputs_for_common()

					for model_class in self.all_model_classes:
									with self.subTest(model_class.__name__       ):
													snake_case__      :Optional[int]         =   self._prepare_for_class(UpperCamelCase					,UpperCamelCase       )
													snake_case__      :Tuple         =   model_class(UpperCamelCase       )

													@jax.jit
													def encode_jitted(UpperCamelCase					,UpperCamelCase=None					,**UpperCamelCase       ):
																	return model.encode(input_ids=UpperCamelCase					,attention_mask=UpperCamelCase       )

													with self.subTest("JIT Enabled"       ):
																	snake_case__      :Dict         =   encode_jitted(**UpperCamelCase       ).to_tuple()

													with self.subTest("JIT Disabled"       ):
																	with jax.disable_jit():
																					snake_case__      :List[Any]         =   encode_jitted(**UpperCamelCase       ).to_tuple()

													self.assertEqual(len(UpperCamelCase       )					,len(UpperCamelCase       )       )
													for jitted_output, output in zip(UpperCamelCase					,UpperCamelCase       ):
																	self.assertEqual(jitted_output.shape					,output.shape       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__    ,						snake_case__      :List[str]         =   self.model_tester.prepare_config_and_inputs_for_common()

					for model_class in self.all_model_classes:
									with self.subTest(model_class.__name__       ):
													snake_case__      :Union[str, Any]         =   model_class(UpperCamelCase       )
													snake_case__      :Optional[Any]         =   model.encode(inputs_dict["input_ids"]					,inputs_dict["attention_mask"]       )

													snake_case__      :List[str]         =   {
													    "decoder_input_ids": inputs_dict["decoder_input_ids"],
													    "decoder_attention_mask": inputs_dict["decoder_attention_mask"],
													    "encoder_outputs": encoder_outputs,
													}

													@jax.jit
													def decode_jitted(UpperCamelCase					,UpperCamelCase					,UpperCamelCase       ):
																	return model.decode(
																	    decoder_input_ids=UpperCamelCase					,decoder_attention_mask=UpperCamelCase					,encoder_outputs=UpperCamelCase					,)

													with self.subTest("JIT Enabled"       ):
																	snake_case__      :Tuple         =   decode_jitted(**UpperCamelCase       ).to_tuple()

													with self.subTest("JIT Disabled"       ):
																	with jax.disable_jit():
																					snake_case__      :int         =   decode_jitted(**UpperCamelCase       ).to_tuple()

													self.assertEqual(len(UpperCamelCase       )					,len(UpperCamelCase       )       )
													for jitted_output, output in zip(UpperCamelCase					,UpperCamelCase       ):
																	self.assertEqual(jitted_output.shape					,output.shape       )






	@slow
	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					for model_class_name in self.all_model_classes:
									snake_case__      :Optional[int]         =   model_class_name.from_pretrained("facebook/blenderbot-400M-distill"       )
									# FlaxBlenderbotForSequenceClassification expects eos token in input_ids
									snake_case__      :Any         =   np.ones((1, 1)       ) * model.config.eos_token_id
									snake_case__      :Dict         =   model(UpperCamelCase       )
									self.assertIsNotNone(UpperCamelCase       )






	@unittest.skipUnless(jax_device != "cpu"					,"3B test too slow on CPU."       )
	@slow
	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					snake_case__      :Any         =   {"num_beams": 1, "early_stopping": True, "min_length": 15, "max_length": 25}
					snake_case__      :Any         =   {"skip_special_tokens": True, "clean_up_tokenization_spaces": True}

					snake_case__      :str         =   FlaxBlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-3B"					,from_pt=UpperCamelCase       )
					snake_case__      :List[str]         =   BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B"       )

					snake_case__      :List[str]         =   ["Sam"]
					snake_case__      :Dict         =   tokenizer(UpperCamelCase					,return_tensors="jax"       )

					snake_case__      :List[Any]         =   model.generate(**UpperCamelCase					,**UpperCamelCase       )
					snake_case__      :Union[str, Any]         =   "Sam is a great name. It means \"sun\" in Gaelic."

					snake_case__      :List[Any]         =   tokenizer.batch_decode(UpperCamelCase					,**UpperCamelCase       )
					assert generated_txt[0].strip() == tgt_text







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast


class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					# A mock response for an HTTP head request to emulate server down
					snake_case__      :Tuple         =   mock.Mock()
					snake_case__      :List[str]         =   500
					snake_case__      :Any         =   {}
					snake_case__      :Union[str, Any]         =   HTTPError
					snake_case__      :Tuple         =   {}

					# Download this model to make sure it's in the cache.
					snake_case__      :Any         =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch("requests.Session.request"					,return_value=UpperCamelCase       ) as mock_head:
									snake_case__      :Dict         =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
									# This check we did call the fake head request
									mock_head.assert_called()






	@require_tokenizers
	def    lowerCAmelCase_							(		self       )						->    Dict:
					# A mock response for an HTTP head request to emulate server down
					snake_case__      :Union[str, Any]         =   mock.Mock()
					snake_case__      :int         =   500
					snake_case__      :Any         =   {}
					snake_case__      :Dict         =   HTTPError
					snake_case__      :List[Any]         =   {}

					# Download this model to make sure it's in the cache.
					snake_case__      :Optional[int]         =   GPTaTokenizerFast.from_pretrained("gpt2"       )

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch("requests.Session.request"					,return_value=UpperCamelCase       ) as mock_head:
									snake_case__      :Any         =   GPTaTokenizerFast.from_pretrained("gpt2"       )
									# This check we did call the fake head request
									mock_head.assert_called()






	def    lowerCAmelCase_							(		self       )						->    int:
					# This test is for deprecated behavior and can be removed in v5
					try:
									snake_case__      :Union[str, Any]         =   tempfile.mktemp()
									with open(UpperCamelCase					,"wb"       ) as f:
													http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"					,UpperCamelCase       )

									snake_case__      :Tuple         =   AlbertTokenizer.from_pretrained(UpperCamelCase       )
					finally:
									os.remove(UpperCamelCase       )

					# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
					# the current folder and have the right name.
					if os.path.isfile("tokenizer.json"       ):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
					try:
									with open("tokenizer.json"					,"wb"       ) as f:
													http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"					,UpperCamelCase       )
									snake_case__      :Dict         =   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size					,1_000       )
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

					finally:
									os.remove("tokenizer.json"       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					# This test is for deprecated behavior and can be removed in v5
					snake_case__      :Union[str, Any]         =   AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )




@is_staging_test
class 					_snake_case						(			unittest.TestCase ):
	_A          =   ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






	@classmethod
	def    lowerCAmelCase_							(		cls       )						->    Optional[int]:
					snake_case__      :List[str]         =   TOKEN
					HfFolder.save_token(UpperCamelCase       )






	@classmethod
	def    lowerCAmelCase_							(		cls       )						->    Union[str, Any]:
					try:
									delete_repo(token=cls._token					,repo_id="test-tokenizer"       )
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token					,repo_id="valid_org/test-tokenizer-org"       )
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token					,repo_id="test-dynamic-tokenizer"       )
					except HTTPError:
									pass






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :List[str]         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :str         =   BertTokenizer(UpperCamelCase       )

					tokenizer.push_to_hub("test-tokenizer"					,use_auth_token=self._token       )
					snake_case__      :Dict         =   BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )

					# Reset repo
					delete_repo(token=self._token					,repo_id="test-tokenizer"       )

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(UpperCamelCase					,repo_id="test-tokenizer"					,push_to_hub=UpperCamelCase					,use_auth_token=self._token       )

					snake_case__      :List[str]         =   BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :List[Any]         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :Any         =   BertTokenizer(UpperCamelCase       )

					tokenizer.push_to_hub("valid_org/test-tokenizer-org"					,use_auth_token=self._token       )
					snake_case__      :Any         =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )

					# Reset repo
					delete_repo(token=self._token					,repo_id="valid_org/test-tokenizer-org"       )

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    UpperCamelCase					,repo_id="valid_org/test-tokenizer-org"					,push_to_hub=UpperCamelCase					,use_auth_token=self._token       )

					snake_case__      :Union[str, Any]         =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )






	@require_tokenizers
	def    lowerCAmelCase_							(		self       )						->    Any:
					CustomTokenizer.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :str         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :Optional[int]         =   CustomTokenizer(UpperCamelCase       )

					# No fast custom tokenizer
					tokenizer.push_to_hub("test-dynamic-tokenizer"					,use_auth_token=self._token       )

					snake_case__      :Union[str, Any]         =   AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizer"       )

					# Fast and slow custom tokenizer
					CustomTokenizerFast.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :int         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

									snake_case__      :Tuple         =   BertTokenizerFast.from_pretrained(UpperCamelCase       )
									bert_tokenizer.save_pretrained(UpperCamelCase       )
									snake_case__      :List[Any]         =   CustomTokenizerFast.from_pretrained(UpperCamelCase       )

					tokenizer.push_to_hub("test-dynamic-tokenizer"					,use_auth_token=self._token       )

					snake_case__      :List[Any]         =   AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizerFast"       )
					snake_case__      :List[str]         =   AutoTokenizer.from_pretrained(
					    f'{USER}/test-dynamic-tokenizer'					,use_fast=UpperCamelCase					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizer"       )




class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :int         =   Trie()
					trie.add("Hello 友達"       )
					self.assertEqual(trie.data					,{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
					trie.add("Hello"       )
					trie.data
					self.assertEqual(trie.data					,{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )






	def    lowerCAmelCase_							(		self       )						->    int:
					snake_case__      :List[str]         =   Trie()
					self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )					,["[CLS] This is a extra_id_100"]       )
					trie.add("[CLS]"       )
					trie.add("extra_id_1"       )
					trie.add("extra_id_100"       )
					self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )					,["[CLS]", " This is a ", "extra_id_100"]       )






	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Optional[Any]         =   Trie()
					trie.add("A"       )
					self.assertEqual(trie.split("ABC"       )					,["A", "BC"]       )
					self.assertEqual(trie.split("BCA"       )					,["BC", "A"]       )






	def    lowerCAmelCase_							(		self       )						->    Dict:
					snake_case__      :Any         =   Trie()
					trie.add("TOKEN]"       )
					trie.add("[SPECIAL_TOKEN]"       )
					self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )					,["This is something ", "[SPECIAL_TOKEN]"]       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :List[Any]         =   Trie()
					trie.add("A"       )
					trie.add("P"       )
					trie.add("[SPECIAL_TOKEN]"       )
					self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )					,["This is something ", "[SPECIAL_TOKEN]"]       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :str         =   Trie()
					trie.add("AB"       )
					trie.add("B"       )
					trie.add("C"       )
					self.assertEqual(trie.split("ABC"       )					,["AB", "C"]       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :Dict         =   Trie()
					trie.add("ABC"       )
					trie.add("B"       )
					trie.add("CD"       )
					self.assertEqual(trie.split("ABCD"       )					,["ABC", "D"]       )






	def    lowerCAmelCase_							(		self       )						->    int:
					# Even if the offsets are wrong, we necessarily output correct string
					# parts.
					snake_case__      :Optional[int]         =   Trie()
					snake_case__      :Union[str, Any]         =   trie.cut_text("ABC"					,[0, 0, 2, 1, 2, 3]       )
					self.assertEqual(UpperCamelCase					,["AB", "C"]       )