
"""simple docstring"""



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case          =						logging.get_logger(__name__)

_snake_case          =						{
    '''tanreinama/GPTSAN-2.8B-spout_is_uniform''': (
        '''https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json'''
    ),
}
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					"gptsan-japanese"
				SCREAMING_SNAKE_CASE_:							List[str]							=					[
				    "past_key_values",
				]
				SCREAMING_SNAKE_CASE_:							Dict							=					{
				    "hidden_size": "d_model",
				    "num_attention_heads": "num_heads",
				    "num_hidden_layers": "num_layers",
				}


				def __init__(					self	:			Optional[Any]				,     UpperCAmelCase_	:			int=36_000				,     UpperCAmelCase_	:			Dict=1_280				,     UpperCAmelCase_	:			Tuple=1_024				,     UpperCAmelCase_	:			Union[str, Any]=8_192				,     UpperCAmelCase_	:			Dict=4_096				,     UpperCAmelCase_	:			str=128				,     UpperCAmelCase_	:			Dict=10				,     UpperCAmelCase_	:			Dict=0				,     UpperCAmelCase_	:			Dict=16				,     UpperCAmelCase_	:			Dict=16				,     UpperCAmelCase_	:			Any=128				,     UpperCAmelCase_	:			str=0.0				,     UpperCAmelCase_	:			Union[str, Any]=1E-5				,     UpperCAmelCase_	:			Union[str, Any]=False				,     UpperCAmelCase_	:			Optional[Any]=0.0				,     UpperCAmelCase_	:			Optional[int]="float32"				,     UpperCAmelCase_	:			int=False				,     UpperCAmelCase_	:			int=False				,     UpperCAmelCase_	:			List[str]=False				,     UpperCAmelCase_	:			Optional[Any]=0.002				,     UpperCAmelCase_	:			Tuple=False				,     UpperCAmelCase_	:			Any=True				,     UpperCAmelCase_	:			List[Any]=35_998				,     UpperCAmelCase_	:			Any=35_995				,     UpperCAmelCase_	:			List[Any]=35_999				,     **UpperCAmelCase_	:			Dict				,     )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			vocab_size
											_lowerCAmelCase  	=			max_position_embeddings
											_lowerCAmelCase  	=			d_model
											_lowerCAmelCase  	=			d_ff
											_lowerCAmelCase  	=			d_ext
											_lowerCAmelCase  	=			d_spout
											_lowerCAmelCase  	=			num_switch_layers
											_lowerCAmelCase  	=			num_ext_layers
											_lowerCAmelCase  	=			num_switch_layers + num_ext_layers
											_lowerCAmelCase  	=			num_heads
											_lowerCAmelCase  	=			num_experts
											_lowerCAmelCase  	=			expert_capacity
											_lowerCAmelCase  	=			dropout_rate
											_lowerCAmelCase  	=			layer_norm_epsilon
											_lowerCAmelCase  	=			router_bias
											_lowerCAmelCase  	=			router_jitter_noise
											_lowerCAmelCase  	=			router_dtype
											_lowerCAmelCase  	=			router_ignore_padding_tokens
											_lowerCAmelCase  	=			output_hidden_states
											_lowerCAmelCase  	=			output_attentions
											_lowerCAmelCase  	=			initializer_factor
											_lowerCAmelCase  	=			output_router_logits
											_lowerCAmelCase  	=			use_cache

											super().__init__(
											    separator_token_id=UpperCAmelCase_				,     pad_token_id=UpperCAmelCase_				,     eos_token_id=UpperCAmelCase_				,     **UpperCAmelCase_				,     )








"""simple docstring"""



import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					None
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							Any							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					"tokenizer_file"
				SCREAMING_SNAKE_CASE_:							str							=					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											super().setUp()
											_lowerCAmelCase  	=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     **UpperCAmelCase_	:			Any  )		->  Optional[Any]:







											"""simple docstring"""




											kwargs.update(self.special_tokens_map  )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_lowerCAmelCase  	=			[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

											_lowerCAmelCase  	=			tokenizer.batch_encode_plus(UpperCAmelCase_  )['input_ids']
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			tokenizer.batch_decode(UpperCAmelCase_  )
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any				,     UpperCAmelCase_	:			List[str]=6  )		->  Any:







											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																									_lowerCAmelCase  	=			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									_lowerCAmelCase  	=			'This is a simple input'
																									_lowerCAmelCase  	=			['This is a simple input 1', 'This is a simple input 2']
																									_lowerCAmelCase  	=			('This is a simple input', 'This is a pair')
																									_lowerCAmelCase  	=			[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )

																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'  )

																									_lowerCAmelCase  	=			None  # Hotfixing padding = None
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			load_dataset('xnli'				,     'all_languages'				,     split='test'				,     streaming=UpperCAmelCase_  )

											_lowerCAmelCase  	=			next(iter(UpperCAmelCase_  )  )['premise']  # pick up one data
											_lowerCAmelCase  	=			list(sample_data.values()  )

											_lowerCAmelCase  	=			list(map(tokenizer.encode				,     UpperCAmelCase_  )  )
											_lowerCAmelCase  	=			[tokenizer.decode(UpperCAmelCase_				,     clean_up_tokenization_spaces=UpperCAmelCase_  ) for x in output_tokens]
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )







				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[Any]:







											"""simple docstring"""




											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )				,     1  )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )				,     1  )






