






'''simple docstring'''

import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							__magic_name__     (	UpperCAmelCase_,						unittest.TestCase				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      List[str]        =  None
	SCREAMING_SNAKE_CASE_					:      Tuple        =  BloomTokenizerFast
	SCREAMING_SNAKE_CASE_					:      List[str]        =  BloomTokenizerFast
	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  True
	SCREAMING_SNAKE_CASE_					:      Optional[int]        =  False
	SCREAMING_SNAKE_CASE_					:      Dict        =  'tokenizer_file'
	SCREAMING_SNAKE_CASE_					:      List[str]        =  {'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>'}
	def        lowerCamelCase__		(							self       )						->  List[Any]:
							super().setUp()
							lowercase_      :   int        =			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'       )
							tokenizer.save_pretrained(self.tmpdirname       )
	def        lowerCamelCase__		(							self  ,      **_lowercase       )						->  Any:
							kwargs.update(self.special_tokens_map       )
							return BloomTokenizerFast.from_pretrained(self.tmpdirname  ,      **_lowercase       )
	def        lowerCamelCase__		(							self       )						->  Any:
							lowercase_      :   Optional[int]        =			self.get_rust_tokenizer()

							lowercase_      :   Any        =			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
							lowercase_      :   Dict        =			[[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

							lowercase_      :   List[Any]        =			tokenizer.batch_encode_plus(_lowercase       )['input_ids']
							self.assertListEqual(_lowercase  ,      _lowercase       )

							lowercase_      :   Optional[Any]        =			tokenizer.batch_decode(_lowercase       )
							self.assertListEqual(_lowercase  ,      _lowercase       )
	def        lowerCamelCase__		(							self  ,      _lowercase=6       )						->  Any:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
																			lowercase_      :   int        =			self.rust_tokenizer_class.from_pretrained(_lowercase  ,      **_lowercase       )
																			# tokenizer_r.pad_token = None # Hotfixing padding = None
																			# Simple input
																			lowercase_      :   Dict        =			'This is a simple input'
																			lowercase_      :   List[str]        =			['This is a simple input 1', 'This is a simple input 2']
																			lowercase_      :   List[Any]        =			('This is a simple input', 'This is a pair')
																			lowercase_      :   List[str]        =			[
																			    ('This is a simple input 1', 'This is a simple input 2'),
																			    ('This is a simple pair 1', 'This is a simple pair 2'),
																			]

																			# Simple input tests
																			try:
																									tokenizer_r.encode(_lowercase  ,      max_length=_lowercase       )
																									tokenizer_r.encode_plus(_lowercase  ,      max_length=_lowercase       )

																									tokenizer_r.batch_encode_plus(_lowercase  ,      max_length=_lowercase       )
																									tokenizer_r.encode(_lowercase  ,      max_length=_lowercase       )
																									tokenizer_r.batch_encode_plus(_lowercase  ,      max_length=_lowercase       )
																			except ValueError:
																									self.fail('Bloom Tokenizer should be able to deal with padding'       )

																			lowercase_      :   List[Any]        =			None  # Hotfixing padding = None
																			self.assertRaises(_lowercase  ,      tokenizer_r.encode  ,      _lowercase  ,      max_length=_lowercase  ,      padding='max_length'       )

																			# Simple input
																			self.assertRaises(_lowercase  ,      tokenizer_r.encode_plus  ,      _lowercase  ,      max_length=_lowercase  ,      padding='max_length'       )

																			# Simple input
																			self.assertRaises(
																			    _lowercase  ,      tokenizer_r.batch_encode_plus  ,      _lowercase  ,      max_length=_lowercase  ,      padding='max_length'  ,      )

																			# Pair input
																			self.assertRaises(_lowercase  ,      tokenizer_r.encode  ,      _lowercase  ,      max_length=_lowercase  ,      padding='max_length'       )

																			# Pair input
																			self.assertRaises(_lowercase  ,      tokenizer_r.encode_plus  ,      _lowercase  ,      max_length=_lowercase  ,      padding='max_length'       )

																			# Pair input
																			self.assertRaises(
																			    _lowercase  ,      tokenizer_r.batch_encode_plus  ,      _lowercase  ,      max_length=_lowercase  ,      padding='max_length'  ,      )
	def        lowerCamelCase__		(							self       )						->  int:
							lowercase_      :   List[str]        =			self.get_rust_tokenizer()
							lowercase_      :   Dict        =			load_dataset('xnli'  ,      'all_languages'  ,      split='test'  ,      streaming=_lowercase       )

							lowercase_      :   Optional[Any]        =			next(iter(_lowercase       )       )['premise']  # pick up one data
							lowercase_      :   Any        =			list(sample_data.values()       )

							lowercase_      :   int        =			list(map(tokenizer.encode  ,      _lowercase       )       )
							lowercase_      :   Optional[Any]        =			[tokenizer.decode(_lowercase  ,      clean_up_tokenization_spaces=_lowercase       ) for x in output_tokens]
							self.assertListEqual(_lowercase  ,      _lowercase       )







	def        lowerCamelCase__		(							self       )						->  int:
							# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
							# any sequence length constraints. This test of the parent class will fail since it relies on the
							# maximum sequence length of the positoonal embeddings.
							self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map       )  ,      1       )
							self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()       )[0]       )  ,      1       )








'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A:      int  				=				logging.get_logger(__name__)

A:      int  				=				{
    "bigcode/gpt_bigcode-santacoder": "https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json",
}






class 							__magic_name__     (	UpperCAmelCase_				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      List[str]        =  'gpt_bigcode'
	SCREAMING_SNAKE_CASE_					:      int        =  ['past_key_values']
	SCREAMING_SNAKE_CASE_					:      Any        =  {
	    'hidden_size': 'n_embd',
	    'max_position_embeddings': 'n_positions',
	    'num_attention_heads': 'n_head',
	    'num_hidden_layers': 'n_layer',
	}
	def __init__(							self  ,      _lowercase=5_0257  ,      _lowercase=1024  ,      _lowercase=768  ,      _lowercase=12  ,      _lowercase=12  ,      _lowercase=None  ,      _lowercase="gelu_pytorch_tanh"  ,      _lowercase=0.1  ,      _lowercase=0.1  ,      _lowercase=0.1  ,      _lowercase=1E-5  ,      _lowercase=0.02  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=5_0256  ,      _lowercase=5_0256  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=True  ,      **_lowercase  ,      )						->  Any:
							lowercase_      :   Tuple        =			vocab_size
							lowercase_      :   str        =			n_positions
							lowercase_      :   List[str]        =			n_embd
							lowercase_      :   str        =			n_layer
							lowercase_      :   Optional[Any]        =			n_head
							lowercase_      :   Optional[int]        =			n_inner
							lowercase_      :   Union[str, Any]        =			activation_function
							lowercase_      :   Dict        =			resid_pdrop
							lowercase_      :   str        =			embd_pdrop
							lowercase_      :   Optional[Any]        =			attn_pdrop
							lowercase_      :   List[Any]        =			layer_norm_epsilon
							lowercase_      :   Optional[int]        =			initializer_range
							lowercase_      :   List[Any]        =			scale_attn_weights
							lowercase_      :   Any        =			use_cache
							lowercase_      :   List[str]        =			attention_softmax_in_fpaa
							lowercase_      :   Any        =			scale_attention_softmax_in_fpaa
							lowercase_      :   Optional[Any]        =			multi_query

							lowercase_      :   Optional[Any]        =			bos_token_id
							lowercase_      :   Optional[Any]        =			eos_token_id

							super().__init__(bos_token_id=_lowercase  ,      eos_token_id=_lowercase  ,      **_lowercase       )
