




"""simple docstring"""






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__						=			get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')




@require_sentencepiece
@require_tokenizers
class   a__			(       UpperCamelCase_					,	unittest.TestCase  ):
						snake_case__             = GPTSwaTokenizer
						snake_case__             = False
						snake_case__             = True
						snake_case__             = False





						def       __UpperCamelCase     (							self      :	str)						->       List[str]:




											"""simple docstring"""







											super().setUp()

											# We have a SentencePiece fixture for testing
											_lowerCAmelCase:str									=  GPTSwaTokenizer(__a     ,eos_token='''<unk>'''     ,bos_token='''<unk>'''     ,pad_token='''<unk>''')

											tokenizer.save_pretrained(self.tmpdirname)





						def       __UpperCamelCase     (							self      :	Any     ,a__      :	int)						->       Union[str, Any]:




											"""simple docstring"""







											_lowerCAmelCase:List[str]									=  '''This is a test'''
											_lowerCAmelCase:Dict									=  '''This is a test'''
											return input_text, output_text





						def       __UpperCamelCase     (							self      :	Optional[Any])						->       List[str]:




											"""simple docstring"""







											_lowerCAmelCase:List[str]									=  '''<s>'''
											_lowerCAmelCase:Optional[Any]									=  1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a)     ,__a)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a)     ,__a)





						def       __UpperCamelCase     (							self      :	List[Any])						->       Any:




											"""simple docstring"""







											_lowerCAmelCase:Optional[Any]									=  list(self.get_tokenizer().get_vocab().keys())

											self.assertEqual(vocab_keys[0]     ,'''<unk>''')
											self.assertEqual(vocab_keys[1]     ,'''<s>''')
											self.assertEqual(vocab_keys[-1]     ,'''j''')
											self.assertEqual(len(__a)     ,2000)





						def       __UpperCamelCase     (							self      :	Tuple)						->       Optional[Any]:




											"""simple docstring"""







											self.assertEqual(self.get_tokenizer().vocab_size     ,2000)





						def       __UpperCamelCase     (							self      :	List[Any])						->       Optional[Any]:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  GPTSwaTokenizer(__a)

											_lowerCAmelCase:Optional[int]									=  tokenizer.tokenize('''This is a test''')
											self.assertListEqual(__a     ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

											self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)     ,[465, 287, 265, 631, 842])

											_lowerCAmelCase:List[Any]									=  tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
											# fmt: off
											self.assertListEqual(
											    __a     ,['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']     ,)
											# fmt: on

											_lowerCAmelCase:Optional[Any]									=  tokenizer.convert_tokens_to_ids(__a)
											self.assertListEqual(
											    __a     ,[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]     ,)

											_lowerCAmelCase:Dict									=  tokenizer.convert_ids_to_tokens(__a)
											# fmt: off
											self.assertListEqual(
											    __a     ,['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.'''])
											# fmt: on





						def       __UpperCamelCase     (							self      :	str)						->       Dict:




											"""simple docstring"""







											_lowerCAmelCase:List[Any]									=  GPTSwaTokenizer(__a)
											_lowerCAmelCase:List[Any]									=  ['''This is a test''', '''I was born in 92000, and this is falsé.''']
											_lowerCAmelCase:Optional[Any]									=  [
											    [465, 287, 265, 631, 842],
											    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
											]

											# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
											for text, expected_ids in zip(__a     ,__a):
																self.assertListEqual(tokenizer.encode_fast(__a)     ,__a)

											# Test that decode_fast returns the input text
											for text, token_ids in zip(__a     ,__a):
																self.assertEqual(tokenizer.decode_fast(__a)     ,__a)







						@slow
						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       Optional[Any]:




											"""simple docstring"""







											_lowerCAmelCase:Dict									=  [
											    '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
											    '''Hey there, how are you doing this fine day?''',
											    '''This is a text with a trailing spaces followed by a dot     .''',
											    '''Häj sväjs lillebrör! =)''',
											    '''Det är inget fel på Mr. Cool''',
											]

											# fmt: off
											_lowerCAmelCase:Any									=  {'''input_ids''': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
											# fmt: on
											self.tokenizer_integration_test_util(
											    expected_encoding=__a     ,model_name='''AI-Sweden/gpt-sw3-126m'''     ,sequences=__a     ,)












"""simple docstring"""






from ...configuration_utils import PretrainedConfig




class   a__			(       UpperCamelCase_  ):
						snake_case__             = '''bert-generation'''





						def __init__(							self      :	Dict     ,a__      :	str=5_0358     ,a__      :	List[str]=1024     ,a__      :	int=24     ,a__      :	Optional[Any]=16     ,a__      :	List[str]=4096     ,a__      :	Optional[int]="gelu"     ,a__      :	str=0.1     ,a__      :	Union[str, Any]=0.1     ,a__      :	int=512     ,a__      :	Dict=0.02     ,a__      :	List[Any]=1E-12     ,a__      :	List[Any]=0     ,a__      :	int=2     ,a__      :	str=1     ,a__      :	Dict="absolute"     ,a__      :	int=True     ,**a__      :	List[str]     ,)						->       List[Any]:




											"""simple docstring"""







											super().__init__(pad_token_id=a__     ,bos_token_id=a__     ,eos_token_id=a__     ,**a__)

											_lowerCAmelCase:Optional[Any]									=  vocab_size
											_lowerCAmelCase:Union[str, Any]									=  hidden_size
											_lowerCAmelCase:Any									=  num_hidden_layers
											_lowerCAmelCase:int									=  num_attention_heads
											_lowerCAmelCase:int									=  hidden_act
											_lowerCAmelCase:List[Any]									=  intermediate_size
											_lowerCAmelCase:Optional[Any]									=  hidden_dropout_prob
											_lowerCAmelCase:int									=  attention_probs_dropout_prob
											_lowerCAmelCase:Optional[int]									=  max_position_embeddings
											_lowerCAmelCase:Dict									=  initializer_range
											_lowerCAmelCase:Union[str, Any]									=  layer_norm_eps
											_lowerCAmelCase:int									=  position_embedding_type
											_lowerCAmelCase:Tuple									=  use_cache






