





import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_					     =	get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")
@require_sentencepiece
@require_tokenizers
class        SCREAMING_SNAKE_CASE     (UpperCAmelCase    ,			unittest.TestCase   ):
   _UpperCamelCase					:	List[Any]				      = GPTSwaTokenizer
   _UpperCamelCase					:	str				      = False
   _UpperCamelCase					:	List[str]				      = True
   _UpperCamelCase					:	List[str]				      = False


   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[Any] )->     Optional[int]:





     """simple docstring"""







     super().setUp()

     # We have a SentencePiece fixture for testing
     lowercase__			    =						GPTSwaTokenizer(a    ,  eos_token='<unk>'    ,  bos_token='<unk>'    ,  pad_token='<unk>' )

     tokenizer.save_pretrained(self.tmpdirname )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[int]    ,  a    :       str )->     Dict:





     """simple docstring"""







     lowercase__			    =						'This is a test'
     lowercase__			    =						'This is a test'
     return input_text, output_text


   def  SCREAMING_SNAKE_CASE_							(					self    :       str )->     int:





     """simple docstring"""







     lowercase__			    =						'<s>'
     lowercase__			    =						1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(a )    ,  a )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(a )    ,  a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Union[str, Any] )->     Dict:





     """simple docstring"""







     lowercase__			    =						list(self.get_tokenizer().get_vocab().keys() )

     self.assertEqual(vocab_keys[0]    ,  '<unk>' )
     self.assertEqual(vocab_keys[1]    ,  '<s>' )
     self.assertEqual(vocab_keys[-1]    ,  'j' )
     self.assertEqual(len(a )    ,  2_000 )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Dict )->     Optional[Any]:





     """simple docstring"""







     self.assertEqual(self.get_tokenizer().vocab_size    ,  2_000 )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     str:





     """simple docstring"""







     lowercase__			    =						GPTSwaTokenizer(a )

     lowercase__			    =						tokenizer.tokenize('This is a test' )
     self.assertListEqual(a    ,  ['▁This', '▁is', '▁a', '▁t', 'est'] )

     self.assertListEqual(tokenizer.convert_tokens_to_ids(a )    ,  [465, 287, 265, 631, 842] )

     lowercase__			    =						tokenizer.tokenize('I was born in 92000, and this is falsé.' )
     # fmt: off
     self.assertListEqual(
         a    ,  ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']    ,  )
     # fmt: on

     lowercase__			    =						tokenizer.convert_tokens_to_ids(a )
     self.assertListEqual(
         a    ,  [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]    ,  )

     lowercase__			    =						tokenizer.convert_ids_to_tokens(a )
     # fmt: off
     self.assertListEqual(
         a    ,  ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.'] )
     # fmt: on


   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[Any] )->     List[str]:





     """simple docstring"""







     lowercase__			    =						GPTSwaTokenizer(a )
     lowercase__			    =						['This is a test', 'I was born in 92000, and this is falsé.']
     lowercase__			    =						[
         [465, 287, 265, 631, 842],
         [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
     ]

     # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
     for text, expected_ids in zip(a    ,  a ):
       self.assertListEqual(tokenizer.encode_fast(a )    ,  a )

     # Test that decode_fast returns the input text
     for text, token_ids in zip(a    ,  a ):
       self.assertEqual(tokenizer.decode_fast(a )    ,  a )


   @slow
   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						[
         '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
         'Hey there, how are you doing this fine day?',
         'This is a text with a trailing spaces followed by a dot     .',
         'Häj sväjs lillebrör! =)',
         'Det är inget fel på Mr. Cool',
     ]

     # fmt: off
     lowercase__			    =						{'input_ids': [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
     # fmt: on
     self.tokenizer_integration_test_util(
         expected_encoding=a    ,  model_name='AI-Sweden/gpt-sw3-126m'    ,  sequences=a    ,  )









def     __UpperCamelCase						(_SCREAMING_SNAKE_CASE	)       ->     bool:
  if not isinstance(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	):
    raise ValueError('Input series is not valid, valid series - [2, 4, 6]'	)
  if len(_SCREAMING_SNAKE_CASE	) == 0:
    raise ValueError('Input list must be a non empty list'	)
  if len(_SCREAMING_SNAKE_CASE	) == 1:
    return True
  lowercase__			    =						series[1] - series[0]
  for index in range(len(_SCREAMING_SNAKE_CASE	) - 1	):
    if series[index + 1] - series[index] != common_diff:
      return False
  return True




def     __UpperCamelCase						(_SCREAMING_SNAKE_CASE	)       ->     float:
  if not isinstance(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	):
    raise ValueError('Input series is not valid, valid series - [2, 4, 6]'	)
  if len(_SCREAMING_SNAKE_CASE	) == 0:
    raise ValueError('Input list must be a non empty list'	)
  lowercase__			    =						0
  for val in series:
    answer += val
  return answer / len(_SCREAMING_SNAKE_CASE	)


if __name__ == "__main__":
      import doctest

      doctest.testmod()


