





'''simple docstring'''

import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a        =						get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")







@require_sentencepiece
@require_tokenizers
class __a (   _snake_case,   unittest.TestCase			):
   __UpperCamelCase   : Any	      =	GPTSwaTokenizer
   __UpperCamelCase   : List[str]	      =	False
   __UpperCamelCase   : Optional[Any]	      =	True
   __UpperCamelCase   : List[str]	      =	False





   def      UpperCAmelCase__     (					self				:			Dict    ):

      '''simple docstring'''

      super().setUp()

      # We have a SentencePiece fixture for testing
      __SCREAMING_SNAKE_CASE      							=       GPTSwaTokenizer(lowerCamelCase				,eos_token="""<unk>"""				,bos_token="""<unk>"""				,pad_token="""<unk>"""    )

      tokenizer.save_pretrained(self.tmpdirname    )





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			Dict    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       """This is a test"""
      __SCREAMING_SNAKE_CASE      							=       """This is a test"""
      return input_text, output_text





   def      UpperCAmelCase__     (					self				:			Any    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       """<s>"""
      __SCREAMING_SNAKE_CASE      							=       1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase    )				,lowerCamelCase    )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase    )				,lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			List[Any]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       list(self.get_tokenizer().get_vocab().keys()    )

      self.assertEqual(vocab_keys[0]				,"""<unk>"""    )
      self.assertEqual(vocab_keys[1]				,"""<s>"""    )
      self.assertEqual(vocab_keys[-1]				,"""j"""    )
      self.assertEqual(len(lowerCamelCase    )				,2000    )





   def      UpperCAmelCase__     (					self				:			Optional[int]    ):

      '''simple docstring'''

      self.assertEqual(self.get_tokenizer().vocab_size				,2000    )





   def      UpperCAmelCase__     (					self				:			Optional[int]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       GPTSwaTokenizer(lowerCamelCase    )

      __SCREAMING_SNAKE_CASE      							=       tokenizer.tokenize("""This is a test"""    )
      self.assertListEqual(lowerCamelCase				,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase    )				,[465, 287, 265, 631, 842]    )

      __SCREAMING_SNAKE_CASE      							=       tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
      # fmt: off
      self.assertListEqual(
          lowerCamelCase				,["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]				,)
      # fmt: on

      __SCREAMING_SNAKE_CASE      							=       tokenizer.convert_tokens_to_ids(lowerCamelCase    )
      self.assertListEqual(
          lowerCamelCase				,[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]				,)

      __SCREAMING_SNAKE_CASE      							=       tokenizer.convert_ids_to_tokens(lowerCamelCase    )
      # fmt: off
      self.assertListEqual(
          lowerCamelCase				,["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]    )
      # fmt: on





   def      UpperCAmelCase__     (					self				:			str    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       GPTSwaTokenizer(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       ["""This is a test""", """I was born in 92000, and this is falsé."""]
      __SCREAMING_SNAKE_CASE      							=       [
          [465, 287, 265, 631, 842],
          [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
      ]

      # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
      for text, expected_ids in zip(lowerCamelCase				,lowerCamelCase    ):
         self.assertListEqual(tokenizer.encode_fast(lowerCamelCase    )				,lowerCamelCase    )

      # Test that decode_fast returns the input text
      for text, token_ids in zip(lowerCamelCase				,lowerCamelCase    ):
         self.assertEqual(tokenizer.decode_fast(lowerCamelCase    )				,lowerCamelCase    )





   @slow
   def      UpperCAmelCase__     (					self				:			Optional[int]    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       [
          """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
          """Hey there, how are you doing this fine day?""",
          """This is a text with a trailing spaces followed by a dot     .""",
          """Häj sväjs lillebrör! =)""",
          """Det är inget fel på Mr. Cool""",
      ]

      # fmt: off
      __SCREAMING_SNAKE_CASE      							=       {"""input_ids""": [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
      # fmt: on
      self.tokenizer_integration_test_util(
          expected_encoding=lowerCamelCase				,model_name="""AI-Sweden/gpt-sw3-126m"""				,sequences=lowerCamelCase				,)












'''simple docstring'''

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

a        =						[
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
    if pkg in deps:
        if pkg == "tokenizers":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_tokenizers_available

            if not is_tokenizers_available():
                continue  # not required, check version only if installed
        elif pkg == "accelerate":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_accelerate_available

            # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
            # Transformers with PyTorch
            if not is_accelerate_available():
                continue  # not required, check version only if installed

        require_version_core(deps[pkg])
    else:
        raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase=None      )						->					Optional[Any]:

   '''simple docstring'''







   require_version(deps[pkg]							,	__UpperCAmelCase      )





