
'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

class    _a				(_lowerCamelCase  ,					unittest.TestCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  CanineTokenizer
       SCREAMING_SNAKE_CASE      =  False






       def        UpperCamelCase				(							self      )    ->			str:
          super().setUp()
          _SCREAMING_SNAKE_CASE          =				CanineTokenizer()
          tokenizer.save_pretrained(self.tmpdirname      )






       @cached_property
       def        UpperCamelCase				(							self      )    ->			str:
          return CanineTokenizer.from_pretrained("""google/canine-s"""      )






       def        UpperCamelCase				(							self       ,							**A__      )    ->			CanineTokenizer:
          _SCREAMING_SNAKE_CASE          =				self.tokenizer_class.from_pretrained(self.tmpdirname       ,							**A__      )
          _SCREAMING_SNAKE_CASE          =				10_24
          return tokenizer






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE          =				self.canine_tokenizer
          _SCREAMING_SNAKE_CASE          =				["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
          # fmt: off
          _SCREAMING_SNAKE_CASE          =				[5_73_44, 76, 1_05, 1_02, 1_01, 32, 1_05, 1_15, 32, 1_08, 1_05, 1_07, 1_01, 32, 97, 32, 98, 1_11, 1_20, 32, 1_11, 1_02, 32, 99, 1_04, 1_11, 99, 1_11, 1_08, 97, 1_16, 1_01, 1_15, 46, 5_73_45, 0, 0, 0, 0]
          # fmt: on
          _SCREAMING_SNAKE_CASE          =				tokenizer(A__       ,							padding=A__       ,							return_tensors="""pt"""      )
          self.assertIsInstance(A__       ,							A__      )

          _SCREAMING_SNAKE_CASE          =				list(batch.input_ids.numpy()[0]      )

          self.assertListEqual(A__       ,							A__      )

          self.assertEqual((2, 39)       ,							batch.input_ids.shape      )
          self.assertEqual((2, 39)       ,							batch.attention_mask.shape      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE          =				self.canine_tokenizer
          _SCREAMING_SNAKE_CASE          =				["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
          _SCREAMING_SNAKE_CASE          =				tokenizer(A__       ,							padding=A__       ,							return_tensors="""pt"""      )
          # check if input_ids, attention_mask and token_type_ids are returned
          self.assertIn("""input_ids"""       ,							A__      )
          self.assertIn("""attention_mask"""       ,							A__      )
          self.assertIn("""token_type_ids"""       ,							A__      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE          =				self.canine_tokenizer
          _SCREAMING_SNAKE_CASE          =				[
              """What's the weater?""",
              """It's about 25 degrees.""",
          ]
          _SCREAMING_SNAKE_CASE          =				tokenizer(
              text_target=A__       ,							max_length=32       ,							padding="""max_length"""       ,							truncation=A__       ,							return_tensors="""pt"""      )
          self.assertEqual(32       ,							targets["""input_ids"""].shape[1]      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          # safety check on max_len default value so we are sure the test works
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                self.assertNotEqual(tokenizer.model_max_length       ,							42      )

        # Now let's start the test
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                # Isolate this from the other tests because we save additional tokens/etc
                _SCREAMING_SNAKE_CASE          =				tempfile.mkdtemp()

                _SCREAMING_SNAKE_CASE          =				""" He is very happy, UNwant\u00E9d,running"""
                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
                tokenizer.save_pretrained(A__      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.__class__.from_pretrained(A__      )
                _SCREAMING_SNAKE_CASE          =				after_tokenizer.encode(A__       ,							add_special_tokens=A__      )
                self.assertListEqual(A__       ,							A__      )

                shutil.rmtree(A__      )

          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers(model_max_length=42      )
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                # Isolate this from the other tests because we save additional tokens/etc
                _SCREAMING_SNAKE_CASE          =				tempfile.mkdtemp()

                _SCREAMING_SNAKE_CASE          =				""" He is very happy, UNwant\u00E9d,running"""

                _SCREAMING_SNAKE_CASE          =				tokenizer.additional_special_tokens

                # We can add a new special token for Canine as follows:
                _SCREAMING_SNAKE_CASE          =				chr(0Xe_0_0_7      )
                additional_special_tokens.append(A__      )
                tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}      )
                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
                tokenizer.save_pretrained(A__      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.__class__.from_pretrained(A__      )
                _SCREAMING_SNAKE_CASE          =				after_tokenizer.encode(A__       ,							add_special_tokens=A__      )
                self.assertListEqual(A__       ,							A__      )
                self.assertIn(A__       ,							after_tokenizer.additional_special_tokens      )
                self.assertEqual(after_tokenizer.model_max_length       ,							42      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.__class__.from_pretrained(A__       ,							model_max_length=43      )
                self.assertEqual(tokenizer.model_max_length       ,							43      )

                shutil.rmtree(A__      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers(do_lower_case=A__      )
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                _SCREAMING_SNAKE_CASE     ,							_SCREAMING_SNAKE_CASE          =				self.get_clean_sequence(A__      )

                # a special token for Canine can be defined as follows:
                _SCREAMING_SNAKE_CASE          =				0Xe_0_0_5
                _SCREAMING_SNAKE_CASE          =				chr(A__      )

                tokenizer.add_special_tokens({"""cls_token""": special_token}      )
                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
                self.assertEqual(len(A__      )       ,							1      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.decode(ids + encoded_special_token       ,							clean_up_tokenization_spaces=A__      )
                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
                self.assertEqual(A__       ,							input_encoded + special_token_id      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.decode(A__       ,							skip_special_tokens=A__      )
                self.assertTrue(special_token not in decoded      )






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers(do_lower_case=A__      )
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                _SCREAMING_SNAKE_CASE          =				chr(0Xe_0_0_5      )
                _SCREAMING_SNAKE_CASE          =				chr(0Xe_0_0_6      )

                # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                tokenizer.add_tokens([SPECIAL_TOKEN_1]       ,							special_tokens=A__      )
                # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}      )

                _SCREAMING_SNAKE_CASE          =				tokenizer.tokenize(A__      )
                _SCREAMING_SNAKE_CASE          =				tokenizer.tokenize(A__      )

                self.assertEqual(len(A__      )       ,							1      )
                self.assertEqual(len(A__      )       ,							1      )
                self.assertEqual(token_a[0]       ,							A__      )
                self.assertEqual(token_a[0]       ,							A__      )






       @require_tokenizers
       def        UpperCamelCase				(							self      )    ->			int:
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers(do_lower_case=A__      )
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                # a special token for Canine can be defined as follows:
                _SCREAMING_SNAKE_CASE          =				0Xe_0_0_6
                _SCREAMING_SNAKE_CASE          =				chr(A__      )

                _SCREAMING_SNAKE_CASE          =				AddedToken(A__       ,							lstrip=A__      )
                tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}      )

                with tempfile.TemporaryDirectory() as tmp_dir_name:
                   tokenizer.save_pretrained(A__      )
                   tokenizer.from_pretrained(A__      )






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE          =				[]
          if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

          if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

          for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer_utils.save_pretrained(A__      )

                with open(os.path.join(A__       ,							"""special_tokens_map.json"""      )       ,							encoding="""utf-8"""      ) as json_file:
                   _SCREAMING_SNAKE_CASE          =				json.load(A__      )

                with open(os.path.join(A__       ,							"""tokenizer_config.json"""      )       ,							encoding="""utf-8"""      ) as json_file:
                   _SCREAMING_SNAKE_CASE          =				json.load(A__      )

                # a special token for Canine can be defined as follows:
                _SCREAMING_SNAKE_CASE          =				0Xe_0_0_6
                _SCREAMING_SNAKE_CASE          =				chr(A__      )

                _SCREAMING_SNAKE_CASE          =				[new_token_a]
                _SCREAMING_SNAKE_CASE          =				[new_token_a]

                with open(os.path.join(A__       ,							"""special_tokens_map.json"""      )       ,							"""w"""       ,							encoding="""utf-8"""      ) as outfile:
                   json.dump(A__       ,							A__      )
                with open(os.path.join(A__       ,							"""tokenizer_config.json"""      )       ,							"""w"""       ,							encoding="""utf-8"""      ) as outfile:
                   json.dump(A__       ,							A__      )

                # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                # "special_tokens_map.json" files
                _SCREAMING_SNAKE_CASE          =				tokenizer_class.from_pretrained(A__       ,							extra_ids=0      )
                self.assertIn(A__       ,							tokenizer_without_change_in_init.additional_special_tokens      )
                # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                self.assertEqual(
                    [new_token_a]       ,							tokenizer_without_change_in_init.convert_ids_to_tokens(
                        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]      )      )       ,							)

                _SCREAMING_SNAKE_CASE          =				0Xe_0_0_7
                _SCREAMING_SNAKE_CASE          =				chr(A__      )
                # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                _SCREAMING_SNAKE_CASE          =				[AddedToken(A__       ,							lstrip=A__      )]
                _SCREAMING_SNAKE_CASE          =				tokenizer_class.from_pretrained(
                    A__       ,							additional_special_tokens=A__       ,							extra_ids=0      )

                self.assertIn(A__       ,							tokenizer.additional_special_tokens      )
                # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                self.assertEqual(
                    [new_token_a]       ,							tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]      )      )      )






       @require_tokenizers
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers(do_lower_case=A__      )
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                _SCREAMING_SNAKE_CASE          =				"""hello world"""
                if self.space_between_special_tokens:
                   _SCREAMING_SNAKE_CASE          =				"""[CLS] hello world [SEP]"""
                else:
                   _SCREAMING_SNAKE_CASE          =				input
                _SCREAMING_SNAKE_CASE          =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
                _SCREAMING_SNAKE_CASE          =				tokenizer.decode(A__       ,							spaces_between_special_tokens=self.space_between_special_tokens      )
                self.assertIn(A__       ,							[output, output.lower()]      )






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE          =				self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                _SCREAMING_SNAKE_CASE          =				[
                    """bos_token""",
                    """eos_token""",
                    """unk_token""",
                    """sep_token""",
                    """pad_token""",
                    """cls_token""",
                    """mask_token""",
                ]

          _SCREAMING_SNAKE_CASE          =				"""a"""
          _SCREAMING_SNAKE_CASE          =				ord(A__      )

          for attr in attributes_list:
             setattr(A__       ,							attr + """_id"""       ,							A__      )
             self.assertEqual(getattr(A__       ,							A__      )       ,							A__      )
             self.assertEqual(getattr(A__       ,							attr + """_id"""      )       ,							A__      )

             setattr(A__       ,							attr + """_id"""       ,							A__      )
             self.assertEqual(getattr(A__       ,							A__      )       ,							A__      )
             self.assertEqual(getattr(A__       ,							attr + """_id"""      )       ,							A__      )

          setattr(A__       ,							"""additional_special_tokens_ids"""       ,							[]      )
          self.assertListEqual(getattr(A__       ,							"""additional_special_tokens"""      )       ,							[]      )
          self.assertListEqual(getattr(A__       ,							"""additional_special_tokens_ids"""      )       ,							[]      )

          _SCREAMING_SNAKE_CASE          =				0Xe_0_0_6
          _SCREAMING_SNAKE_CASE          =				chr(A__      )
          setattr(A__       ,							"""additional_special_tokens_ids"""       ,							[additional_special_token_id]      )
          self.assertListEqual(getattr(A__       ,							"""additional_special_tokens"""      )       ,							[additional_special_token]      )
          self.assertListEqual(getattr(A__       ,							"""additional_special_tokens_ids"""      )       ,							[additional_special_token_id]      )






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          pass






       def        UpperCamelCase				(							self      )    ->			List[str]:
          pass






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          pass






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          pass






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          pass






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          pass






       def        UpperCamelCase				(							self      )    ->			int:
          pass






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          pass








'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCamelCase__      :		str       							=  {
    "configuration_canine": ["CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP", "CanineConfig"],
    "tokenization_canine": ["CanineTokenizer"],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCamelCase__      :		List[Any]       							=  [
         "CANINE_PRETRAINED_MODEL_ARCHIVE_LIST",
         "CanineForMultipleChoice",
         "CanineForQuestionAnswering",
         "CanineForSequenceClassification",
         "CanineForTokenClassification",
         "CanineLayer",
         "CanineModel",
         "CaninePreTrainedModel",
         "load_tf_weights_in_canine",
     ]


if TYPE_CHECKING:
     from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
     from .tokenization_canine import CanineTokenizer

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_canine import (
              CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
              CanineForMultipleChoice,
              CanineForQuestionAnswering,
              CanineForSequenceClassification,
              CanineForTokenClassification,
              CanineLayer,
              CanineModel,
              CaninePreTrainedModel,
              load_tf_weights_in_canine,
          )


else:
     import sys

     UpperCamelCase__      :		str       							=  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






