



"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__  ,				unittest.TestCase		):





		"""simple docstring"""






		_lowerCamelCase			  =       CanineTokenizer
		_lowerCamelCase			  =       False





		def    UpperCAmelCase__							(					self   ) ->						Union[str, Any]:







								'''simple docstring'''

								super().setUp()
								snake_case_   : List[str]						=			CanineTokenizer()
								tokenizer.save_pretrained(self.tmpdirname   )





		@cached_property
		def    UpperCAmelCase__							(					self   ) ->						Optional[Any]:







								'''simple docstring'''

								return CanineTokenizer.from_pretrained("""google/canine-s"""   )





		def    UpperCAmelCase__							(					self							,    **_lowercase   ) ->						CanineTokenizer:







								'''simple docstring'''

								snake_case_   : Optional[int]						=			self.tokenizer_class.from_pretrained(self.tmpdirname							,    **_lowercase   )
								snake_case_   : Any						=			1_0_2_4
								return tokenizer





		@require_torch
		def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







								'''simple docstring'''

								snake_case_   : Dict						=			self.canine_tokenizer
								snake_case_   : Any						=			["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
								# fmt: off
								snake_case_   : str						=			[5_7_3_4_4, 7_6, 1_0_5, 1_0_2, 1_0_1, 3_2, 1_0_5, 1_1_5, 3_2, 1_0_8, 1_0_5, 1_0_7, 1_0_1, 3_2, 9_7, 3_2, 9_8, 1_1_1, 1_2_0, 3_2, 1_1_1, 1_0_2, 3_2, 9_9, 1_0_4, 1_1_1, 9_9, 1_1_1, 1_0_8, 9_7, 1_1_6, 1_0_1, 1_1_5, 4_6, 5_7_3_4_5, 0, 0, 0, 0]
								# fmt: on
								snake_case_   : str						=			tokenizer(_lowercase							,    padding=_lowercase							,    return_tensors="""pt"""   )
								self.assertIsInstance(_lowercase							,    _lowercase   )

								snake_case_   : Tuple						=			list(batch.input_ids.numpy()[0]   )

								self.assertListEqual(_lowercase							,    _lowercase   )

								self.assertEqual((2, 3_9)							,    batch.input_ids.shape   )
								self.assertEqual((2, 3_9)							,    batch.attention_mask.shape   )





		@require_torch
		def    UpperCAmelCase__							(					self   ) ->						List[Any]:







								'''simple docstring'''

								snake_case_   : Tuple						=			self.canine_tokenizer
								snake_case_   : Dict						=			["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
								snake_case_   : str						=			tokenizer(_lowercase							,    padding=_lowercase							,    return_tensors="""pt"""   )
								# check if input_ids, attention_mask and token_type_ids are returned
								self.assertIn("""input_ids"""							,    _lowercase   )
								self.assertIn("""attention_mask"""							,    _lowercase   )
								self.assertIn("""token_type_ids"""							,    _lowercase   )





		@require_torch
		def    UpperCAmelCase__							(					self   ) ->						Union[str, Any]:







								'''simple docstring'''

								snake_case_   : Any						=			self.canine_tokenizer
								snake_case_   : List[Any]						=			[
								    """What's the weater?""",
								    """It's about 25 degrees.""",
								]
								snake_case_   : Union[str, Any]						=			tokenizer(
								    text_target=_lowercase							,    max_length=3_2							,    padding="""max_length"""							,    truncation=_lowercase							,    return_tensors="""pt"""   )
								self.assertEqual(3_2							,    targets["""input_ids"""].shape[1]   )





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								snake_case_   : Optional[Any]						=			self.get_tokenizers()
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				self.assertNotEqual(tokenizer.model_max_length							,    4_2   )

        # Now let's start the test
								snake_case_   : List[str]						=			self.get_tokenizers()
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				# Isolate this from the other tests because we save additional tokens/etc
																				snake_case_   : Any						=			tempfile.mkdtemp()

																				snake_case_   : List[Any]						=			""" He is very happy, UNwant\u00E9d,running"""
																				snake_case_   : str						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				tokenizer.save_pretrained(_lowercase   )

																				snake_case_   : int						=			tokenizer.__class__.from_pretrained(_lowercase   )
																				snake_case_   : int						=			after_tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				self.assertListEqual(_lowercase							,    _lowercase   )

																				shutil.rmtree(_lowercase   )

								snake_case_   : List[str]						=			self.get_tokenizers(model_max_length=4_2   )
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				# Isolate this from the other tests because we save additional tokens/etc
																				snake_case_   : Tuple						=			tempfile.mkdtemp()

																				snake_case_   : Any						=			""" He is very happy, UNwant\u00E9d,running"""

																				snake_case_   : Union[str, Any]						=			tokenizer.additional_special_tokens

																				# We can add a new special token for Canine as follows:
																				snake_case_   : Tuple						=			chr(0Xe0_07   )
																				additional_special_tokens.append(_lowercase   )
																				tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}   )
																				snake_case_   : Tuple						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				tokenizer.save_pretrained(_lowercase   )

																				snake_case_   : Optional[Any]						=			tokenizer.__class__.from_pretrained(_lowercase   )
																				snake_case_   : Any						=			after_tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				self.assertListEqual(_lowercase							,    _lowercase   )
																				self.assertIn(_lowercase							,    after_tokenizer.additional_special_tokens   )
																				self.assertEqual(after_tokenizer.model_max_length							,    4_2   )

																				snake_case_   : str						=			tokenizer.__class__.from_pretrained(_lowercase							,    model_max_length=4_3   )
																				self.assertEqual(tokenizer.model_max_length							,    4_3   )

																				shutil.rmtree(_lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						Optional[Any]:







								'''simple docstring'''

								snake_case_   : Tuple						=			self.get_tokenizers(do_lower_case=_lowercase   )
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				snake_case_			,		snake_case_   : int						=			self.get_clean_sequence(_lowercase   )

																				# a special token for Canine can be defined as follows:
																				snake_case_   : int						=			0Xe0_05
																				snake_case_   : Dict						=			chr(_lowercase   )

																				tokenizer.add_special_tokens({"""cls_token""": special_token}   )
																				snake_case_   : Dict						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				self.assertEqual(len(_lowercase   )							,    1   )

																				snake_case_   : List[str]						=			tokenizer.decode(ids + encoded_special_token							,    clean_up_tokenization_spaces=_lowercase   )
																				snake_case_   : int						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )

																				snake_case_   : List[Any]						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				snake_case_   : Any						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				self.assertEqual(_lowercase							,    input_encoded + special_token_id   )

																				snake_case_   : int						=			tokenizer.decode(_lowercase							,    skip_special_tokens=_lowercase   )
																				self.assertTrue(special_token not in decoded   )





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								snake_case_   : str						=			self.get_tokenizers(do_lower_case=_lowercase   )
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				snake_case_   : Any						=			chr(0Xe0_05   )
																				snake_case_   : Optional[int]						=			chr(0Xe0_06   )

																				# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
																				tokenizer.add_tokens([SPECIAL_TOKEN_1]							,    special_tokens=_lowercase   )
																				# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
																				# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
																				tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}   )

																				snake_case_   : Tuple						=			tokenizer.tokenize(_lowercase   )
																				snake_case_   : Optional[Any]						=			tokenizer.tokenize(_lowercase   )

																				self.assertEqual(len(_lowercase   )							,    1   )
																				self.assertEqual(len(_lowercase   )							,    1   )
																				self.assertEqual(token_a[0]							,    _lowercase   )
																				self.assertEqual(token_a[0]							,    _lowercase   )





		@require_tokenizers
		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								snake_case_   : List[Any]						=			self.get_tokenizers(do_lower_case=_lowercase   )
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				# a special token for Canine can be defined as follows:
																				snake_case_   : Tuple						=			0Xe0_06
																				snake_case_   : Any						=			chr(_lowercase   )

																				snake_case_   : List[Any]						=			AddedToken(_lowercase							,    lstrip=_lowercase   )
																				tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}   )

																				with tempfile.TemporaryDirectory() as tmp_dir_name:
																										tokenizer.save_pretrained(_lowercase   )
																										tokenizer.from_pretrained(_lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								snake_case_   : Any						=			[]
								if self.test_slow_tokenizer:
														tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

								if self.test_rust_tokenizer:
														tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

								for tokenizer_class, tokenizer_utils in tokenizer_list:
														with tempfile.TemporaryDirectory() as tmp_dir:
																				tokenizer_utils.save_pretrained(_lowercase   )

																				with open(os.path.join(_lowercase							,    """special_tokens_map.json"""   )							,    encoding="""utf-8"""   ) as json_file:
																										snake_case_   : Optional[int]						=			json.load(_lowercase   )

																				with open(os.path.join(_lowercase							,    """tokenizer_config.json"""   )							,    encoding="""utf-8"""   ) as json_file:
																										snake_case_   : str						=			json.load(_lowercase   )

																				# a special token for Canine can be defined as follows:
																				snake_case_   : Tuple						=			0Xe0_06
																				snake_case_   : List[Any]						=			chr(_lowercase   )

																				snake_case_   : List[Any]						=			[new_token_a]
																				snake_case_   : List[str]						=			[new_token_a]

																				with open(os.path.join(_lowercase							,    """special_tokens_map.json"""   )							,    """w"""							,    encoding="""utf-8"""   ) as outfile:
																										json.dump(_lowercase							,    _lowercase   )
																				with open(os.path.join(_lowercase							,    """tokenizer_config.json"""   )							,    """w"""							,    encoding="""utf-8"""   ) as outfile:
																										json.dump(_lowercase							,    _lowercase   )

																				# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																				# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																				# "special_tokens_map.json" files
																				snake_case_   : Any						=			tokenizer_class.from_pretrained(_lowercase							,    extra_ids=0   )
																				self.assertIn(_lowercase							,    tokenizer_without_change_in_init.additional_special_tokens   )
																				# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																				self.assertEqual(
																				    [new_token_a]							,    tokenizer_without_change_in_init.convert_ids_to_tokens(
																				        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]   )   )							,    )

																				snake_case_   : Any						=			0Xe0_07
																				snake_case_   : int						=			chr(_lowercase   )
																				# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																				snake_case_   : Tuple						=			[AddedToken(_lowercase							,    lstrip=_lowercase   )]
																				snake_case_   : Dict						=			tokenizer_class.from_pretrained(
																				    _lowercase							,    additional_special_tokens=_lowercase							,    extra_ids=0   )

																				self.assertIn(_lowercase							,    tokenizer.additional_special_tokens   )
																				# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
																				self.assertEqual(
																				    [new_token_a]							,    tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]   )   )   )





		@require_tokenizers
		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								snake_case_   : int						=			self.get_tokenizers(do_lower_case=_lowercase   )
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				snake_case_   : Optional[Any]						=			"""hello world"""
																				if self.space_between_special_tokens:
																										snake_case_   : str						=			"""[CLS] hello world [SEP]"""
																				else:
																										snake_case_   : List[Any]						=			input
																				snake_case_   : str						=			tokenizer.encode(_lowercase							,    add_special_tokens=_lowercase   )
																				snake_case_   : Union[str, Any]						=			tokenizer.decode(_lowercase							,    spaces_between_special_tokens=self.space_between_special_tokens   )
																				self.assertIn(_lowercase							,    [output, output.lower()]   )





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								snake_case_   : int						=			self.get_tokenizers()
								for tokenizer in tokenizers:
														with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																				snake_case_   : Optional[int]						=			[
																				    """bos_token""",
																				    """eos_token""",
																				    """unk_token""",
																				    """sep_token""",
																				    """pad_token""",
																				    """cls_token""",
																				    """mask_token""",
																				]

								snake_case_   : int						=			"""a"""
								snake_case_   : Dict						=			ord(_lowercase   )

								for attr in attributes_list:
														setattr(_lowercase							,    attr + """_id"""							,    _lowercase   )
														self.assertEqual(getattr(_lowercase							,    _lowercase   )							,    _lowercase   )
														self.assertEqual(getattr(_lowercase							,    attr + """_id"""   )							,    _lowercase   )

														setattr(_lowercase							,    attr + """_id"""							,    _lowercase   )
														self.assertEqual(getattr(_lowercase							,    _lowercase   )							,    _lowercase   )
														self.assertEqual(getattr(_lowercase							,    attr + """_id"""   )							,    _lowercase   )

								setattr(_lowercase							,    """additional_special_tokens_ids"""							,    []   )
								self.assertListEqual(getattr(_lowercase							,    """additional_special_tokens"""   )							,    []   )
								self.assertListEqual(getattr(_lowercase							,    """additional_special_tokens_ids"""   )							,    []   )

								snake_case_   : Dict						=			0Xe0_06
								snake_case_   : Dict						=			chr(_lowercase   )
								setattr(_lowercase							,    """additional_special_tokens_ids"""							,    [additional_special_token_id]   )
								self.assertListEqual(getattr(_lowercase							,    """additional_special_tokens"""   )							,    [additional_special_token]   )
								self.assertListEqual(getattr(_lowercase							,    """additional_special_tokens_ids"""   )							,    [additional_special_token_id]   )





		def    UpperCAmelCase__							(					self   ) ->						List[str]:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						str:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						List[str]:







								'''simple docstring'''

								pass






"""simple docstring"""





def 	__lowerCAmelCase						(		__UpperCamelCase      :			str    ):


						'''simple docstring'''






						return " ".join(input_str.split()[::-1]    )


if __name__ == "__main__":
					import doctest

					doctest.testmod()

