



"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin




class 			lowercase__			(      SCREAMING_SNAKE_CASE__,					unittest.TestCase    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					CanineTokenizer
							_UpperCAmelCase      				=					False






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									super().setUp()
									_UpperCAmelCase							 =		CanineTokenizer()
									tokenizer.save_pretrained(self.tmpdirname						)






							@cached_property
							def        lowerCamelCase_   (  self						)			->							Optional[Any]:
									return CanineTokenizer.from_pretrained('google/canine-s'						)






							def        lowerCamelCase_   (  self	,       **snake_case						)			->							CanineTokenizer:
									_UpperCAmelCase							 =		self.tokenizer_class.from_pretrained(self.tmpdirname	,       **_lowercase						)
									_UpperCAmelCase							 =		1024
									return tokenizer






							@require_torch
							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									_UpperCAmelCase							 =		self.canine_tokenizer
									_UpperCAmelCase							 =		["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
									# fmt: off
									_UpperCAmelCase							 =		[57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
									# fmt: on
									_UpperCAmelCase							 =		tokenizer(_lowercase	,       padding=_lowercase	,       return_tensors='pt'						)
									self.assertIsInstance(_lowercase	,       _lowercase						)

									_UpperCAmelCase							 =		list(batch.input_ids.numpy()[0]						)

									self.assertListEqual(_lowercase	,       _lowercase						)

									self.assertEqual((2, 39)	,       batch.input_ids.shape						)
									self.assertEqual((2, 39)	,       batch.attention_mask.shape						)






							@require_torch
							def        lowerCamelCase_   (  self						)			->							List[Any]:
									_UpperCAmelCase							 =		self.canine_tokenizer
									_UpperCAmelCase							 =		["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
									_UpperCAmelCase							 =		tokenizer(_lowercase	,       padding=_lowercase	,       return_tensors='pt'						)
									# check if input_ids, attention_mask and token_type_ids are returned
									self.assertIn('input_ids'	,       _lowercase						)
									self.assertIn('attention_mask'	,       _lowercase						)
									self.assertIn('token_type_ids'	,       _lowercase						)






							@require_torch
							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		self.canine_tokenizer
									_UpperCAmelCase							 =		[
									    """What's the weater?""",
									    """It's about 25 degrees.""",
									]
									_UpperCAmelCase							 =		tokenizer(
									    text_target=_lowercase	,       max_length=32	,       padding='max_length'	,       truncation=_lowercase	,       return_tensors='pt'						)
									self.assertEqual(32	,       targets['input_ids'].shape[1]						)






							def        lowerCamelCase_   (  self						)			->							Dict:
									_UpperCAmelCase							 =		self.get_tokenizers()
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													self.assertNotEqual(tokenizer.model_max_length	,       42						)

        # Now let's start the test
									_UpperCAmelCase							 =		self.get_tokenizers()
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													# Isolate this from the other tests because we save additional tokens/etc
													_UpperCAmelCase							 =		tempfile.mkdtemp()

													_UpperCAmelCase							 =		""" He is very happy, UNwant\u00E9d,running"""
													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													tokenizer.save_pretrained(_lowercase						)

													_UpperCAmelCase							 =		tokenizer.__class__.from_pretrained(_lowercase						)
													_UpperCAmelCase							 =		after_tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													self.assertListEqual(_lowercase	,       _lowercase						)

													shutil.rmtree(_lowercase						)

									_UpperCAmelCase							 =		self.get_tokenizers(model_max_length=42						)
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													# Isolate this from the other tests because we save additional tokens/etc
													_UpperCAmelCase							 =		tempfile.mkdtemp()

													_UpperCAmelCase							 =		""" He is very happy, UNwant\u00E9d,running"""

													_UpperCAmelCase							 =		tokenizer.additional_special_tokens

													# We can add a new special token for Canine as follows:
													_UpperCAmelCase							 =		chr(0xe_0_0_7						)
													additional_special_tokens.append(_lowercase						)
													tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}						)
													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													tokenizer.save_pretrained(_lowercase						)

													_UpperCAmelCase							 =		tokenizer.__class__.from_pretrained(_lowercase						)
													_UpperCAmelCase							 =		after_tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													self.assertListEqual(_lowercase	,       _lowercase						)
													self.assertIn(_lowercase	,       after_tokenizer.additional_special_tokens						)
													self.assertEqual(after_tokenizer.model_max_length	,       42						)

													_UpperCAmelCase							 =		tokenizer.__class__.from_pretrained(_lowercase	,       model_max_length=43						)
													self.assertEqual(tokenizer.model_max_length	,       43						)

													shutil.rmtree(_lowercase						)






							def        lowerCamelCase_   (  self						)			->							Optional[Any]:
									_UpperCAmelCase							 =		self.get_tokenizers(do_lower_case=_lowercase						)
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													_UpperCAmelCase							 =		self.get_clean_sequence(_lowercase						)

													# a special token for Canine can be defined as follows:
													_UpperCAmelCase							 =		0xe_0_0_5
													_UpperCAmelCase							 =		chr(_lowercase						)

													tokenizer.add_special_tokens({'cls_token': special_token}						)
													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													self.assertEqual(len(_lowercase						)	,       1						)

													_UpperCAmelCase							 =		tokenizer.decode(ids + encoded_special_token	,       clean_up_tokenization_spaces=_lowercase						)
													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)

													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													self.assertEqual(_lowercase	,       input_encoded + special_token_id						)

													_UpperCAmelCase							 =		tokenizer.decode(_lowercase	,       skip_special_tokens=_lowercase						)
													self.assertTrue(special_token not in decoded						)






							def        lowerCamelCase_   (  self						)			->							Dict:
									_UpperCAmelCase							 =		self.get_tokenizers(do_lower_case=_lowercase						)
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													_UpperCAmelCase							 =		chr(0xe_0_0_5						)
													_UpperCAmelCase							 =		chr(0xe_0_0_6						)

													# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
													tokenizer.add_tokens([SPECIAL_TOKEN_1]	,       special_tokens=_lowercase						)
													# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
													# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
													tokenizer.add_special_tokens({'additional_special_tokens': [SPECIAL_TOKEN_2]}						)

													_UpperCAmelCase							 =		tokenizer.tokenize(_lowercase						)
													_UpperCAmelCase							 =		tokenizer.tokenize(_lowercase						)

													self.assertEqual(len(_lowercase						)	,       1						)
													self.assertEqual(len(_lowercase						)	,       1						)
													self.assertEqual(token_a[0]	,       _lowercase						)
													self.assertEqual(token_a[0]	,       _lowercase						)






							@require_tokenizers
							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		self.get_tokenizers(do_lower_case=_lowercase						)
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													# a special token for Canine can be defined as follows:
													_UpperCAmelCase							 =		0xe_0_0_6
													_UpperCAmelCase							 =		chr(_lowercase						)

													_UpperCAmelCase							 =		AddedToken(_lowercase	,       lstrip=_lowercase						)
													tokenizer.add_special_tokens({'additional_special_tokens': [new_token]}						)

													with tempfile.TemporaryDirectory() as tmp_dir_name:
															tokenizer.save_pretrained(_lowercase						)
															tokenizer.from_pretrained(_lowercase						)






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		[]
									if self.test_slow_tokenizer:
											tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

									if self.test_rust_tokenizer:
											tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

									for tokenizer_class, tokenizer_utils in tokenizer_list:
											with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer_utils.save_pretrained(_lowercase						)

													with open(os.path.join(_lowercase	,       'special_tokens_map.json'						)	,       encoding='utf-8'						) as json_file:
															_UpperCAmelCase							 =		json.load(_lowercase						)

													with open(os.path.join(_lowercase	,       'tokenizer_config.json'						)	,       encoding='utf-8'						) as json_file:
															_UpperCAmelCase							 =		json.load(_lowercase						)

													# a special token for Canine can be defined as follows:
													_UpperCAmelCase							 =		0xe_0_0_6
													_UpperCAmelCase							 =		chr(_lowercase						)

													_UpperCAmelCase							 =		[new_token_a]
													_UpperCAmelCase							 =		[new_token_a]

													with open(os.path.join(_lowercase	,       'special_tokens_map.json'						)	,       'w'	,       encoding='utf-8'						) as outfile:
															json.dump(_lowercase	,       _lowercase						)
													with open(os.path.join(_lowercase	,       'tokenizer_config.json'						)	,       'w'	,       encoding='utf-8'						) as outfile:
															json.dump(_lowercase	,       _lowercase						)

													# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
													# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
													# "special_tokens_map.json" files
													_UpperCAmelCase							 =		tokenizer_class.from_pretrained(_lowercase	,       extra_ids=0						)
													self.assertIn(_lowercase	,       tokenizer_without_change_in_init.additional_special_tokens						)
													# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
													self.assertEqual(
													    [new_token_a]	,       tokenizer_without_change_in_init.convert_ids_to_tokens(
													        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]						)						)	,       )

													_UpperCAmelCase							 =		0xe_0_0_7
													_UpperCAmelCase							 =		chr(_lowercase						)
													# Now we test that we can change the value of additional_special_tokens in the from_pretrained
													_UpperCAmelCase							 =		[AddedToken(_lowercase	,       lstrip=_lowercase						)]
													_UpperCAmelCase							 =		tokenizer_class.from_pretrained(
													    _lowercase	,       additional_special_tokens=_lowercase	,       extra_ids=0						)

													self.assertIn(_lowercase	,       tokenizer.additional_special_tokens						)
													# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
													self.assertEqual(
													    [new_token_a]	,       tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]						)						)						)






							@require_tokenizers
							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		self.get_tokenizers(do_lower_case=_lowercase						)
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													_UpperCAmelCase							 =		"""hello world"""
													if self.space_between_special_tokens:
															_UpperCAmelCase							 =		"""[CLS] hello world [SEP]"""
													else:
															_UpperCAmelCase							 =		input
													_UpperCAmelCase							 =		tokenizer.encode(_lowercase	,       add_special_tokens=_lowercase						)
													_UpperCAmelCase							 =		tokenizer.decode(_lowercase	,       spaces_between_special_tokens=self.space_between_special_tokens						)
													self.assertIn(_lowercase	,       [output, output.lower()]						)






							def        lowerCamelCase_   (  self						)			->							Dict:
									_UpperCAmelCase							 =		self.get_tokenizers()
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'						):
													_UpperCAmelCase							 =		[
													    """bos_token""",
													    """eos_token""",
													    """unk_token""",
													    """sep_token""",
													    """pad_token""",
													    """cls_token""",
													    """mask_token""",
													]

									_UpperCAmelCase							 =		"""a"""
									_UpperCAmelCase							 =		ord(_lowercase						)

									for attr in attributes_list:
											setattr(_lowercase	,       attr + '_id'	,       _lowercase						)
											self.assertEqual(getattr(_lowercase	,       _lowercase						)	,       _lowercase						)
											self.assertEqual(getattr(_lowercase	,       attr + '_id'						)	,       _lowercase						)

											setattr(_lowercase	,       attr + '_id'	,       _lowercase						)
											self.assertEqual(getattr(_lowercase	,       _lowercase						)	,       _lowercase						)
											self.assertEqual(getattr(_lowercase	,       attr + '_id'						)	,       _lowercase						)

									setattr(_lowercase	,       'additional_special_tokens_ids'	,       []						)
									self.assertListEqual(getattr(_lowercase	,       'additional_special_tokens'						)	,       []						)
									self.assertListEqual(getattr(_lowercase	,       'additional_special_tokens_ids'						)	,       []						)

									_UpperCAmelCase							 =		0xe_0_0_6
									_UpperCAmelCase							 =		chr(_lowercase						)
									setattr(_lowercase	,       'additional_special_tokens_ids'	,       [additional_special_token_id]						)
									self.assertListEqual(getattr(_lowercase	,       'additional_special_tokens'						)	,       [additional_special_token]						)
									self.assertListEqual(getattr(_lowercase	,       'additional_special_tokens_ids'						)	,       [additional_special_token_id]						)






							def        lowerCamelCase_   (  self						)			->							List[str]:
									pass






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									pass






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									pass






							def        lowerCamelCase_   (  self						)			->							int:
									pass






							def        lowerCamelCase_   (  self						)			->							str:
									pass






							def        lowerCamelCase_   (  self						)			->							Dict:
									pass






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									pass






							def        lowerCamelCase_   (  self						)			->							List[str]:
									pass







"""simple docstring"""





import warnings

from ...utils import logging
from .image_processing_yolos import YolosImageProcessor


lowercase  							=					logging.get_logger(__name__)




class 			lowercase__			(      A    ):
							'''simple docstring'''







							def __init__(  self	,       *snake_case	,       **snake_case						)			->							None:
									warnings.warn(
									    'The class YolosFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'
									    ' use YolosImageProcessor instead.'	,       snake_case	,       )
									super().__init__(*snake_case	,       **snake_case						)


