





import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






class       __A					(					lowerCAmelCase    ,       unittest.TestCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		CanineTokenizer
			lowerCAmelCase_							  =		False





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				super().setUp()
				lowerCamelCase__              =	CanineTokenizer()
				tokenizer.save_pretrained(self.tmpdirname							)





			@cached_property
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				return CanineTokenizer.from_pretrained('''google/canine-s'''							)





			def 	__lowerCamelCase       (					self ,						**__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer_class.from_pretrained(self.tmpdirname ,						**__lowerCAmelCase							)
				lowerCamelCase__              =	1_0_2_4
				return tokenizer





			@require_torch
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.canine_tokenizer
				lowerCamelCase__              =	['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
				# fmt: off
				lowerCamelCase__              =	[5_7_3_4_4, 7_6, 1_0_5, 1_0_2, 1_0_1, 3_2, 1_0_5, 1_1_5, 3_2, 1_0_8, 1_0_5, 1_0_7, 1_0_1, 3_2, 9_7, 3_2, 9_8, 1_1_1, 1_2_0, 3_2, 1_1_1, 1_0_2, 3_2, 9_9, 1_0_4, 1_1_1, 9_9, 1_1_1, 1_0_8, 9_7, 1_1_6, 1_0_1, 1_1_5, 4_6, 5_7_3_4_5, 0, 0, 0, 0]
				# fmt: on
				lowerCamelCase__              =	tokenizer(__lowerCAmelCase ,						padding=__lowerCAmelCase ,						return_tensors='''pt'''							)
				self.assertIsInstance(__lowerCAmelCase ,						__lowerCAmelCase							)

				lowerCamelCase__              =	list(batch.input_ids.numpy()[0]							)

				self.assertListEqual(__lowerCAmelCase ,						__lowerCAmelCase							)

				self.assertEqual((2, 3_9) ,						batch.input_ids.shape							)
				self.assertEqual((2, 3_9) ,						batch.attention_mask.shape							)





			@require_torch
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.canine_tokenizer
				lowerCamelCase__              =	['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
				lowerCamelCase__              =	tokenizer(__lowerCAmelCase ,						padding=__lowerCAmelCase ,						return_tensors='''pt'''							)
				# check if input_ids, attention_mask and token_type_ids are returned
				self.assertIn('''input_ids''' ,						__lowerCAmelCase							)
				self.assertIn('''attention_mask''' ,						__lowerCAmelCase							)
				self.assertIn('''token_type_ids''' ,						__lowerCAmelCase							)





			@require_torch
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.canine_tokenizer
				lowerCamelCase__              =	[
				    '''What\'s the weater?''',
				    '''It\'s about 25 degrees.''',
				]
				lowerCamelCase__              =	tokenizer(
				    text_target=__lowerCAmelCase ,						max_length=3_2 ,						padding='''max_length''' ,						truncation=__lowerCAmelCase ,						return_tensors='''pt'''							)
				self.assertEqual(3_2 ,						targets['''input_ids'''].shape[1]							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.get_tokenizers()
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						self.assertNotEqual(tokenizer.model_max_length ,						4_2							)

        # Now let's start the test
				lowerCamelCase__              =	self.get_tokenizers()
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						# Isolate this from the other tests because we save additional tokens/etc
						lowerCamelCase__              =	tempfile.mkdtemp()

						lowerCamelCase__              =	''' He is very happy, UNwant\u00E9d,running'''
						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						tokenizer.save_pretrained(__lowerCAmelCase							)

						lowerCamelCase__              =	tokenizer.__class__.from_pretrained(__lowerCAmelCase							)
						lowerCamelCase__              =	after_tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						self.assertListEqual(__lowerCAmelCase ,						__lowerCAmelCase							)

						shutil.rmtree(__lowerCAmelCase							)

				lowerCamelCase__              =	self.get_tokenizers(model_max_length=4_2							)
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						# Isolate this from the other tests because we save additional tokens/etc
						lowerCamelCase__              =	tempfile.mkdtemp()

						lowerCamelCase__              =	''' He is very happy, UNwant\u00E9d,running'''

						lowerCamelCase__              =	tokenizer.additional_special_tokens

						# We can add a new special token for Canine as follows:
						lowerCamelCase__              =	chr(0xE_0_0_7							)
						additional_special_tokens.append(__lowerCAmelCase							)
						tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}							)
						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						tokenizer.save_pretrained(__lowerCAmelCase							)

						lowerCamelCase__              =	tokenizer.__class__.from_pretrained(__lowerCAmelCase							)
						lowerCamelCase__              =	after_tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						self.assertListEqual(__lowerCAmelCase ,						__lowerCAmelCase							)
						self.assertIn(__lowerCAmelCase ,						after_tokenizer.additional_special_tokens							)
						self.assertEqual(after_tokenizer.model_max_length ,						4_2							)

						lowerCamelCase__              =	tokenizer.__class__.from_pretrained(__lowerCAmelCase ,						model_max_length=4_3							)
						self.assertEqual(tokenizer.model_max_length ,						4_3							)

						shutil.rmtree(__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.get_tokenizers(do_lower_case=__lowerCAmelCase							)
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						lowerCamelCase__       ,			lowerCamelCase__              =	self.get_clean_sequence(__lowerCAmelCase							)

						# a special token for Canine can be defined as follows:
						lowerCamelCase__              =	0xE_0_0_5
						lowerCamelCase__              =	chr(__lowerCAmelCase							)

						tokenizer.add_special_tokens({'''cls_token''': special_token}							)
						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						self.assertEqual(len(__lowerCAmelCase							) ,						1							)

						lowerCamelCase__              =	tokenizer.decode(ids + encoded_special_token ,						clean_up_tokenization_spaces=__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)

						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						self.assertEqual(__lowerCAmelCase ,						input_encoded + special_token_id							)

						lowerCamelCase__              =	tokenizer.decode(__lowerCAmelCase ,						skip_special_tokens=__lowerCAmelCase							)
						self.assertTrue(special_token not in decoded							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.get_tokenizers(do_lower_case=__lowerCAmelCase							)
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						lowerCamelCase__              =	chr(0xE_0_0_5							)
						lowerCamelCase__              =	chr(0xE_0_0_6							)

						# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
						tokenizer.add_tokens([SPECIAL_TOKEN_1] ,						special_tokens=__lowerCAmelCase							)
						# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
						# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
						tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}							)

						lowerCamelCase__              =	tokenizer.tokenize(__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer.tokenize(__lowerCAmelCase							)

						self.assertEqual(len(__lowerCAmelCase							) ,						1							)
						self.assertEqual(len(__lowerCAmelCase							) ,						1							)
						self.assertEqual(token_a[0] ,						__lowerCAmelCase							)
						self.assertEqual(token_a[0] ,						__lowerCAmelCase							)





			@require_tokenizers
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.get_tokenizers(do_lower_case=__lowerCAmelCase							)
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						# a special token for Canine can be defined as follows:
						lowerCamelCase__              =	0xE_0_0_6
						lowerCamelCase__              =	chr(__lowerCAmelCase							)

						lowerCamelCase__              =	AddedToken(__lowerCAmelCase ,						lstrip=__lowerCAmelCase							)
						tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}							)

						with tempfile.TemporaryDirectory() as tmp_dir_name:
							tokenizer.save_pretrained(__lowerCAmelCase							)
							tokenizer.from_pretrained(__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	[]
				if self.test_slow_tokenizer:
					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

				if self.test_rust_tokenizer:
					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
					with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer_utils.save_pretrained(__lowerCAmelCase							)

						with open(os.path.join(__lowerCAmelCase ,						'''special_tokens_map.json'''							) ,						encoding='''utf-8'''							) as json_file:
							lowerCamelCase__              =	json.load(__lowerCAmelCase							)

						with open(os.path.join(__lowerCAmelCase ,						'''tokenizer_config.json'''							) ,						encoding='''utf-8'''							) as json_file:
							lowerCamelCase__              =	json.load(__lowerCAmelCase							)

						# a special token for Canine can be defined as follows:
						lowerCamelCase__              =	0xE_0_0_6
						lowerCamelCase__              =	chr(__lowerCAmelCase							)

						lowerCamelCase__              =	[new_token_a]
						lowerCamelCase__              =	[new_token_a]

						with open(os.path.join(__lowerCAmelCase ,						'''special_tokens_map.json'''							) ,						'''w''' ,						encoding='''utf-8'''							) as outfile:
							json.dump(__lowerCAmelCase ,						__lowerCAmelCase							)
						with open(os.path.join(__lowerCAmelCase ,						'''tokenizer_config.json'''							) ,						'''w''' ,						encoding='''utf-8'''							) as outfile:
							json.dump(__lowerCAmelCase ,						__lowerCAmelCase							)

						# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
						# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
						# "special_tokens_map.json" files
						lowerCamelCase__              =	tokenizer_class.from_pretrained(__lowerCAmelCase ,						extra_ids=0							)
						self.assertIn(__lowerCAmelCase ,						tokenizer_without_change_in_init.additional_special_tokens							)
						# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
						self.assertEqual(
						    [new_token_a] ,						tokenizer_without_change_in_init.convert_ids_to_tokens(
						        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]							)							) ,						)

						lowerCamelCase__              =	0xE_0_0_7
						lowerCamelCase__              =	chr(__lowerCAmelCase							)
						# Now we test that we can change the value of additional_special_tokens in the from_pretrained
						lowerCamelCase__              =	[AddedToken(__lowerCAmelCase ,						lstrip=__lowerCAmelCase							)]
						lowerCamelCase__              =	tokenizer_class.from_pretrained(
						    __lowerCAmelCase ,						additional_special_tokens=__lowerCAmelCase ,						extra_ids=0							)

						self.assertIn(__lowerCAmelCase ,						tokenizer.additional_special_tokens							)
						# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
						self.assertEqual(
						    [new_token_a] ,						tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]							)							)							)





			@require_tokenizers
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.get_tokenizers(do_lower_case=__lowerCAmelCase							)
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						lowerCamelCase__              =	'''hello world'''
						if self.space_between_special_tokens:
							lowerCamelCase__              =	'''[CLS] hello world [SEP]'''
						else:
							lowerCamelCase__              =	input
						lowerCamelCase__              =	tokenizer.encode(__lowerCAmelCase ,						add_special_tokens=__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer.decode(__lowerCAmelCase ,						spaces_between_special_tokens=self.space_between_special_tokens							)
						self.assertIn(__lowerCAmelCase ,						[output, output.lower()]							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.get_tokenizers()
				for tokenizer in tokenizers:
					with self.subTest(F'{tokenizer.__class__.__name__}'							):
						lowerCamelCase__              =	[
						    '''bos_token''',
						    '''eos_token''',
						    '''unk_token''',
						    '''sep_token''',
						    '''pad_token''',
						    '''cls_token''',
						    '''mask_token''',
						]

				lowerCamelCase__              =	'''a'''
				lowerCamelCase__              =	ord(__lowerCAmelCase							)

				for attr in attributes_list:
					setattr(__lowerCAmelCase ,						attr + '''_id''' ,						__lowerCAmelCase							)
					self.assertEqual(getattr(__lowerCAmelCase ,						__lowerCAmelCase							) ,						__lowerCAmelCase							)
					self.assertEqual(getattr(__lowerCAmelCase ,						attr + '''_id'''							) ,						__lowerCAmelCase							)

					setattr(__lowerCAmelCase ,						attr + '''_id''' ,						__lowerCAmelCase							)
					self.assertEqual(getattr(__lowerCAmelCase ,						__lowerCAmelCase							) ,						__lowerCAmelCase							)
					self.assertEqual(getattr(__lowerCAmelCase ,						attr + '''_id'''							) ,						__lowerCAmelCase							)

				setattr(__lowerCAmelCase ,						'''additional_special_tokens_ids''' ,						[]							)
				self.assertListEqual(getattr(__lowerCAmelCase ,						'''additional_special_tokens'''							) ,						[]							)
				self.assertListEqual(getattr(__lowerCAmelCase ,						'''additional_special_tokens_ids'''							) ,						[]							)

				lowerCamelCase__              =	0xE_0_0_6
				lowerCamelCase__              =	chr(__lowerCAmelCase							)
				setattr(__lowerCAmelCase ,						'''additional_special_tokens_ids''' ,						[additional_special_token_id]							)
				self.assertListEqual(getattr(__lowerCAmelCase ,						'''additional_special_tokens'''							) ,						[additional_special_token]							)
				self.assertListEqual(getattr(__lowerCAmelCase ,						'''additional_special_tokens_ids'''							) ,						[additional_special_token_id]							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				pass










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a						=   {
    "configuration_funnel": ["FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP", "FunnelConfig"],
    "convert_funnel_original_tf_checkpoint_to_pytorch": [],
    "tokenization_funnel": ["FunnelTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   ["FunnelTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   [
			    "FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "FunnelBaseModel",
			    "FunnelForMaskedLM",
			    "FunnelForMultipleChoice",
			    "FunnelForPreTraining",
			    "FunnelForQuestionAnswering",
			    "FunnelForSequenceClassification",
			    "FunnelForTokenClassification",
			    "FunnelModel",
			    "FunnelPreTrainedModel",
			    "load_tf_weights_in_funnel",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   [
			    "TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFFunnelBaseModel",
			    "TFFunnelForMaskedLM",
			    "TFFunnelForMultipleChoice",
			    "TFFunnelForPreTraining",
			    "TFFunnelForQuestionAnswering",
			    "TFFunnelForSequenceClassification",
			    "TFFunnelForTokenClassification",
			    "TFFunnelModel",
			    "TFFunnelPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
			from .tokenization_funnel import FunnelTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_funnel_fast import FunnelTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_funnel import (
						    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    FunnelBaseModel,
						    FunnelForMaskedLM,
						    FunnelForMultipleChoice,
						    FunnelForPreTraining,
						    FunnelForQuestionAnswering,
						    FunnelForSequenceClassification,
						    FunnelForTokenClassification,
						    FunnelModel,
						    FunnelPreTrainedModel,
						    load_tf_weights_in_funnel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_funnel import (
						    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFFunnelBaseModel,
						    TFFunnelForMaskedLM,
						    TFFunnelForMultipleChoice,
						    TFFunnelForPreTraining,
						    TFFunnelForQuestionAnswering,
						    TFFunnelForSequenceClassification,
						    TFFunnelForTokenClassification,
						    TFFunnelModel,
						    TFFunnelPreTrainedModel,
						)

else:
			import sys

			_a						=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



