




"""simple docstring"""






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				UpperCamelCase__						=			'''pt'''
elif is_tf_available():
				UpperCamelCase__						=			'''tf'''
else:
				UpperCamelCase__						=			'''jax'''




class   a__			(       UpperCamelCase_					,	unittest.TestCase  ):
						snake_case__             = PerceiverTokenizer
						snake_case__             = False





						def       __UpperCamelCase     (							self      :	List[Any])						->       Tuple:




											"""simple docstring"""







											super().setUp()
											_lowerCAmelCase:List[str]									=  PerceiverTokenizer()
											tokenizer.save_pretrained(self.tmpdirname)





						@cached_property
						def       __UpperCamelCase     (							self      :	str)						->       Dict:




											"""simple docstring"""







											return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver''')





						def       __UpperCamelCase     (							self      :	List[Any]     ,**a__      :	Optional[int])						->       PerceiverTokenizer:




											"""simple docstring"""







											return self.tokenizer_class.from_pretrained(self.tmpdirname     ,**a__)





						def       __UpperCamelCase     (							self      :	List[str]     ,a__      :	List[Any]     ,a__      :	List[Any]=False     ,a__      :	Tuple=20     ,a__      :	int=5)						->       Tuple[str, list]:




											"""simple docstring"""







											_lowerCAmelCase:Dict									=  []
											for i in range(len(a__)):
																try:
																					_lowerCAmelCase:int									=  tokenizer.decode([i]     ,clean_up_tokenization_spaces=a__)
																except UnicodeDecodeError:
																					pass
																toks.append((i, tok))

											_lowerCAmelCase:Optional[int]									=  list(filter(lambda a__: re.match(R'''^[ a-zA-Z]+$'''     ,t[1])     ,a__))
											_lowerCAmelCase:str									=  list(filter(lambda a__: [t[0]] == tokenizer.encode(t[1]     ,add_special_tokens=a__)     ,a__))
											if max_length is not None and len(a__) > max_length:
																_lowerCAmelCase:Dict									=  toks[:max_length]
											if min_length is not None and len(a__) < min_length and len(a__) > 0:
																while len(a__) < min_length:
																					_lowerCAmelCase:Optional[int]									=  toks + toks
        # toks_str = [t[1] for t in toks]
											_lowerCAmelCase:Optional[Any]									=  [t[0] for t in toks]

											# Ensure consistency
											_lowerCAmelCase:Dict									=  tokenizer.decode(a__     ,clean_up_tokenization_spaces=a__)
											if " " not in output_txt and len(a__) > 1:
																_lowerCAmelCase:Union[str, Any]									=  (
																    tokenizer.decode([toks_ids[0]]     ,clean_up_tokenization_spaces=a__)
																    + ''' '''
																    + tokenizer.decode(toks_ids[1:]     ,clean_up_tokenization_spaces=a__)
																)
											if with_prefix_space:
																_lowerCAmelCase:Optional[int]									=  ''' ''' + output_txt
											_lowerCAmelCase:Optional[Any]									=  tokenizer.encode(a__     ,add_special_tokens=a__)
											return output_txt, output_ids





						def       __UpperCamelCase     (							self      :	List[str])						->       Union[str, Any]:




											"""simple docstring"""







											_lowerCAmelCase:Any									=  self.perceiver_tokenizer
											_lowerCAmelCase:int									=  '''Unicode €.'''
											_lowerCAmelCase:int									=  tokenizer(a__)
											_lowerCAmelCase:Any									=  [4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
											self.assertEqual(encoded['''input_ids''']     ,a__)

											# decoding
											_lowerCAmelCase:List[Any]									=  tokenizer.decode(a__)
											self.assertEqual(a__     ,'''[CLS]Unicode €.[SEP]''')

											_lowerCAmelCase:Tuple									=  tokenizer('''e è é ê ë''')
											_lowerCAmelCase:str									=  [4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
											self.assertEqual(encoded['''input_ids''']     ,a__)
											# decoding
											_lowerCAmelCase:List[str]									=  tokenizer.decode(a__)
											self.assertEqual(a__     ,'''[CLS]e è é ê ë[SEP]''')

											# encode/decode, but with `encode` instead of `__call__`
											self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''))     ,'''[CLS]e è é ê ë[SEP]''')





						def       __UpperCamelCase     (							self      :	str)						->       Dict:




											"""simple docstring"""







											_lowerCAmelCase:List[str]									=  self.perceiver_tokenizer
											_lowerCAmelCase:Tuple									=  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
											# fmt: off
											_lowerCAmelCase:List[Any]									=  [4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
											# fmt: on
											_lowerCAmelCase:str									=  tokenizer(a__     ,padding=a__     ,return_tensors=a__)
											self.assertIsInstance(a__     ,a__)

											if FRAMEWORK != "jax":
																_lowerCAmelCase:Any									=  list(batch.input_ids.numpy()[0])
											else:
																_lowerCAmelCase:Optional[int]									=  list(batch.input_ids.tolist()[0])

											self.assertListEqual(a__     ,a__)

											self.assertEqual((2, 38)     ,batch.input_ids.shape)
											self.assertEqual((2, 38)     ,batch.attention_mask.shape)





						def       __UpperCamelCase     (							self      :	Optional[int])						->       Optional[int]:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  self.perceiver_tokenizer
											_lowerCAmelCase:Optional[int]									=  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
											_lowerCAmelCase:Dict									=  tokenizer(a__     ,padding=a__     ,return_tensors=a__)
											# check if input_ids are returned and no decoder_input_ids
											self.assertIn('''input_ids'''     ,a__)
											self.assertIn('''attention_mask'''     ,a__)
											self.assertNotIn('''decoder_input_ids'''     ,a__)
											self.assertNotIn('''decoder_attention_mask'''     ,a__)





						def       __UpperCamelCase     (							self      :	Dict)						->       Tuple:




											"""simple docstring"""







											_lowerCAmelCase:str									=  self.perceiver_tokenizer
											_lowerCAmelCase:Any									=  [
											    '''Summary of the text.''',
											    '''Another summary.''',
											]
											_lowerCAmelCase:Union[str, Any]									=  tokenizer(
											    text_target=a__     ,max_length=32     ,padding='''max_length'''     ,truncation=a__     ,return_tensors=a__)
											self.assertEqual(32     ,targets['''input_ids'''].shape[1])





						def       __UpperCamelCase     (							self      :	Dict)						->       int:




											"""simple docstring"""







											_lowerCAmelCase:Tuple									=  self.get_tokenizers()
											for tokenizer in tokenizers:
																with self.subTest(F'{tokenizer.__class__.__name__}'):
																					self.assertNotEqual(tokenizer.model_max_length     ,42)

        # Now let's start the test
											_lowerCAmelCase:Dict									=  self.get_tokenizers()
											for tokenizer in tokenizers:
																with self.subTest(F'{tokenizer.__class__.__name__}'):
																					# Isolate this from the other tests because we save additional tokens/etc
																					_lowerCAmelCase:Union[str, Any]									=  tempfile.mkdtemp()

																					_lowerCAmelCase:Optional[int]									=  ''' He is very happy, UNwant\u00E9d,running'''
																					_lowerCAmelCase:Optional[Any]									=  tokenizer.encode(a__     ,add_special_tokens=a__)
																					tokenizer.save_pretrained(a__)

																					_lowerCAmelCase:List[str]									=  tokenizer.__class__.from_pretrained(a__)
																					_lowerCAmelCase:List[str]									=  after_tokenizer.encode(a__     ,add_special_tokens=a__)
																					self.assertListEqual(a__     ,a__)

																					shutil.rmtree(a__)

											_lowerCAmelCase:Optional[int]									=  self.get_tokenizers(model_max_length=42)
											for tokenizer in tokenizers:
																with self.subTest(F'{tokenizer.__class__.__name__}'):
																					# Isolate this from the other tests because we save additional tokens/etc
																					_lowerCAmelCase:Optional[Any]									=  tempfile.mkdtemp()

																					_lowerCAmelCase:Optional[Any]									=  ''' He is very happy, UNwant\u00E9d,running'''
																					tokenizer.add_tokens(['''bim''', '''bambam'''])
																					_lowerCAmelCase:str									=  tokenizer.additional_special_tokens
																					additional_special_tokens.append('''new_additional_special_token''')
																					tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens})
																					_lowerCAmelCase:Optional[int]									=  tokenizer.encode(a__     ,add_special_tokens=a__)
																					tokenizer.save_pretrained(a__)

																					_lowerCAmelCase:Optional[int]									=  tokenizer.__class__.from_pretrained(a__)
																					_lowerCAmelCase:List[str]									=  after_tokenizer.encode(a__     ,add_special_tokens=a__)
																					self.assertListEqual(a__     ,a__)
																					self.assertIn('''new_additional_special_token'''     ,after_tokenizer.additional_special_tokens)
																					self.assertEqual(after_tokenizer.model_max_length     ,42)

																					_lowerCAmelCase:Any									=  tokenizer.__class__.from_pretrained(a__     ,model_max_length=43)
																					self.assertEqual(tokenizer.model_max_length     ,43)

																					shutil.rmtree(a__)





						def       __UpperCamelCase     (							self      :	Optional[int])						->       List[Any]:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  []
											if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

											if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																					tokenizer_utils.save_pretrained(a__)

																					with open(os.path.join(a__     ,'''special_tokens_map.json''')     ,encoding='''utf-8''') as json_file:
																										_lowerCAmelCase:Optional[Any]									=  json.load(a__)

																					with open(os.path.join(a__     ,'''tokenizer_config.json''')     ,encoding='''utf-8''') as json_file:
																										_lowerCAmelCase:Optional[Any]									=  json.load(a__)

																					_lowerCAmelCase:Dict									=  [F'<extra_id_{i}>' for i in range(125)]

																					_lowerCAmelCase:str									=  added_tokens_extra_ids + [
																					    '''an_additional_special_token'''
																					]
																					_lowerCAmelCase:str									=  added_tokens_extra_ids + [
																					    '''an_additional_special_token'''
																					]

																					with open(os.path.join(a__     ,'''special_tokens_map.json''')     ,'''w'''     ,encoding='''utf-8''') as outfile:
																										json.dump(a__     ,a__)
																					with open(os.path.join(a__     ,'''tokenizer_config.json''')     ,'''w'''     ,encoding='''utf-8''') as outfile:
																										json.dump(a__     ,a__)

																					# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																					# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																					# "special_tokens_map.json" files
																					_lowerCAmelCase:List[Any]									=  tokenizer_class.from_pretrained(
																					    a__     ,)
																					self.assertIn(
																					    '''an_additional_special_token'''     ,tokenizer_without_change_in_init.additional_special_tokens)
																					self.assertEqual(
																					    ['''an_additional_special_token''']     ,tokenizer_without_change_in_init.convert_ids_to_tokens(
																					        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']))     ,)

																					# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																					_lowerCAmelCase:str									=  added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''     ,lstrip=a__)]
																					_lowerCAmelCase:str									=  tokenizer_class.from_pretrained(
																					    a__     ,additional_special_tokens=a__     ,)

																					self.assertIn('''a_new_additional_special_token'''     ,tokenizer.additional_special_tokens)
																					self.assertEqual(
																					    ['''a_new_additional_special_token''']     ,tokenizer.convert_ids_to_tokens(
																					        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']))     ,)





						def       __UpperCamelCase     (							self      :	Optional[Any])						->       Optional[int]:




											"""simple docstring"""







											_lowerCAmelCase:Any									=  self.perceiver_tokenizer
											self.assertEqual(tokenizer.decode([178])     ,'''�''')





						def       __UpperCamelCase     (							self      :	Dict)						->       Tuple:




											"""simple docstring"""







											pass





						def       __UpperCamelCase     (							self      :	str)						->       Dict:




											"""simple docstring"""







											pass





						def       __UpperCamelCase     (							self      :	str)						->       List[Any]:




											"""simple docstring"""







											pass





						def       __UpperCamelCase     (							self      :	Optional[int])						->       List[str]:




											"""simple docstring"""







											pass







						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       Tuple:




											"""simple docstring"""







											_lowerCAmelCase:List[str]									=  self.get_tokenizers(fast=a__     ,do_lower_case=a__)
											for tokenizer in tokenizers:
																with self.subTest(F'{tokenizer.__class__.__name__}'):
																					_lowerCAmelCase:Dict									=  ['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
																					_lowerCAmelCase:Tuple									=  tokenizer.convert_tokens_to_string(a__)

																					self.assertIsInstance(a__     ,a__)












"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__						=			{
    '''configuration_convbert''': ['''CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ConvBertConfig''', '''ConvBertOnnxConfig'''],
    '''tokenization_convbert''': ['''ConvBertTokenizer'''],
}

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				UpperCamelCase__						=			['''ConvBertTokenizerFast''']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				UpperCamelCase__						=			[
				    '''CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''ConvBertForMaskedLM''',
				    '''ConvBertForMultipleChoice''',
				    '''ConvBertForQuestionAnswering''',
				    '''ConvBertForSequenceClassification''',
				    '''ConvBertForTokenClassification''',
				    '''ConvBertLayer''',
				    '''ConvBertModel''',
				    '''ConvBertPreTrainedModel''',
				    '''load_tf_weights_in_convbert''',
				]


try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				UpperCamelCase__						=			[
				    '''TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''TFConvBertForMaskedLM''',
				    '''TFConvBertForMultipleChoice''',
				    '''TFConvBertForQuestionAnswering''',
				    '''TFConvBertForSequenceClassification''',
				    '''TFConvBertForTokenClassification''',
				    '''TFConvBertLayer''',
				    '''TFConvBertModel''',
				    '''TFConvBertPreTrainedModel''',
				]


if TYPE_CHECKING:
				from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
				from .tokenization_convbert import ConvBertTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_convbert_fast import ConvBertTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_convbert import (
								    CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    ConvBertForMaskedLM,
								    ConvBertForMultipleChoice,
								    ConvBertForQuestionAnswering,
								    ConvBertForSequenceClassification,
								    ConvBertForTokenClassification,
								    ConvBertLayer,
								    ConvBertModel,
								    ConvBertPreTrainedModel,
								    load_tf_weights_in_convbert,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_convbert import (
								    TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFConvBertForMaskedLM,
								    TFConvBertForMultipleChoice,
								    TFConvBertForQuestionAnswering,
								    TFConvBertForSequenceClassification,
								    TFConvBertForTokenClassification,
								    TFConvBertLayer,
								    TFConvBertModel,
								    TFConvBertPreTrainedModel,
								)


else:
				import sys

				UpperCamelCase__						=			_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






