



'''simple docstring'''

import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
		import optax
		from flax.training.common_utils import onehot

		from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
		from transformers.models.ta.modeling_flax_ta import shift_tokens_right




@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 		lowerCamelCase__      (						unittest.TestCase     ):




					'''simple docstring'''



					@slow
					def 						__UpperCAmelCase     (  self				:	Optional[Any]					)  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		FlaxMTaForConditionalGeneration.from_pretrained('google/mt5-small'					)
						_lowercase					:       Dict       =		AutoTokenizer.from_pretrained('google/mt5-small'					)

						_lowercase					:       Any       =		tokenizer('Hello there'     ,  return_tensors='np'					).input_ids
						_lowercase					:       Any       =		tokenizer('Hi I am'     ,  return_tensors='np'					).input_ids

						_lowercase					:       Union[str, Any]       =		shift_tokens_right(UpperCamelCase_     ,  model.config.pad_token_id     ,  model.config.decoder_start_token_id					)

						_lowercase					:       List[Any]       =		model(UpperCamelCase_     ,  decoder_input_ids=UpperCamelCase_					).logits
						_lowercase					:       Dict       =		optax.softmax_cross_entropy(UpperCamelCase_     ,  onehot(UpperCamelCase_     ,  logits.shape[-1]					)					).mean()

						_lowercase					:       Tuple       =		-(labels.shape[-1] * loss.item())

						_lowercase					:       Optional[int]       =		-84.91_27
						self.assertTrue(abs(mtf_score - EXPECTED_SCORE					) < 1E-4					)





'''simple docstring'''

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		_A				:    List[str]     						='''pt'''
elif is_tf_available():
		_A				:    Tuple     						='''tf'''
else:
		_A				:    Optional[int]     						='''jax'''



class 		lowerCamelCase__      (						A	,   unittest.TestCase     ):




					'''simple docstring'''



					A_						    =						ByTaTokenizer
					A_						    =						False




					def 						__UpperCAmelCase     (  self				:	Tuple					)  ->       Union[str, Any]:



						'''simple docstring'''





						super().setUp()
						_lowercase					:       Any       =		ByTaTokenizer()
						tokenizer.save_pretrained(self.tmpdirname					)




					@cached_property
					def 						__UpperCAmelCase     (  self				:	int					)  ->       int:



						'''simple docstring'''





						return ByTaTokenizer.from_pretrained('google/byt5-small'					)




					def 						__UpperCAmelCase     (  self				:	int     ,  **UpperCamelCase_				:	List[Any]					)  ->       ByTaTokenizer:



						'''simple docstring'''





						return self.tokenizer_class.from_pretrained(self.tmpdirname     ,  **UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Union[str, Any]=False     ,  UpperCamelCase_				:	Tuple=20     ,  UpperCamelCase_				:	Optional[int]=5					)  ->       Tuple[str, list]:



						'''simple docstring'''





						_lowercase					:       Dict       =		[]
						for i in range(len(UpperCamelCase_					)					):
							try:
								_lowercase					:       List[Any]       =		tokenizer.decode([i]     ,  clean_up_tokenization_spaces=UpperCamelCase_					)
							except UnicodeDecodeError:
								pass
							toks.append((i, tok)					)

						_lowercase					:       Optional[Any]       =		list(filter(lambda UpperCamelCase_					: re.match(r'^[ a-zA-Z]+$'     ,  t[1]					)     ,  UpperCamelCase_					)					)
						_lowercase					:       List[Any]       =		list(filter(lambda UpperCamelCase_					: [t[0]] == tokenizer.encode(t[1]     ,  add_special_tokens=UpperCamelCase_					)     ,  UpperCamelCase_					)					)
						if max_length is not None and len(UpperCamelCase_					) > max_length:
							_lowercase					:       List[Any]       =		toks[:max_length]
						if min_length is not None and len(UpperCamelCase_					) < min_length and len(UpperCamelCase_					) > 0:
							while len(UpperCamelCase_					) < min_length:
								_lowercase					:       Tuple       =		toks + toks
        # toks_str = [t[1] for t in toks]
						_lowercase					:       Dict       =		[t[0] for t in toks]

						# Ensure consistency
						_lowercase					:       Any       =		tokenizer.decode(UpperCamelCase_     ,  clean_up_tokenization_spaces=UpperCamelCase_					)
						if " " not in output_txt and len(UpperCamelCase_					) > 1:
							_lowercase					:       Any       =		(
							    tokenizer.decode([toks_ids[0]]     ,  clean_up_tokenization_spaces=UpperCamelCase_					)
							    + ' '
							    + tokenizer.decode(toks_ids[1:]     ,  clean_up_tokenization_spaces=UpperCamelCase_					)
							)
						if with_prefix_space:
							_lowercase					:       Union[str, Any]       =		' ' + output_txt
						_lowercase					:       int       =		tokenizer.encode(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_					)
						return output_txt, output_ids




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       int:



						'''simple docstring'''





						_lowercase					:       List[str]       =		self.ta_base_tokenizer
						_lowercase					:       Union[str, Any]       =		tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']					)
						_lowercase					:       Tuple       =		tokenizer(['hi', 'I went to the gym', '']					)
						self.assertListEqual(batch_with_eos_added['input_ids']     ,  batch_without_eos_added['input_ids']					)




					def 						__UpperCAmelCase     (  self				:	Tuple					)  ->       Union[str, Any]:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		self.ta_base_tokenizer
						_lowercase					:       Tuple       =		'Unicode €.'
						_lowercase					:       List[Any]       =		tokenizer(UpperCamelCase_					)
						_lowercase					:       List[Any]       =		[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
						self.assertEqual(encoded['input_ids']     ,  UpperCamelCase_					)

						# decoding
						_lowercase					:       List[str]       =		tokenizer.decode(UpperCamelCase_					)
						self.assertEqual(UpperCamelCase_     ,  'Unicode €.</s>'					)

						_lowercase					:       Any       =		tokenizer('e è é ê ë'					)
						_lowercase					:       Optional[int]       =		[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
						self.assertEqual(encoded['input_ids']     ,  UpperCamelCase_					)
						# decoding
						_lowercase					:       Tuple       =		tokenizer.decode(UpperCamelCase_					)
						self.assertEqual(UpperCamelCase_     ,  'e è é ê ë</s>'					)

						# encode/decode, but with `encode` instead of `__call__`
						self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'					)					)     ,  'e è é ê ë</s>'					)




					def 						__UpperCAmelCase     (  self				:	Tuple					)  ->       List[str]:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		self.ta_base_tokenizer
						_lowercase					:       int       =		['A long paragraph for summarization.', 'Another paragraph for summarization.']
						# fmt: off
						_lowercase					:       Any       =		[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
						# fmt: on
						_lowercase					:       Dict       =		tokenizer(UpperCamelCase_     ,  padding=UpperCamelCase_     ,  return_tensors=UpperCamelCase_					)
						self.assertIsInstance(UpperCamelCase_     ,  UpperCamelCase_					)

						if FRAMEWORK != "jax":
							_lowercase					:       Optional[Any]       =		list(batch.input_ids.numpy()[0]					)
						else:
							_lowercase					:       List[str]       =		list(batch.input_ids.tolist()[0]					)

						self.assertListEqual(UpperCamelCase_     ,  UpperCamelCase_					)

						self.assertEqual((2, 37)     ,  batch.input_ids.shape					)
						self.assertEqual((2, 37)     ,  batch.attention_mask.shape					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       str:



						'''simple docstring'''





						_lowercase					:       Union[str, Any]       =		self.ta_base_tokenizer
						_lowercase					:       List[str]       =		['A long paragraph for summarization.', 'Another paragraph for summarization.']
						_lowercase					:       str       =		tokenizer(UpperCamelCase_     ,  padding=UpperCamelCase_     ,  return_tensors=UpperCamelCase_					)
						# check if input_ids are returned and no decoder_input_ids
						self.assertIn('input_ids'     ,  UpperCamelCase_					)
						self.assertIn('attention_mask'     ,  UpperCamelCase_					)
						self.assertNotIn('decoder_input_ids'     ,  UpperCamelCase_					)
						self.assertNotIn('decoder_attention_mask'     ,  UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Any					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Tuple       =		self.ta_base_tokenizer
						_lowercase					:       Optional[Any]       =		[
						    'Summary of the text.',
						    'Another summary.',
						]
						_lowercase					:       str       =		tokenizer(
						    text_target=UpperCamelCase_     ,  max_length=32     ,  padding='max_length'     ,  truncation=UpperCamelCase_     ,  return_tensors=UpperCamelCase_					)
						self.assertEqual(32     ,  targets['input_ids'].shape[1]					)




					def 						__UpperCAmelCase     (  self				:	Dict					)  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       str       =		self.ta_base_tokenizer
						_lowercase					:       str       =		['A long paragraph for summarization. </s>']
						_lowercase					:       Optional[int]       =		['Summary of the text. </s>']
						# fmt: off
						_lowercase					:       Optional[Any]       =		[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
						_lowercase					:       Optional[Any]       =		[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
						# fmt: on

						_lowercase					:       Any       =		tokenizer(UpperCamelCase_     ,  text_target=UpperCamelCase_					)

						self.assertEqual(UpperCamelCase_     ,  batch['input_ids'][0]					)
						self.assertEqual(UpperCamelCase_     ,  batch['labels'][0]					)




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Dict       =		self.get_tokenizers()
						for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
								self.assertNotEqual(tokenizer.model_max_length     ,  42					)

        # Now let's start the test
						_lowercase					:       List[Any]       =		self.get_tokenizers()
						for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
								# Isolate this from the other tests because we save additional tokens/etc
								_lowercase					:       List[Any]       =		tempfile.mkdtemp()

								_lowercase					:       Any       =		' He is very happy, UNwant\u00E9d,running'
								_lowercase					:       Union[str, Any]       =		tokenizer.encode(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_					)
								tokenizer.save_pretrained(UpperCamelCase_					)

								_lowercase					:       Optional[int]       =		tokenizer.__class__.from_pretrained(UpperCamelCase_					)
								_lowercase					:       Tuple       =		after_tokenizer.encode(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_					)
								self.assertListEqual(UpperCamelCase_     ,  UpperCamelCase_					)

								shutil.rmtree(UpperCamelCase_					)

						_lowercase					:       str       =		self.get_tokenizers(model_max_length=42					)
						for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
								# Isolate this from the other tests because we save additional tokens/etc
								_lowercase					:       Dict       =		tempfile.mkdtemp()

								_lowercase					:       List[Any]       =		' He is very happy, UNwant\u00E9d,running'
								tokenizer.add_tokens(['bim', 'bambam']					)
								_lowercase					:       Optional[int]       =		tokenizer.additional_special_tokens
								additional_special_tokens.append('new_additional_special_token'					)
								tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}					)
								_lowercase					:       Optional[Any]       =		tokenizer.encode(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_					)
								tokenizer.save_pretrained(UpperCamelCase_					)

								_lowercase					:       List[str]       =		tokenizer.__class__.from_pretrained(UpperCamelCase_					)
								_lowercase					:       Dict       =		after_tokenizer.encode(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_					)
								self.assertListEqual(UpperCamelCase_     ,  UpperCamelCase_					)
								self.assertIn('new_additional_special_token'     ,  after_tokenizer.additional_special_tokens					)
								self.assertEqual(after_tokenizer.model_max_length     ,  42					)

								_lowercase					:       Dict       =		tokenizer.__class__.from_pretrained(UpperCamelCase_     ,  model_max_length=43					)
								self.assertEqual(tokenizer.model_max_length     ,  43					)

								shutil.rmtree(UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		[]
						if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

						if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

						for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer_utils.save_pretrained(UpperCamelCase_					)

								with open(os.path.join(UpperCamelCase_     ,  'special_tokens_map.json'					)     ,  encoding='utf-8'					) as json_file:
									_lowercase					:       int       =		json.load(UpperCamelCase_					)

								with open(os.path.join(UpperCamelCase_     ,  'tokenizer_config.json'					)     ,  encoding='utf-8'					) as json_file:
									_lowercase					:       Tuple       =		json.load(UpperCamelCase_					)

								_lowercase					:       List[Any]       =		[F'''<extra_id_{i}>''' for i in range(125					)]

								_lowercase					:       Any       =		added_tokens_extra_ids + [
								    'an_additional_special_token'
								]
								_lowercase					:       int       =		added_tokens_extra_ids + [
								    'an_additional_special_token'
								]

								with open(os.path.join(UpperCamelCase_     ,  'special_tokens_map.json'					)     ,  'w'     ,  encoding='utf-8'					) as outfile:
									json.dump(UpperCamelCase_     ,  UpperCamelCase_					)
								with open(os.path.join(UpperCamelCase_     ,  'tokenizer_config.json'					)     ,  'w'     ,  encoding='utf-8'					) as outfile:
									json.dump(UpperCamelCase_     ,  UpperCamelCase_					)

								# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
								# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
								# "special_tokens_map.json" files
								_lowercase					:       Optional[Any]       =		tokenizer_class.from_pretrained(
								    UpperCamelCase_     ,  )
								self.assertIn(
								    'an_additional_special_token'     ,  tokenizer_without_change_in_init.additional_special_tokens					)
								# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
								self.assertEqual(
								    ['an_additional_special_token']     ,  tokenizer_without_change_in_init.convert_ids_to_tokens(
								        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']					)					)     ,  )

								# Now we test that we can change the value of additional_special_tokens in the from_pretrained
								_lowercase					:       List[str]       =		added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'     ,  lstrip=UpperCamelCase_					)]
								_lowercase					:       Tuple       =		tokenizer_class.from_pretrained(
								    UpperCamelCase_     ,  additional_special_tokens=UpperCamelCase_     ,  )

								self.assertIn('a_new_additional_special_token'     ,  tokenizer.additional_special_tokens					)
								self.assertEqual(
								    ['a_new_additional_special_token']     ,  tokenizer.convert_ids_to_tokens(
								        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']					)					)     ,  )




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       str:



						'''simple docstring'''





						_lowercase					:       Union[str, Any]       =		[]
						if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

						if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

						for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer_utils.save_pretrained(UpperCamelCase_					)

								_lowercase					:       str       =		tokenizer_class.from_pretrained(UpperCamelCase_					)

								self.assertTrue(tokenizer.decode([255]					) == ''					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       int:



						'''simple docstring'''





						pass




					def 						__UpperCAmelCase     (  self				:	str					)  ->       Tuple:



						'''simple docstring'''





						pass




					def 						__UpperCAmelCase     (  self				:	List[Any]					)  ->       List[Any]:



						'''simple docstring'''





						pass




					def 						__UpperCAmelCase     (  self				:	List[Any]					)  ->       int:



						'''simple docstring'''





						pass




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       Optional[Any]:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		self.get_tokenizers(fast=UpperCamelCase_     ,  do_lower_case=UpperCamelCase_					)
						for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
								_lowercase					:       Any       =		['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
								_lowercase					:       Tuple       =		tokenizer.convert_tokens_to_string(UpperCamelCase_					)

								self.assertIsInstance(UpperCamelCase_     ,  UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	List[Any]					)  ->       str:



						'''simple docstring'''





						_lowercase					:       Dict       =		self.get_tokenizers()
						for tokenizer in tokenizers:
							with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
								_lowercase					:       Optional[int]       =		[
								    'bos_token',
								    'eos_token',
								    'unk_token',
								    'sep_token',
								    'pad_token',
								    'cls_token',
								    'mask_token',
								]

								_lowercase					:       Optional[int]       =		0
								_lowercase					:       int       =		tokenizer.convert_ids_to_tokens(
								    UpperCamelCase_     ,  skip_special_tokens=UpperCamelCase_					)

								for attr in attributes_list:
									setattr(UpperCamelCase_     ,  attr + '_id'     ,  UpperCamelCase_					)
									self.assertEqual(getattr(UpperCamelCase_     ,  UpperCamelCase_					)     ,  UpperCamelCase_					)
									self.assertEqual(getattr(UpperCamelCase_     ,  attr + '_id'					)     ,  UpperCamelCase_					)

									setattr(UpperCamelCase_     ,  attr + '_id'     ,  UpperCamelCase_					)
									self.assertEqual(getattr(UpperCamelCase_     ,  UpperCamelCase_					)     ,  UpperCamelCase_					)
									self.assertEqual(getattr(UpperCamelCase_     ,  attr + '_id'					)     ,  UpperCamelCase_					)

								setattr(UpperCamelCase_     ,  'additional_special_tokens_ids'     ,  []					)
								self.assertListEqual(getattr(UpperCamelCase_     ,  'additional_special_tokens'					)     ,  []					)
								self.assertListEqual(getattr(UpperCamelCase_     ,  'additional_special_tokens_ids'					)     ,  []					)

								setattr(UpperCamelCase_     ,  'additional_special_tokens_ids'     ,  [token_id_to_test_setters]					)
								self.assertListEqual(getattr(UpperCamelCase_     ,  'additional_special_tokens'					)     ,  [token_to_test_setters]					)
								self.assertListEqual(getattr(UpperCamelCase_     ,  'additional_special_tokens_ids'					)     ,  [token_id_to_test_setters]					)
