
"""simple docstring"""



import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case          =						get_tests_dir('''fixtures/test_sentencepiece.model''')
_snake_case          =						get_tests_dir('''fixtures/test_sentencepiece_bpe.model''')

_snake_case          =						"""pt""" if is_torch_available() else """tf"""
@require_sentencepiece
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  SCREAMING_SNAKE_CASE__      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							int							=					CamembertTokenizer
				SCREAMING_SNAKE_CASE_:							List[str]							=					CamembertTokenizerFast
				SCREAMING_SNAKE_CASE_:							Tuple							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					True


				def 				__lowerCamelCase	(					self	:			Dict  )		->  Optional[int]:







											"""simple docstring"""




											super().setUp()

											# We have a SentencePiece fixture for testing
											_lowerCAmelCase  	=			CamembertTokenizer(snake_case__  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			str  )		->  List[str]:







											"""simple docstring"""




											_lowerCAmelCase  	=			"<pad>"
											_lowerCAmelCase  	=			1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__  )				,     snake_case__  )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__  )				,     snake_case__  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  Dict:







											"""simple docstring"""




											_lowerCAmelCase  	=			list(self.get_tokenizer().get_vocab().keys()  )

											self.assertEqual(vocab_keys[0]				,     '<s>NOTUSED'  )
											self.assertEqual(vocab_keys[1]				,     '<pad>'  )
											self.assertEqual(vocab_keys[-1]				,     '<mask>'  )
											self.assertEqual(len(snake_case__  )				,     1_004  )


				def 				__lowerCamelCase	(					self	:			int  )		->  str:







											"""simple docstring"""




											self.assertEqual(self.get_tokenizer().vocab_size				,     1_005  )


				def 				__lowerCamelCase	(					self	:			str  )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			CamembertTokenizer(snake_case__  )
											tokenizer.save_pretrained(self.tmpdirname  )
											_lowerCAmelCase  	=			CamembertTokenizerFast.from_pretrained(self.tmpdirname  )

											_lowerCAmelCase  	=			"I was born in 92000, and this is falsé."

											_lowerCAmelCase  	=			tokenizer.encode(snake_case__  )
											_lowerCAmelCase  	=			rust_tokenizer.encode(snake_case__  )
											self.assertListEqual(snake_case__				,     snake_case__  )

											_lowerCAmelCase  	=			tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__  )
											_lowerCAmelCase  	=			rust_tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__  )
											self.assertListEqual(snake_case__				,     snake_case__  )

											# <unk> tokens are not the same for `rust` than for `slow`.
											# Because spm gives back raw token instead of `unk` in EncodeAsPieces
											# tokens = tokenizer.tokenize(sequence)
											_lowerCAmelCase  	=			tokenizer.convert_ids_to_tokens(snake_case__  )
											_lowerCAmelCase  	=			rust_tokenizer.tokenize(snake_case__  )
											self.assertListEqual(snake_case__				,     snake_case__  )


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											if not self.test_rust_tokenizer:
																		return

											_lowerCAmelCase  	=			self.get_tokenizer()
											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			"I was born in 92000, and this is falsé."

											_lowerCAmelCase  	=			tokenizer.tokenize(snake_case__  )
											_lowerCAmelCase  	=			rust_tokenizer.tokenize(snake_case__  )
											self.assertListEqual(snake_case__				,     snake_case__  )

											_lowerCAmelCase  	=			tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__  )
											_lowerCAmelCase  	=			rust_tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__  )
											self.assertListEqual(snake_case__				,     snake_case__  )

											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			tokenizer.encode(snake_case__  )
											_lowerCAmelCase  	=			rust_tokenizer.encode(snake_case__  )
											self.assertListEqual(snake_case__				,     snake_case__  )







				@slow
				def 				__lowerCamelCase	(					self	:			int  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			{"input_ids": [[5, 54, 7_196, 297, 30, 23, 776, 18, 11, 3_215, 3_705, 8_252, 22, 3_164, 1_181, 2_116, 29, 16, 813, 25, 791, 3_314, 20, 3_446, 38, 27_575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9_088, 20, 1_517, 8, 22_804, 18_818, 10, 38, 629, 607, 607, 142, 19, 7_196, 867, 56, 10_326, 24, 2_267, 20, 416, 5_072, 15_612, 233, 734, 7, 2_399, 27, 16, 3_015, 1_649, 7, 24, 20, 4_338, 2_399, 27, 13, 3_400, 14, 13, 6_189, 8, 930, 9, 6]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
											# fmt: on

											# camembert is a french model. So we also use french texts.
											_lowerCAmelCase  	=			[
											    "Le transformeur est un modèle d'apprentissage profond introduit en 2017, "
											    "utilisé principalement dans le domaine du traitement automatique des langues (TAL).",
											    "À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus "
											    "pour gérer des données séquentielles, telles que le langage naturel, pour des tâches "
											    "telles que la traduction et la synthèse de texte.",
											]

											self.tokenizer_integration_test_util(
											    expected_encoding=snake_case__				,     model_name='camembert-base'				,     revision='3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf'				,     sequences=snake_case__				,     )








"""simple docstring"""



def 	__snake_case  (    SCREAMING_SNAKE_CASE:					list[int]						):






							"""simple docstring"""



							_lowerCAmelCase  	=			[]
							if len(SCREAMING_SNAKE_CASE						) == 1:
														return [nums.copy()]
							for _ in range(len(SCREAMING_SNAKE_CASE						)						):
														_lowerCAmelCase  	=			nums.pop(0						)
														_lowerCAmelCase  	=			permute(SCREAMING_SNAKE_CASE						)
														for perm in permutations:
																					perm.append(SCREAMING_SNAKE_CASE						)
														result.extend(SCREAMING_SNAKE_CASE						)
														nums.append(SCREAMING_SNAKE_CASE						)
							return result






def 	__snake_case  (    SCREAMING_SNAKE_CASE:					Any						):






							"""simple docstring"""



							def backtrack(SCREAMING_SNAKE_CASE:					Tuple						):
														if start == len(SCREAMING_SNAKE_CASE						) - 1:
																					output.append(nums[:]						)
														else:
																					for i in range(SCREAMING_SNAKE_CASE     ,  len(SCREAMING_SNAKE_CASE						)						):
																												_lowerCAmelCase	,		_lowerCAmelCase  	=			nums[i], nums[start]
																												backtrack(start + 1						)
																												_lowerCAmelCase	,		_lowerCAmelCase  	=			nums[i], nums[start]  # backtrack

							_lowerCAmelCase  	=			[]
							backtrack(0						)
							return output


if __name__ == "__main__":
	import doctest

	# use res to print the data in permute2 function
	_snake_case          =						permutea([1, 2, 3])
	print(res)
	doctest.testmod()






