



import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE           =  get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_    ( _A						,unittest.TestCase						):






						'''simple docstring'''
						a__   =  XLNetTokenizer
						a__   =  XLNetTokenizerFast
						a__   =  True
						a__   =  True







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Union[str, Any] )      ->				Tuple:
									super().setUp()

									# We have a SentencePiece fixture for testing
									A							:					Dict              =						XLNetTokenizer(__lowerCamelCase	,	keep_accents=__lowerCamelCase )
									tokenizer.sanitize_special_tokens()
									tokenizer.save_pretrained(self.tmpdirname )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			List[str] )      ->				Dict:
									A							:					List[Any]              =						"<s>"
									A							:					List[str]              =						1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase )	,	__lowerCamelCase )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase )	,	__lowerCamelCase )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[Any] )      ->				Union[str, Any]:
									A							:					List[str]              =						list(self.get_tokenizer().get_vocab().keys() )

									self.assertEqual(vocab_keys[0]	,	"<unk>" )
									self.assertEqual(vocab_keys[1]	,	"<s>" )
									self.assertEqual(vocab_keys[-1]	,	"<eod>" )
									self.assertEqual(len(__lowerCamelCase )	,	10_06 )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			int )      ->				str:
									self.assertEqual(self.get_tokenizer().vocab_size	,	10_00 )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Any )      ->				Optional[Any]:
									A							:					List[str]              =						XLNetTokenizer(__lowerCamelCase	,	keep_accents=__lowerCamelCase )

									A							:					Optional[Any]              =						tokenizer.tokenize("This is a test" )
									self.assertListEqual(__lowerCamelCase	,	["▁This", "▁is", "▁a", "▁t", "est"] )

									self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase )	,	[2_85, 46, 10, 1_70, 3_82] )

									A							:					Optional[Any]              =						tokenizer.tokenize("I was born in 92000, and this is falsé." )
									self.assertListEqual(
									    __lowerCamelCase	,	[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "é",
									        ".",
									    ]	,	)
									A							:					Tuple              =						tokenizer.convert_tokens_to_ids(__lowerCamelCase )
									self.assertListEqual(__lowerCamelCase	,	[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

									A							:					Dict              =						tokenizer.convert_ids_to_tokens(__lowerCamelCase )
									self.assertListEqual(
									    __lowerCamelCase	,	[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "<unk>",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "<unk>",
									        ".",
									    ]	,	)







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int] )      ->				Tuple:
									A							:					Dict              =						XLNetTokenizer(__lowerCamelCase	,	do_lower_case=__lowerCamelCase )
									A							:					Any              =						tokenizer.tokenize("I was born in 92000, and this is falsé." )
									self.assertListEqual(
									    __lowerCamelCase	,	[
									        SPIECE_UNDERLINE + "",
									        "i",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "se",
									        ".",
									    ]	,	)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )	,	["▁he", "ll", "o"] )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Tuple )      ->				List[Any]:
									A							:					int              =						XLNetTokenizer(__lowerCamelCase	,	do_lower_case=__lowerCamelCase )
									A							:					List[Any]              =						tokenizer.tokenize("I was born in 92000, and this is falsé." )
									self.assertListEqual(
									    __lowerCamelCase	,	[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "se",
									        ".",
									    ]	,	)







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			int )      ->				str:
									A							:					List[Any]              =						XLNetTokenizer.from_pretrained("xlnet-base-cased" )

									A							:					Tuple              =						tokenizer.encode("sequence builders"	,	add_special_tokens=__lowerCamelCase )
									A							:					List[Any]              =						tokenizer.encode("multi-sequence build"	,	add_special_tokens=__lowerCamelCase )

									A							:					Any              =						tokenizer.build_inputs_with_special_tokens(__lowerCamelCase )
									A							:					List[Any]              =						tokenizer.build_inputs_with_special_tokens(__lowerCamelCase	,	__lowerCamelCase )

									assert encoded_sentence == text + [4, 3]
									assert encoded_pair == text + [4] + text_a + [4, 3]







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Dict )      ->				Union[str, Any]:
									# fmt: off
									A							:					str              =						{"input_ids": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=__lowerCamelCase	,	model_name="xlnet-base-cased"	,	revision="c841166438c31ec7ca9a106dee7bb312b73ae511"	,	)




import unittest
from typing import Tuple

import torch

from diffusers.utils import floats_tensor, randn_tensor, torch_all_close, torch_device
from diffusers.utils.testing_utils import require_torch

@require_torch
class 	lowerCamelCase_    :






						'''simple docstring'''







						@property
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int] )      ->				str:
									return self.get_dummy_input()







						@property
						def 			SCREAMING_SNAKE_CASE__		(	self      :			int )      ->				Optional[Any]:
									if self.block_type == "down":
												return (4, 32, 16, 16)
									elif self.block_type == "mid":
												return (4, 32, 32, 32)
									elif self.block_type == "up":
												return (4, 32, 64, 64)

									raise ValueError(F"""'{self.block_type}' is not a supported block_type. Set it to 'up', 'mid', or 'down'.""" )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Dict	,	__lowerCamelCase      :			Union[str, Any]=True	,	__lowerCamelCase      :			int=False	,	__lowerCamelCase      :			int=False	,	__lowerCamelCase      :			Optional[int]=False	,	)      ->				Dict:
									A							:					Optional[Any]              =						4
									A							:					List[str]              =						32
									A							:					Any              =						(32, 32)

									A							:					str              =						torch.manual_seed(0 )
									A							:					int              =						torch.device(__lowerCamelCase )
									A							:					List[str]              =						(batch_size, num_channels) + sizes
									A							:					Dict              =						randn_tensor(__lowerCamelCase	,	generator=__lowerCamelCase	,	device=__lowerCamelCase )
									A							:					int              =						{"hidden_states": hidden_states}

									if include_temb:
												A							:					Any              =						1_28
												A							:					List[str]              =						randn_tensor((batch_size, temb_channels)	,	generator=__lowerCamelCase	,	device=__lowerCamelCase )

									if include_res_hidden_states_tuple:
												A							:					str              =						torch.manual_seed(1 )
												A							:					Tuple              =						(randn_tensor(__lowerCamelCase	,	generator=__lowerCamelCase	,	device=__lowerCamelCase ),)

									if include_encoder_hidden_states:
												A							:					Dict              =						floats_tensor((batch_size, 32, 32) ).to(__lowerCamelCase )

									if include_skip_sample:
												A							:					Optional[int]              =						randn_tensor(((batch_size, 3) + sizes)	,	generator=__lowerCamelCase	,	device=__lowerCamelCase )

									return dummy_input







						def 			SCREAMING_SNAKE_CASE__		(	self      :			List[Any] )      ->				Union[str, Any]:
									A							:					Dict              =						{
									    "in_channels": 32,
									    "out_channels": 32,
									    "temb_channels": 1_28,
									}
									if self.block_type == "up":
												A							:					Dict              =						32

									if self.block_type == "mid":
												init_dict.pop("out_channels" )

									A							:					str              =						self.dummy_input
									return init_dict, inputs_dict







						def 			SCREAMING_SNAKE_CASE__		(	self      :			str	,	__lowerCamelCase      :			Optional[int] )      ->				Union[str, Any]:
									A       , A							:					str              =						self.prepare_init_args_and_inputs_for_common()
									A							:					List[Any]              =						self.block_class(**__lowerCamelCase )
									unet_block.to(__lowerCamelCase )
									unet_block.eval()

									with torch.no_grad():
												A							:					int              =						unet_block(**__lowerCamelCase )

									if isinstance(__lowerCamelCase	,	__lowerCamelCase ):
												A							:					Union[str, Any]              =						output[0]

									self.assertEqual(output.shape	,	self.output_shape )

									A							:					Any              =						output[0, -1, -3:, -3:]
									A							:					Union[str, Any]              =						torch.tensor(__lowerCamelCase ).to(__lowerCamelCase )
									assert torch_all_close(output_slice.flatten()	,	__lowerCamelCase	,	atol=5e-3 )







						@unittest.skipIf(torch_device == "mps"	,	"Training is not supported in mps" )
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Any )      ->				Dict:
									A       , A							:					Tuple              =						self.prepare_init_args_and_inputs_for_common()
									A							:					str              =						self.block_class(**__lowerCamelCase )
									model.to(__lowerCamelCase )
									model.train()
									A							:					Optional[int]              =						model(**__lowerCamelCase )

									if isinstance(__lowerCamelCase	,	__lowerCamelCase ):
												A							:					Optional[Any]              =						output[0]

									A							:					List[str]              =						torch.device(__lowerCamelCase )
									A							:					List[str]              =						randn_tensor(output.shape	,	device=__lowerCamelCase )
									A							:					Dict              =						torch.nn.functional.mse_loss(__lowerCamelCase	,	__lowerCamelCase )
									loss.backward()