






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase      :      Tuple     						=						get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class        __UpperCamelCase    (     _lowerCAmelCase   ,			unittest.TestCase   ):
	__snake_case	:Any   =       XLNetTokenizer
	__snake_case	:int   =       XLNetTokenizerFast
	__snake_case	:Union[str, Any]   =       True
	__snake_case	:str   =       True







	def  _a   (  self      :							Optional[Any]			)		-> Tuple:
		"""simple docstring"""







		super().setUp()

		# We have a SentencePiece fixture for testing
		__lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				keep_accents=_lowerCAmelCase			)
		tokenizer.sanitize_special_tokens()
		tokenizer.save_pretrained(self.tmpdirname			)







	def  _a   (  self      :							Tuple			)		-> Optional[Any]:
		"""simple docstring"""







		__lowercase             =  """<s>"""
		__lowercase             =  1

		self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase			)       ,				_lowerCAmelCase			)
		self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase			)       ,				_lowerCAmelCase			)







	def  _a   (  self      :							Tuple			)		-> Optional[int]:
		"""simple docstring"""







		__lowercase             =  list(self.get_tokenizer().get_vocab().keys()			)

		self.assertEqual(vocab_keys[0]       ,				"""<unk>"""			)
		self.assertEqual(vocab_keys[1]       ,				"""<s>"""			)
		self.assertEqual(vocab_keys[-1]       ,				"""<eod>"""			)
		self.assertEqual(len(_lowerCAmelCase			)       ,				1006			)







	def  _a   (  self      :							Optional[int]			)		-> List[str]:
		"""simple docstring"""







		self.assertEqual(self.get_tokenizer().vocab_size       ,				1000			)







	def  _a   (  self      :							Optional[int]			)		-> Optional[int]:
		"""simple docstring"""







		__lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				keep_accents=_lowerCAmelCase			)

		__lowercase             =  tokenizer.tokenize("""This is a test"""			)
		self.assertListEqual(_lowerCAmelCase       ,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

		self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase			)       ,				[285, 46, 10, 170, 382]			)

		__lowercase             =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
		self.assertListEqual(
		    _lowerCAmelCase       ,				[
		        SPIECE_UNDERLINE + """I""",
		        SPIECE_UNDERLINE + """was""",
		        SPIECE_UNDERLINE + """b""",
		        """or""",
		        """n""",
		        SPIECE_UNDERLINE + """in""",
		        SPIECE_UNDERLINE + """""",
		        """9""",
		        """2""",
		        """0""",
		        """0""",
		        """0""",
		        """,""",
		        SPIECE_UNDERLINE + """and""",
		        SPIECE_UNDERLINE + """this""",
		        SPIECE_UNDERLINE + """is""",
		        SPIECE_UNDERLINE + """f""",
		        """al""",
		        """s""",
		        """é""",
		        """.""",
		    ]       ,				)
		__lowercase             =  tokenizer.convert_tokens_to_ids(_lowerCAmelCase			)
		self.assertListEqual(_lowerCAmelCase       ,				[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			)

		__lowercase             =  tokenizer.convert_ids_to_tokens(_lowerCAmelCase			)
		self.assertListEqual(
		    _lowerCAmelCase       ,				[
		        SPIECE_UNDERLINE + """I""",
		        SPIECE_UNDERLINE + """was""",
		        SPIECE_UNDERLINE + """b""",
		        """or""",
		        """n""",
		        SPIECE_UNDERLINE + """in""",
		        SPIECE_UNDERLINE + """""",
		        """<unk>""",
		        """2""",
		        """0""",
		        """0""",
		        """0""",
		        """,""",
		        SPIECE_UNDERLINE + """and""",
		        SPIECE_UNDERLINE + """this""",
		        SPIECE_UNDERLINE + """is""",
		        SPIECE_UNDERLINE + """f""",
		        """al""",
		        """s""",
		        """<unk>""",
		        """.""",
		    ]       ,				)







	def  _a   (  self      :							Optional[Any]			)		-> Tuple:
		"""simple docstring"""







		__lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				do_lower_case=_lowerCAmelCase			)
		__lowercase             =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
		self.assertListEqual(
		    _lowerCAmelCase       ,				[
		        SPIECE_UNDERLINE + """""",
		        """i""",
		        SPIECE_UNDERLINE + """was""",
		        SPIECE_UNDERLINE + """b""",
		        """or""",
		        """n""",
		        SPIECE_UNDERLINE + """in""",
		        SPIECE_UNDERLINE + """""",
		        """9""",
		        """2""",
		        """0""",
		        """0""",
		        """0""",
		        """,""",
		        SPIECE_UNDERLINE + """and""",
		        SPIECE_UNDERLINE + """this""",
		        SPIECE_UNDERLINE + """is""",
		        SPIECE_UNDERLINE + """f""",
		        """al""",
		        """se""",
		        """.""",
		    ]       ,				)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)       ,				["""▁he""", """ll""", """o"""]			)







	def  _a   (  self      :							List[Any]			)		-> Tuple:
		"""simple docstring"""







		__lowercase             =  XLNetTokenizer(_lowerCAmelCase       ,				do_lower_case=_lowerCAmelCase			)
		__lowercase             =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
		self.assertListEqual(
		    _lowerCAmelCase       ,				[
		        SPIECE_UNDERLINE + """I""",
		        SPIECE_UNDERLINE + """was""",
		        SPIECE_UNDERLINE + """b""",
		        """or""",
		        """n""",
		        SPIECE_UNDERLINE + """in""",
		        SPIECE_UNDERLINE + """""",
		        """9""",
		        """2""",
		        """0""",
		        """0""",
		        """0""",
		        """,""",
		        SPIECE_UNDERLINE + """and""",
		        SPIECE_UNDERLINE + """this""",
		        SPIECE_UNDERLINE + """is""",
		        SPIECE_UNDERLINE + """f""",
		        """al""",
		        """se""",
		        """.""",
		    ]       ,				)







	@slow
	def  _a   (  self      :							Union[str, Any]			)		-> Any:
		"""simple docstring"""







		__lowercase             =  XLNetTokenizer.from_pretrained("""xlnet-base-cased"""			)

		__lowercase             =  tokenizer.encode("""sequence builders"""       ,				add_special_tokens=_lowerCAmelCase			)
		__lowercase             =  tokenizer.encode("""multi-sequence build"""       ,				add_special_tokens=_lowerCAmelCase			)

		__lowercase             =  tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase			)
		__lowercase             =  tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase       ,				_lowerCAmelCase			)

		assert encoded_sentence == text + [4, 3]
		assert encoded_pair == text + [4] + text_a + [4, 3]







	@slow
	def  _a   (  self      :							List[Any]			)		-> Union[str, Any]:
		"""simple docstring"""







		__lowercase             =  {"""input_ids""": [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
		# fmt: on

		self.tokenizer_integration_test_util(
		    expected_encoding=_lowerCAmelCase       ,				model_name="""xlnet-base-cased"""       ,				revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""       ,				)









import random
import unittest

import torch

from diffusers import IFInpaintingPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin


@skip_mps
class        __UpperCamelCase    (     _lowerCAmelCase   ,			_lowerCAmelCase   ,			unittest.TestCase   ):
	__snake_case	:Any   =       IFInpaintingPipeline
	__snake_case	:str   =       TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {'width', 'height'}
	__snake_case	:Optional[int]   =       TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
	__snake_case	:str   =       PipelineTesterMixin.required_optional_params - {'latents'}







	def  _a   (  self      :							Union[str, Any]			)		-> Tuple:
		"""simple docstring"""







		return self._get_dummy_components()







	def  _a   (  self      :							Optional[int]       ,				_lowerCAmelCase      :							List[str]       ,				_lowerCAmelCase      :							Dict=0			)		-> Any:
		"""simple docstring"""







		if str(_lowerCAmelCase			).startswith("""mps"""			):
			__lowercase             =  torch.manual_seed(_lowerCAmelCase			)
		else:
			__lowercase             =  torch.Generator(device=_lowerCAmelCase			).manual_seed(_lowerCAmelCase			)

		__lowercase             =  floats_tensor((1, 3, 32, 32)       ,				rng=random.Random(_lowerCAmelCase			)			).to(_lowerCAmelCase			)
		__lowercase             =  floats_tensor((1, 3, 32, 32)       ,				rng=random.Random(_lowerCAmelCase			)			).to(_lowerCAmelCase			)

		__lowercase             =  {
		    """prompt""": """A painting of a squirrel eating a burger""",
		    """image""": image,
		    """mask_image""": mask_image,
		    """generator""": generator,
		    """num_inference_steps""": 2,
		    """output_type""": """numpy""",
		}

		return inputs







	@unittest.skipIf(
	    torch_device != """cuda""" or not is_xformers_available()       ,				reason="""XFormers attention is only available with CUDA and `xformers` installed"""       ,				)
	def  _a   (  self      :							Tuple			)		-> Dict:
		"""simple docstring"""







		self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3			)







	def  _a   (  self      :							Union[str, Any]			)		-> Dict:
		"""simple docstring"""







		self._test_save_load_optional_components()







	@unittest.skipIf(torch_device != """cuda"""       ,				reason="""float16 requires CUDA"""			)
	def  _a   (  self      :							List[str]			)		-> Tuple:
		"""simple docstring"""







		super().test_save_load_floataa(expected_max_diff=1e-1			)







	def  _a   (  self      :							Tuple			)		-> Optional[Any]:
		"""simple docstring"""







		self._test_attention_slicing_forward_pass(expected_max_diff=1e-2			)







	def  _a   (  self      :							str			)		-> Optional[int]:
		"""simple docstring"""







		self._test_save_load_local()







	def  _a   (  self      :							List[str]			)		-> List[Any]:
		"""simple docstring"""







		self._test_inference_batch_single_identical(
		    expected_max_diff=1e-2       ,				)

