




"""simple docstring"""






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__						=			get_tests_dir('''fixtures/test_sentencepiece.model''')




@require_sentencepiece
class   a__			(       UpperCamelCase_					,	unittest.TestCase  ):
						snake_case__             = XLMProphetNetTokenizer
						snake_case__             = False
						snake_case__             = True





						def       __UpperCamelCase     (							self      :	Optional[Any])						->       Union[str, Any]:




											"""simple docstring"""







											super().setUp()

											# We have a SentencePiece fixture for testing
											_lowerCAmelCase:Union[str, Any]									=  XLMProphetNetTokenizer(a__     ,keep_accents=a__)
											tokenizer.save_pretrained(self.tmpdirname)





						def       __UpperCamelCase     (							self      :	str)						->       List[Any]:




											"""simple docstring"""







											_lowerCAmelCase:Union[str, Any]									=  '''[PAD]'''
											_lowerCAmelCase:List[Any]									=  0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__)     ,a__)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__)     ,a__)





						def       __UpperCamelCase     (							self      :	int)						->       Optional[int]:




											"""simple docstring"""







											_lowerCAmelCase:Any									=  list(self.get_tokenizer().get_vocab().keys())

											self.assertEqual(vocab_keys[0]     ,'''[PAD]''')
											self.assertEqual(vocab_keys[1]     ,'''[CLS]''')
											self.assertEqual(vocab_keys[-1]     ,'''j''')
											self.assertEqual(len(a__)     ,1012)





						def       __UpperCamelCase     (							self      :	str)						->       int:




											"""simple docstring"""







											self.assertEqual(self.get_tokenizer().vocab_size     ,1012)





						def       __UpperCamelCase     (							self      :	Optional[int])						->       str:




											"""simple docstring"""







											_lowerCAmelCase:Tuple									=  XLMProphetNetTokenizer(a__     ,keep_accents=a__)

											_lowerCAmelCase:Any									=  tokenizer.tokenize('''This is a test''')
											self.assertListEqual(a__     ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(a__)     ,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,)

											_lowerCAmelCase:Optional[Any]									=  tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
											self.assertListEqual(
											    a__     ,[
											        SPIECE_UNDERLINE + '''I''',
											        SPIECE_UNDERLINE + '''was''',
											        SPIECE_UNDERLINE + '''b''',
											        '''or''',
											        '''n''',
											        SPIECE_UNDERLINE + '''in''',
											        SPIECE_UNDERLINE + '''''',
											        '''9''',
											        '''2''',
											        '''0''',
											        '''0''',
											        '''0''',
											        ''',''',
											        SPIECE_UNDERLINE + '''and''',
											        SPIECE_UNDERLINE + '''this''',
											        SPIECE_UNDERLINE + '''is''',
											        SPIECE_UNDERLINE + '''f''',
											        '''al''',
											        '''s''',
											        '''é''',
											        '''.''',
											    ]     ,)
											_lowerCAmelCase:int									=  tokenizer.convert_tokens_to_ids(a__)
											self.assertListEqual(
											    a__     ,[
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
											    ]     ,)

											_lowerCAmelCase:List[Any]									=  tokenizer.convert_ids_to_tokens(a__)
											self.assertListEqual(
											    a__     ,[
											        SPIECE_UNDERLINE + '''I''',
											        SPIECE_UNDERLINE + '''was''',
											        SPIECE_UNDERLINE + '''b''',
											        '''or''',
											        '''n''',
											        SPIECE_UNDERLINE + '''in''',
											        SPIECE_UNDERLINE + '''''',
											        '''[UNK]''',
											        '''2''',
											        '''0''',
											        '''0''',
											        '''0''',
											        ''',''',
											        SPIECE_UNDERLINE + '''and''',
											        SPIECE_UNDERLINE + '''this''',
											        SPIECE_UNDERLINE + '''is''',
											        SPIECE_UNDERLINE + '''f''',
											        '''al''',
											        '''s''',
											        '''[UNK]''',
											        '''.''',
											    ]     ,)





						@cached_property
						def       __UpperCamelCase     (							self      :	str)						->       Optional[int]:




											"""simple docstring"""







											return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased''')





						@slow
						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       List[Any]:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  '''Hello World!'''
											_lowerCAmelCase:Union[str, Any]									=  [3_5389, 6672, 49, 2]
											self.assertListEqual(a__     ,self.big_tokenizer.encode(a__))







						@slow
						def       __UpperCamelCase     (							self      :	Optional[Any])						->       Optional[int]:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  {'''input_ids''': [[1_1073, 8_2783, 18, 26, 8_2783, 549, 5_1540, 248, 1_7209, 1301, 217, 20, 21_5186, 1325, 147, 1_7209, 1301, 217, 20, 5_6370, 53, 12_2020, 20, 1_6477, 27, 8_7355, 4548, 20, 4728, 7_8392, 17, 15_9969, 18, 26, 2_4491, 629, 15, 538, 2_2704, 5439, 15, 2788, 2_4491, 9885, 15, 4_3534, 605, 15, 814, 1_8403, 3_3200, 29, 15, 4_3534, 2_4458, 1_2410, 111, 2_4966, 8_3669, 9637, 14_4068, 26, 850, 2_2346, 27, 147, 2_4966, 8_3669, 8_3490, 26, 3_9113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 12_2020, 11_5785, 34, 816, 1339, 4_6887, 18, 147, 5_3905, 1951, 4_2238, 4_1170, 1_7732, 834, 436, 15, 2_7523, 9_8733, 217, 147, 5542, 4981, 930, 1_7347, 16, 2], [2_0091, 629, 94, 8_2786, 58, 490, 20, 1528, 84, 5_3905, 344, 8_0592, 11_0128, 1_8822, 5267, 1306, 62, 15_2537, 308, 7997, 401, 12_4427, 549, 3_5442, 225, 109, 1_5055, 2_5748, 147, 7119, 4_3712, 34, 767, 13_5366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 6_3784, 11_9466, 17, 14_7808, 8_8214, 18, 656, 81, 32, 3296, 1_0280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=a__     ,model_name='''microsoft/xprophetnet-large-wiki100-cased'''     ,revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''     ,)












"""simple docstring"""






import torch

from diffusers import KDPMaDiscreteScheduler
from diffusers.utils import torch_device

from .test_schedulers import SchedulerCommonTest




class   a__			(       UpperCamelCase_  ):
						snake_case__             = (KDPMaDiscreteScheduler,)
						snake_case__             = 1_0





						def       __UpperCamelCase     (							self      :	Dict     ,**a__      :	str)						->       int:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  {
											    '''num_train_timesteps''': 1100,
											    '''beta_start''': 0.0001,
											    '''beta_end''': 0.02,
											    '''beta_schedule''': '''linear''',
											}

											config.update(**a__)
											return config





						def       __UpperCamelCase     (							self      :	Optional[int])						->       Optional[int]:




											"""simple docstring"""







											for timesteps in [10, 50, 100, 1000]:
																self.check_over_configs(num_train_timesteps=a__)





						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       Optional[int]:




											"""simple docstring"""







											for beta_start, beta_end in zip([0.00001, 0.0001, 0.001]     ,[0.0002, 0.002, 0.02]):
																self.check_over_configs(beta_start=a__     ,beta_end=a__)





						def       __UpperCamelCase     (							self      :	Optional[Any])						->       Union[str, Any]:




											"""simple docstring"""







											for schedule in ["linear", "scaled_linear"]:
																self.check_over_configs(beta_schedule=a__)





						def       __UpperCamelCase     (							self      :	str)						->       str:




											"""simple docstring"""







											for prediction_type in ["epsilon", "v_prediction"]:
																self.check_over_configs(prediction_type=a__)





						def       __UpperCamelCase     (							self      :	Tuple)						->       Tuple:




											"""simple docstring"""







											_lowerCAmelCase:Union[str, Any]									=  self.scheduler_classes[0]
											_lowerCAmelCase:Union[str, Any]									=  self.get_scheduler_config(prediction_type='''v_prediction''')
											_lowerCAmelCase:Tuple									=  scheduler_class(**a__)

											scheduler.set_timesteps(self.num_inference_steps)

											_lowerCAmelCase:Optional[Any]									=  self.dummy_model()
											_lowerCAmelCase:List[Any]									=  self.dummy_sample_deter * scheduler.init_noise_sigma
											_lowerCAmelCase:Optional[Any]									=  sample.to(a__)

											for i, t in enumerate(scheduler.timesteps):
																_lowerCAmelCase:Tuple									=  scheduler.scale_model_input(a__     ,a__)

																_lowerCAmelCase:Any									=  model(a__     ,a__)

																_lowerCAmelCase:Union[str, Any]									=  scheduler.step(a__     ,a__     ,a__)
																_lowerCAmelCase:Optional[int]									=  output.prev_sample

											_lowerCAmelCase:str									=  torch.sum(torch.abs(a__))
											_lowerCAmelCase:Union[str, Any]									=  torch.mean(torch.abs(a__))

											if torch_device in ["cpu", "mps"]:
																assert abs(result_sum.item() - 4.6_934E-07) < 1E-2
																assert abs(result_mean.item() - 6.1_112E-10) < 1E-3
											else:
																# CUDA
																assert abs(result_sum.item() - 4.693_428_650_170_972E-07) < 1E-2
																assert abs(result_mean.item() - 0.0002) < 1E-3





						def       __UpperCamelCase     (							self      :	Tuple)						->       Dict:




											"""simple docstring"""







											if torch_device == "mps":
																return
											_lowerCAmelCase:Tuple									=  self.scheduler_classes[0]
											_lowerCAmelCase:List[str]									=  self.get_scheduler_config()
											_lowerCAmelCase:Union[str, Any]									=  scheduler_class(**a__)

											scheduler.set_timesteps(self.num_inference_steps)

											_lowerCAmelCase:Any									=  self.dummy_model()
											_lowerCAmelCase:str									=  self.dummy_sample_deter * scheduler.init_noise_sigma
											_lowerCAmelCase:List[str]									=  sample.to(a__)

											for i, t in enumerate(scheduler.timesteps):
																_lowerCAmelCase:Tuple									=  scheduler.scale_model_input(a__     ,a__)

																_lowerCAmelCase:List[str]									=  model(a__     ,a__)

																_lowerCAmelCase:int									=  scheduler.step(a__     ,a__     ,a__)
																_lowerCAmelCase:List[Any]									=  output.prev_sample

											_lowerCAmelCase:Tuple									=  torch.sum(torch.abs(a__))
											_lowerCAmelCase:Dict									=  torch.mean(torch.abs(a__))

											if torch_device in ["cpu", "mps"]:
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3
											else:
																# CUDA
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3







						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       List[Any]:




											"""simple docstring"""







											if torch_device == "mps":
																return
											_lowerCAmelCase:str									=  self.scheduler_classes[0]
											_lowerCAmelCase:Optional[Any]									=  self.get_scheduler_config()
											_lowerCAmelCase:Any									=  scheduler_class(**a__)

											scheduler.set_timesteps(self.num_inference_steps     ,device=a__)

											_lowerCAmelCase:Optional[int]									=  self.dummy_model()
											_lowerCAmelCase:Optional[Any]									=  self.dummy_sample_deter.to(a__) * scheduler.init_noise_sigma

											for t in scheduler.timesteps:
																_lowerCAmelCase:Union[str, Any]									=  scheduler.scale_model_input(a__     ,a__)

																_lowerCAmelCase:Tuple									=  model(a__     ,a__)

																_lowerCAmelCase:str									=  scheduler.step(a__     ,a__     ,a__)
																_lowerCAmelCase:Dict									=  output.prev_sample

											_lowerCAmelCase:Any									=  torch.sum(torch.abs(a__))
											_lowerCAmelCase:str									=  torch.mean(torch.abs(a__))

											if str(a__).startswith('''cpu'''):
																# The following sum varies between 148 and 156 on mps. Why?
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3
											else:
																# CUDA
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3






