




import torch

from diffusers import KDPMaDiscreteScheduler
from diffusers.utils import torch_device

from .test_schedulers import SchedulerCommonTest




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     (KDPMaDiscreteScheduler,)
							a__						  =     1_0





							def    _UpperCAmelCase     (					self       :	Optional[Any]       ,     **a       :	Union[str, Any]			)			->					List[str]:
										SCREAMING_SNAKE_CASE           =					{
										    """num_train_timesteps""": 1_100,
										    """beta_start""": 0.0001,
										    """beta_end""": 0.02,
										    """beta_schedule""": """linear""",
										}

										config.update(**a			)
										return config





							def    _UpperCAmelCase     (					self       :	List[Any]			)			->					Optional[Any]:
										for timesteps in [10, 50, 100, 1_000]:
													self.check_over_configs(num_train_timesteps=a			)





							def    _UpperCAmelCase     (					self       :	List[Any]			)			->					Dict:
										for beta_start, beta_end in zip([0.0_0001, 0.0001, 0.001]       ,     [0.0002, 0.002, 0.02]			):
													self.check_over_configs(beta_start=a       ,     beta_end=a			)





							def    _UpperCAmelCase     (					self       :	str			)			->					Union[str, Any]:
										for schedule in ["linear", "scaled_linear"]:
													self.check_over_configs(beta_schedule=a			)





							def    _UpperCAmelCase     (					self       :	Tuple			)			->					Optional[int]:
										for prediction_type in ["epsilon", "v_prediction"]:
													self.check_over_configs(prediction_type=a			)





							def    _UpperCAmelCase     (					self       :	List[str]			)			->					int:
										SCREAMING_SNAKE_CASE           =					self.scheduler_classes[0]
										SCREAMING_SNAKE_CASE           =					self.get_scheduler_config(prediction_type="""v_prediction"""			)
										SCREAMING_SNAKE_CASE           =					scheduler_class(**a			)

										scheduler.set_timesteps(self.num_inference_steps			)

										SCREAMING_SNAKE_CASE           =					self.dummy_model()
										SCREAMING_SNAKE_CASE           =					self.dummy_sample_deter * scheduler.init_noise_sigma
										SCREAMING_SNAKE_CASE           =					sample.to(a			)

										for i, t in enumerate(scheduler.timesteps			):
													SCREAMING_SNAKE_CASE           =					scheduler.scale_model_input(a       ,     a			)

													SCREAMING_SNAKE_CASE           =					model(a       ,     a			)

													SCREAMING_SNAKE_CASE           =					scheduler.step(a       ,     a       ,     a			)
													SCREAMING_SNAKE_CASE           =					output.prev_sample

										SCREAMING_SNAKE_CASE           =					torch.sum(torch.abs(a			)			)
										SCREAMING_SNAKE_CASE           =					torch.mean(torch.abs(a			)			)

										if torch_device in ["cpu", "mps"]:
													assert abs(result_sum.item() - 4.6_9_3_4E-0_7			) < 1E-2
													assert abs(result_mean.item() - 6.1_1_1_2E-1_0			) < 1E-3
										else:
													# CUDA
													assert abs(result_sum.item() - 4.6_9_3_4_2_8_6_5_0_1_7_0_9_7_2E-0_7			) < 1E-2
													assert abs(result_mean.item() - 0.0002			) < 1E-3





							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					str:
										if torch_device == "mps":
													return
										SCREAMING_SNAKE_CASE           =					self.scheduler_classes[0]
										SCREAMING_SNAKE_CASE           =					self.get_scheduler_config()
										SCREAMING_SNAKE_CASE           =					scheduler_class(**a			)

										scheduler.set_timesteps(self.num_inference_steps			)

										SCREAMING_SNAKE_CASE           =					self.dummy_model()
										SCREAMING_SNAKE_CASE           =					self.dummy_sample_deter * scheduler.init_noise_sigma
										SCREAMING_SNAKE_CASE           =					sample.to(a			)

										for i, t in enumerate(scheduler.timesteps			):
													SCREAMING_SNAKE_CASE           =					scheduler.scale_model_input(a       ,     a			)

													SCREAMING_SNAKE_CASE           =					model(a       ,     a			)

													SCREAMING_SNAKE_CASE           =					scheduler.step(a       ,     a       ,     a			)
													SCREAMING_SNAKE_CASE           =					output.prev_sample

										SCREAMING_SNAKE_CASE           =					torch.sum(torch.abs(a			)			)
										SCREAMING_SNAKE_CASE           =					torch.mean(torch.abs(a			)			)

										if torch_device in ["cpu", "mps"]:
													assert abs(result_sum.item() - 20.4125			) < 1E-2
													assert abs(result_mean.item() - 0.0266			) < 1E-3
										else:
													# CUDA
													assert abs(result_sum.item() - 20.4125			) < 1E-2
													assert abs(result_mean.item() - 0.0266			) < 1E-3







							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					int:
										if torch_device == "mps":
													return
										SCREAMING_SNAKE_CASE           =					self.scheduler_classes[0]
										SCREAMING_SNAKE_CASE           =					self.get_scheduler_config()
										SCREAMING_SNAKE_CASE           =					scheduler_class(**a			)

										scheduler.set_timesteps(self.num_inference_steps       ,     device=a			)

										SCREAMING_SNAKE_CASE           =					self.dummy_model()
										SCREAMING_SNAKE_CASE           =					self.dummy_sample_deter.to(a			) * scheduler.init_noise_sigma

										for t in scheduler.timesteps:
													SCREAMING_SNAKE_CASE           =					scheduler.scale_model_input(a       ,     a			)

													SCREAMING_SNAKE_CASE           =					model(a       ,     a			)

													SCREAMING_SNAKE_CASE           =					scheduler.step(a       ,     a       ,     a			)
													SCREAMING_SNAKE_CASE           =					output.prev_sample

										SCREAMING_SNAKE_CASE           =					torch.sum(torch.abs(a			)			)
										SCREAMING_SNAKE_CASE           =					torch.mean(torch.abs(a			)			)

										if str(a			).startswith("""cpu"""			):
													# The following sum varies between 148 and 156 on mps. Why?
													assert abs(result_sum.item() - 20.4125			) < 1E-2
													assert abs(result_mean.item() - 0.0266			) < 1E-3
										else:
													# CUDA
													assert abs(result_sum.item() - 20.4125			) < 1E-2
													assert abs(result_mean.item() - 0.0266			) < 1E-3












from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A						:							Any							=  logging.get_logger(__name__)

__A						:							int							=  {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     '''megatron-bert'''





							def __init__(					self       :	str       ,     a       :	Union[str, Any]=29_056       ,     a       :	Optional[int]=1_024       ,     a       :	str=24       ,     a       :	str=16       ,     a       :	str=4_096       ,     a       :	Any="gelu"       ,     a       :	Any=0.1       ,     a       :	Optional[int]=0.1       ,     a       :	Optional[int]=512       ,     a       :	Optional[Any]=2       ,     a       :	List[str]=0.02       ,     a       :	List[Any]=1E-1_2       ,     a       :	Any=0       ,     a       :	List[Any]="absolute"       ,     a       :	List[str]=True       ,     **a       :	Optional[Any]       ,     )			->					List[str]:
										super().__init__(pad_token_id=a       ,     **a			)

										SCREAMING_SNAKE_CASE           =					vocab_size
										SCREAMING_SNAKE_CASE           =					hidden_size
										SCREAMING_SNAKE_CASE           =					num_hidden_layers
										SCREAMING_SNAKE_CASE           =					num_attention_heads
										SCREAMING_SNAKE_CASE           =					hidden_act
										SCREAMING_SNAKE_CASE           =					intermediate_size
										SCREAMING_SNAKE_CASE           =					hidden_dropout_prob
										SCREAMING_SNAKE_CASE           =					attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE           =					max_position_embeddings
										SCREAMING_SNAKE_CASE           =					type_vocab_size
										SCREAMING_SNAKE_CASE           =					initializer_range
										SCREAMING_SNAKE_CASE           =					layer_norm_eps
										SCREAMING_SNAKE_CASE           =					position_embedding_type
										SCREAMING_SNAKE_CASE           =					use_cache






