




import os
import tempfile
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
		import torch
		from torch import nn

		from transformers import (
		    Adafactor,
		    AdamW,
		    get_constant_schedule,
		    get_constant_schedule_with_warmup,
		    get_cosine_schedule_with_warmup,
		    get_cosine_with_hard_restarts_schedule_with_warmup,
		    get_inverse_sqrt_schedule,
		    get_linear_schedule_with_warmup,
		    get_polynomial_decay_schedule_with_warmup,
		)



def       UpperCAmelCase       (     _snake_case						, _snake_case=10							):
							lowerCAmelCase   =		[]
							for _ in range(_snake_case							):
														lrs.append(scheduler.get_lr()[0]							)
														scheduler.step()
							return lrs



def       UpperCAmelCase       (     _snake_case						, _snake_case=10							):
							lowerCAmelCase   =		[]
							for step in range(_snake_case							):
														lrs.append(scheduler.get_lr()[0]							)
														scheduler.step()
														if step == num_steps // 2:
																					with tempfile.TemporaryDirectory() as tmpdirname:
																												lowerCAmelCase   =		os.path.join(_snake_case						, '''schedule.bin'''							)
																												torch.save(scheduler.state_dict()						, _snake_case							)

																												lowerCAmelCase   =		torch.load(_snake_case							)
																												scheduler.load_state_dict(_snake_case							)
							return lrs


@require_torch
class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_						):
											self.assertEqual(len(UpperCAmelCase_						)       , len(UpperCAmelCase_						)						)
											for a, b in zip(UpperCAmelCase_       , UpperCAmelCase_						):
																		self.assertAlmostEqual(UpperCAmelCase_       , UpperCAmelCase_       , delta=UpperCAmelCase_						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		torch.tensor([0.1, -0.2, -0.1]       , requires_grad=UpperCAmelCase_						)
											lowerCAmelCase   =		torch.tensor([0.4, 0.2, -0.5]						)
											lowerCAmelCase   =		nn.MSELoss()
											# No warmup, constant schedule, no gradient clipping
											lowerCAmelCase   =		AdamW(params=[w]       , lr=2E-1       , weight_decay=0.0						)
											for _ in range(1_00						):
																		lowerCAmelCase   =		criterion(UpperCAmelCase_       , UpperCAmelCase_						)
																		loss.backward()
																		optimizer.step()
																		w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
																		w.grad.zero_()
											self.assertListAlmostEqual(w.tolist()       , [0.4, 0.2, -0.5]       , tol=1E-2						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		torch.tensor([0.1, -0.2, -0.1]       , requires_grad=UpperCAmelCase_						)
											lowerCAmelCase   =		torch.tensor([0.4, 0.2, -0.5]						)
											lowerCAmelCase   =		nn.MSELoss()
											# No warmup, constant schedule, no gradient clipping
											lowerCAmelCase   =		Adafactor(
											    params=[w]       , lr=1E-2       , eps=(1E-3_0, 1E-3)       , clip_threshold=1.0       , decay_rate=-0.8       , betaa=UpperCAmelCase_       , weight_decay=0.0       , relative_step=UpperCAmelCase_       , scale_parameter=UpperCAmelCase_       , warmup_init=UpperCAmelCase_       , )
											for _ in range(10_00						):
																		lowerCAmelCase   =		criterion(UpperCAmelCase_       , UpperCAmelCase_						)
																		loss.backward()
																		optimizer.step()
																		w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
																		w.grad.zero_()
											self.assertListAlmostEqual(w.tolist()       , [0.4, 0.2, -0.5]       , tol=1E-2						)




@require_torch
class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				__a							: Optional[Any]										=nn.Linear(5_0     ,					5_0  ) if is_torch_available() else None
				__a							: Union[str, Any]										=AdamW(m.parameters()     ,					lr=10.0  ) if is_torch_available() else None
				__a							: Any										=1_0



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_=None						):
											self.assertEqual(len(UpperCAmelCase_						)       , len(UpperCAmelCase_						)						)
											for a, b in zip(UpperCAmelCase_       , UpperCAmelCase_						):
																		self.assertAlmostEqual(UpperCAmelCase_       , UpperCAmelCase_       , delta=UpperCAmelCase_       , msg=UpperCAmelCase_						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		{'''num_warmup_steps''': 2, '''num_training_steps''': 10}
											# schedulers doct format
											# function: (sched_args_dict, expected_learning_rates)
											lowerCAmelCase   =		{
											    get_constant_schedule: ({}, [10.0] * self.num_steps),
											    get_constant_schedule_with_warmup: (
											        {'''num_warmup_steps''': 4},
											        [0.0, 2.5, 5.0, 7.5, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],
											    ),
											    get_linear_schedule_with_warmup: (
											        {**common_kwargs},
											        [0.0, 5.0, 10.0, 8.75, 7.5, 6.25, 5.0, 3.75, 2.5, 1.25],
											    ),
											    get_cosine_schedule_with_warmup: (
											        {**common_kwargs},
											        [0.0, 5.0, 10.0, 9.61, 8.53, 6.91, 5.0, 3.08, 1.46, 0.38],
											    ),
											    get_cosine_with_hard_restarts_schedule_with_warmup: (
											        {**common_kwargs, '''num_cycles''': 2},
											        [0.0, 5.0, 10.0, 8.53, 5.0, 1.46, 10.0, 8.53, 5.0, 1.46],
											    ),
											    get_polynomial_decay_schedule_with_warmup: (
											        {**common_kwargs, '''power''': 2.0, '''lr_end''': 1E-7},
											        [0.0, 5.0, 10.0, 7.656, 5.625, 3.906, 2.5, 1.406, 0.625, 0.156],
											    ),
											    get_inverse_sqrt_schedule: (
											        {'''num_warmup_steps''': 2},
											        [0.0, 5.0, 10.0, 8.165, 7.071, 6.325, 5.774, 5.345, 5.0, 4.714],
											    ),
											}

											for scheduler_func, data in scheds.items():
																		lowerCAmelCase   ,     lowerCAmelCase   =		data

																		lowerCAmelCase   =		scheduler_func(self.optimizer       , **UpperCAmelCase_						)
																		self.assertEqual(len([scheduler.get_lr()[0]]						)       , 1						)
																		lowerCAmelCase   =		unwrap_schedule(UpperCAmelCase_       , self.num_steps						)
																		self.assertListAlmostEqual(
																		    UpperCAmelCase_       , UpperCAmelCase_       , tol=1E-2       , msg=F"""failed for {scheduler_func} in normal scheduler"""       , )

																		lowerCAmelCase   =		scheduler_func(self.optimizer       , **UpperCAmelCase_						)
																		if scheduler_func.__name__ != "get_constant_schedule":
																									LambdaScheduleWrapper.wrap_scheduler(UpperCAmelCase_						)  # wrap to test picklability of the schedule
																		lowerCAmelCase   =		unwrap_and_save_reload_schedule(UpperCAmelCase_       , self.num_steps						)
																		self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_       , msg=F"""failed for {scheduler_func} in save and reload"""						)




class 							__UpperCamelCase			:





				'''simple docstring'''



				def __init__( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		fn



				def __call__( self       , *UpperCAmelCase_       , **UpperCAmelCase_						):
											return self.fn(*UpperCAmelCase_       , **UpperCAmelCase_						)






				@classmethod
				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		list(map(self       , scheduler.lr_lambdas						)						)









from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


UpperCAmelCase_						=logging.get_logger(__name__)


class 							__UpperCamelCase			( __UpperCAmelCase     ,					__UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Optional[Any]										="""maskformer-swin"""

				__a							: Optional[int]										={
				    """num_attention_heads""": """num_heads""",
				    """num_hidden_layers""": """num_layers""",
				}



				def __init__( self       , UpperCAmelCase_=2_24       , UpperCAmelCase_=4       , UpperCAmelCase_=3       , UpperCAmelCase_=96       , UpperCAmelCase_=[2, 2, 6, 2]       , UpperCAmelCase_=[3, 6, 12, 24]       , UpperCAmelCase_=7       , UpperCAmelCase_=4.0       , UpperCAmelCase_=True       , UpperCAmelCase_=0.0       , UpperCAmelCase_=0.0       , UpperCAmelCase_=0.1       , UpperCAmelCase_="gelu"       , UpperCAmelCase_=False       , UpperCAmelCase_=0.02       , UpperCAmelCase_=1E-5       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , **UpperCAmelCase_       , ):
											super().__init__(**UpperCAmelCase_						)

											lowerCAmelCase   =		image_size
											lowerCAmelCase   =		patch_size
											lowerCAmelCase   =		num_channels
											lowerCAmelCase   =		embed_dim
											lowerCAmelCase   =		depths
											lowerCAmelCase   =		len(UpperCAmelCase_						)
											lowerCAmelCase   =		num_heads
											lowerCAmelCase   =		window_size
											lowerCAmelCase   =		mlp_ratio
											lowerCAmelCase   =		qkv_bias
											lowerCAmelCase   =		hidden_dropout_prob
											lowerCAmelCase   =		attention_probs_dropout_prob
											lowerCAmelCase   =		drop_path_rate
											lowerCAmelCase   =		hidden_act
											lowerCAmelCase   =		use_absolute_embeddings
											lowerCAmelCase   =		layer_norm_eps
											lowerCAmelCase   =		initializer_range
											# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
											# this indicates the channel dimension after the last stage of the model
											lowerCAmelCase   =		int(embed_dim * 2 ** (len(UpperCAmelCase_						) - 1)						)
											lowerCAmelCase   =		['''stem'''] + [F"""stage{idx}""" for idx in range(1       , len(UpperCAmelCase_						) + 1						)]
											lowerCAmelCase   ,     lowerCAmelCase   =		get_aligned_output_features_output_indices(
											    out_features=UpperCAmelCase_       , out_indices=UpperCAmelCase_       , stage_names=self.stage_names						)



