
import collections
import inspect
import unittest

from transformers import SwinvaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import SwinvaForImageClassification, SwinvaForMaskedImageModeling, SwinvaModel
    from transformers.models.swinva.modeling_swinva import SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor




class 				_lowerCamelCase :





     """simple docstring"""







     def __init__(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE=1_3						,						__SCREAMING_SNAKE_CASE=3_2						,						__SCREAMING_SNAKE_CASE=2						,						__SCREAMING_SNAKE_CASE=3						,						__SCREAMING_SNAKE_CASE=1_6						,						__SCREAMING_SNAKE_CASE=[1, 2, 1]						,						__SCREAMING_SNAKE_CASE=[2, 2, 4]						,						__SCREAMING_SNAKE_CASE=2						,						__SCREAMING_SNAKE_CASE=2.0						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=0.0						,						__SCREAMING_SNAKE_CASE=0.0						,						__SCREAMING_SNAKE_CASE=0.1						,						__SCREAMING_SNAKE_CASE="gelu"						,						__SCREAMING_SNAKE_CASE=False						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=0.02						,						__SCREAMING_SNAKE_CASE=1e-5						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=1_0						,						__SCREAMING_SNAKE_CASE=8						,						)  ->   Optional[int]:




           """simple docstring"""







           UpperCamelCase__    :   int					=				parent
           UpperCamelCase__    :   Any					=				batch_size
           UpperCamelCase__    :   Tuple					=				image_size
           UpperCamelCase__    :   List[str]					=				patch_size
           UpperCamelCase__    :   Union[str, Any]					=				num_channels
           UpperCamelCase__    :   List[Any]					=				embed_dim
           UpperCamelCase__    :   Union[str, Any]					=				depths
           UpperCamelCase__    :   int					=				num_heads
           UpperCamelCase__    :   Any					=				window_size
           UpperCamelCase__    :   Optional[int]					=				mlp_ratio
           UpperCamelCase__    :   Union[str, Any]					=				qkv_bias
           UpperCamelCase__    :   str					=				hidden_dropout_prob
           UpperCamelCase__    :   Union[str, Any]					=				attention_probs_dropout_prob
           UpperCamelCase__    :   Dict					=				drop_path_rate
           UpperCamelCase__    :   Any					=				hidden_act
           UpperCamelCase__    :   List[Any]					=				use_absolute_embeddings
           UpperCamelCase__    :   Union[str, Any]					=				patch_norm
           UpperCamelCase__    :   Optional[int]					=				layer_norm_eps
           UpperCamelCase__    :   List[Any]					=				initializer_range
           UpperCamelCase__    :   Optional[Any]					=				is_training
           UpperCamelCase__    :   Union[str, Any]					=				scope
           UpperCamelCase__    :   List[str]					=				use_labels
           UpperCamelCase__    :   Optional[int]					=				type_sequence_label_size
           UpperCamelCase__    :   List[str]					=				encoder_stride






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]  )

           UpperCamelCase__    :   Tuple					=				None
           if self.use_labels:
                 UpperCamelCase__    :   Dict					=				ids_tensor([self.batch_size]						,						self.type_sequence_label_size  )

           UpperCamelCase__    :   List[Any]					=				self.get_config()

           return config, pixel_values, labels






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   int:




           """simple docstring"""







           return SwinvaConfig(
               image_size=self.image_size						,						patch_size=self.patch_size						,						num_channels=self.num_channels						,						embed_dim=self.embed_dim						,						depths=self.depths						,						num_heads=self.num_heads						,						window_size=self.window_size						,						mlp_ratio=self.mlp_ratio						,						qkv_bias=self.qkv_bias						,						hidden_dropout_prob=self.hidden_dropout_prob						,						attention_probs_dropout_prob=self.attention_probs_dropout_prob						,						drop_path_rate=self.drop_path_rate						,						hidden_act=self.hidden_act						,						use_absolute_embeddings=self.use_absolute_embeddings						,						path_norm=self.patch_norm						,						layer_norm_eps=self.layer_norm_eps						,						initializer_range=self.initializer_range						,						encoder_stride=self.encoder_stride						,						)






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   Optional[int]:




           """simple docstring"""







           UpperCamelCase__    :   Union[str, Any]					=				SwinvaModel(config=__SCREAMING_SNAKE_CASE  )
           model.to(__SCREAMING_SNAKE_CASE  )
           model.eval()
           UpperCamelCase__    :   str					=				model(__SCREAMING_SNAKE_CASE  )

           UpperCamelCase__    :   Union[str, Any]					=				((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths  ) - 1))
           UpperCamelCase__    :   List[str]					=				int(config.embed_dim * 2 ** (len(config.depths  ) - 1)  )

           self.parent.assertEqual(result.last_hidden_state.shape						,						(self.batch_size, expected_seq_len, expected_dim)  )






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   Any:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				SwinvaForMaskedImageModeling(config=__SCREAMING_SNAKE_CASE  )
           model.to(__SCREAMING_SNAKE_CASE  )
           model.eval()
           UpperCamelCase__    :   int					=				model(__SCREAMING_SNAKE_CASE  )
           self.parent.assertEqual(
               result.logits.shape						,						(self.batch_size, self.num_channels, self.image_size, self.image_size)  )

           # test greyscale images
           UpperCamelCase__    :   Union[str, Any]					=				1
           UpperCamelCase__    :   List[Any]					=				SwinvaForMaskedImageModeling(__SCREAMING_SNAKE_CASE  )
           model.to(__SCREAMING_SNAKE_CASE  )
           model.eval()

           UpperCamelCase__    :   Union[str, Any]					=				floats_tensor([self.batch_size, 1, self.image_size, self.image_size]  )
           UpperCamelCase__    :   Optional[Any]					=				model(__SCREAMING_SNAKE_CASE  )
           self.parent.assertEqual(result.logits.shape						,						(self.batch_size, 1, self.image_size, self.image_size)  )






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   Optional[Any]:




           """simple docstring"""







           UpperCamelCase__    :   Tuple					=				self.type_sequence_label_size
           UpperCamelCase__    :   Tuple					=				SwinvaForImageClassification(__SCREAMING_SNAKE_CASE  )
           model.to(__SCREAMING_SNAKE_CASE  )
           model.eval()
           UpperCamelCase__    :   Union[str, Any]					=				model(__SCREAMING_SNAKE_CASE						,						labels=__SCREAMING_SNAKE_CASE  )
           self.parent.assertEqual(result.logits.shape						,						(self.batch_size, self.type_sequence_label_size)  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				self.prepare_config_and_inputs()
           UpperCamelCase__				,UpperCamelCase__				,UpperCamelCase__    :   List[str]					=				config_and_inputs
           UpperCamelCase__    :   str					=				{'''pixel_values''': pixel_values}
           return config, inputs_dict




@require_torch
class 				_lowerCamelCase (      UpperCamelCase_	,							UpperCamelCase_	,							unittest.TestCase ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  (
         (SwinvaModel, SwinvaForImageClassification, SwinvaForMaskedImageModeling) if is_torch_available() else ()
     )
     SCREAMING_SNAKE_CASE_		   =  (
         {"feature-extraction": SwinvaModel, "image-classification": SwinvaForImageClassification}
         if is_torch_available()
         else {}
     )

     SCREAMING_SNAKE_CASE_		   =  False
     SCREAMING_SNAKE_CASE_		   =  False
     SCREAMING_SNAKE_CASE_		   =  False
     SCREAMING_SNAKE_CASE_		   =  False






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[str]:




           """simple docstring"""







           UpperCamelCase__    :   Tuple					=				SwinvaModelTester(self  )
           UpperCamelCase__    :   Dict					=				ConfigTester(self						,						config_class=__SCREAMING_SNAKE_CASE						,						embed_dim=3_7  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           self.config_tester.create_and_test_config_to_json_string()
           self.config_tester.create_and_test_config_to_json_file()
           self.config_tester.create_and_test_config_from_and_save_pretrained()
           self.config_tester.create_and_test_config_with_num_labels()
           self.config_tester.check_config_can_be_init_without_params()
           self.config_tester.check_config_arguments_init()






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   Any					=				self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE  )






     @unittest.skip(reason='''Got `CUDA error: misaligned address` with PyTorch 2.0.0.'''  )
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Union[str, Any]:




           """simple docstring"""







           pass






     @unittest.skip(reason='''Swinv2 does not use inputs_embeds'''  )
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           pass






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   str					=				self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 UpperCamelCase__    :   List[str]					=				model_class(__SCREAMING_SNAKE_CASE  )
                 self.assertIsInstance(model.get_input_embeddings()						,						(nn.Module)  )
                 UpperCamelCase__    :   Tuple					=				model.get_output_embeddings()
                 self.assertTrue(x is None or isinstance(__SCREAMING_SNAKE_CASE						,						nn.Linear  )  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Optional[Any]:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   int					=				self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 UpperCamelCase__    :   Optional[Any]					=				model_class(__SCREAMING_SNAKE_CASE  )
                 UpperCamelCase__    :   Optional[int]					=				inspect.signature(model.forward  )
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 UpperCamelCase__    :   Any					=				[*signature.parameters.keys()]

                 UpperCamelCase__    :   List[Any]					=				['''pixel_values''']
                 self.assertListEqual(arg_names[:1]						,						__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Union[str, Any]:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   Optional[int]					=				self.model_tester.prepare_config_and_inputs_for_common()
           UpperCamelCase__    :   Union[str, Any]					=				True

           for model_class in self.all_model_classes:
                 UpperCamelCase__    :   Tuple					=				True
                 UpperCamelCase__    :   List[Any]					=				False
                 UpperCamelCase__    :   Optional[Any]					=				True
                 UpperCamelCase__    :   Union[str, Any]					=				model_class(__SCREAMING_SNAKE_CASE  )
                 model.to(__SCREAMING_SNAKE_CASE  )
                 model.eval()
                 with torch.no_grad():
                       UpperCamelCase__    :   Tuple					=				model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  )
                 UpperCamelCase__    :   Tuple					=				outputs.attentions
                 UpperCamelCase__    :   Any					=				len(self.model_tester.depths  )
                 self.assertEqual(len(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )

                 # check that output_attentions also work using config
                 del inputs_dict["output_attentions"]
                 UpperCamelCase__    :   List[str]					=				True
                 UpperCamelCase__    :   Optional[int]					=				config.window_size**2
                 UpperCamelCase__    :   Tuple					=				model_class(__SCREAMING_SNAKE_CASE  )
                 model.to(__SCREAMING_SNAKE_CASE  )
                 model.eval()
                 with torch.no_grad():
                       UpperCamelCase__    :   List[str]					=				model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  )
                 UpperCamelCase__    :   Optional[Any]					=				outputs.attentions
                 self.assertEqual(len(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )

                 self.assertListEqual(
                     list(attentions[0].shape[-3:]  )						,						[self.model_tester.num_heads[0], window_size_squared, window_size_squared]						,						)
                 UpperCamelCase__    :   Dict					=				len(__SCREAMING_SNAKE_CASE  )

                 # Check attention is always last and order is fine
                 UpperCamelCase__    :   List[Any]					=				True
                 UpperCamelCase__    :   Optional[Any]					=				True
                 UpperCamelCase__    :   Optional[Any]					=				model_class(__SCREAMING_SNAKE_CASE  )
                 model.to(__SCREAMING_SNAKE_CASE  )
                 model.eval()
                 with torch.no_grad():
                       UpperCamelCase__    :   Dict					=				model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  )

                 if hasattr(self.model_tester						,						'''num_hidden_states_types'''  ):
                       UpperCamelCase__    :   Union[str, Any]					=				self.model_tester.num_hidden_states_types
                 else:
                       # also another +1 for reshaped_hidden_states
                       UpperCamelCase__    :   Optional[int]					=				2
                 self.assertEqual(out_len + added_hidden_states						,						len(__SCREAMING_SNAKE_CASE  )  )

                 UpperCamelCase__    :   Dict					=				outputs.attentions

                 self.assertEqual(len(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )

                 self.assertListEqual(
                     list(self_attentions[0].shape[-3:]  )						,						[self.model_tester.num_heads[0], window_size_squared, window_size_squared]						,						)






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__    :   str					=				model_class(__SCREAMING_SNAKE_CASE  )
           model.to(__SCREAMING_SNAKE_CASE  )
           model.eval()

           with torch.no_grad():
                 UpperCamelCase__    :   Any					=				model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  )

           UpperCamelCase__    :   Optional[Any]					=				outputs.hidden_states

           UpperCamelCase__    :   Tuple					=				getattr(
               self.model_tester						,						'''expected_num_hidden_layers'''						,						len(self.model_tester.depths  ) + 1  )
           self.assertEqual(len(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )

           # Swinv2 has a different seq_length
           UpperCamelCase__    :   List[str]					=				(
               config.patch_size
               if isinstance(config.patch_size						,						collections.abc.Iterable  )
               else (config.patch_size, config.patch_size)
           )

           UpperCamelCase__    :   int					=				(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

           self.assertListEqual(
               list(hidden_states[0].shape[-2:]  )						,						[num_patches, self.model_tester.embed_dim]						,						)

           UpperCamelCase__    :   Any					=				outputs.reshaped_hidden_states
           self.assertEqual(len(__SCREAMING_SNAKE_CASE  )						,						__SCREAMING_SNAKE_CASE  )

           UpperCamelCase__				,UpperCamelCase__				,UpperCamelCase__				,UpperCamelCase__    :   Optional[int]					=				reshaped_hidden_states[0].shape
           UpperCamelCase__    :   List[str]					=				(
               reshaped_hidden_states[0].view(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						height * width  ).permute(0						,						2						,						1  )
           )
           self.assertListEqual(
               list(reshaped_hidden_states.shape[-2:]  )						,						[num_patches, self.model_tester.embed_dim]						,						)






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   Tuple					=				self.model_tester.prepare_config_and_inputs_for_common()

           UpperCamelCase__    :   Union[str, Any]					=				(
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size						,						collections.abc.Iterable  )
               else (self.model_tester.image_size, self.model_tester.image_size)
           )

           for model_class in self.all_model_classes:
                 UpperCamelCase__    :   Any					=				True
                 self.check_hidden_states_output(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )

                 # check that output_hidden_states also work using config
                 del inputs_dict["output_hidden_states"]
                 UpperCamelCase__    :   Any					=				True

                 self.check_hidden_states_output(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Optional[int]:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   List[str]					=				self.model_tester.prepare_config_and_inputs_for_common()
           UpperCamelCase__    :   int					=				3

           UpperCamelCase__    :   int					=				(
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size						,						collections.abc.Iterable  )
               else (self.model_tester.image_size, self.model_tester.image_size)
           )
           UpperCamelCase__    :   Tuple					=				(
               config.patch_size
               if isinstance(config.patch_size						,						collections.abc.Iterable  )
               else (config.patch_size, config.patch_size)
           )

           UpperCamelCase__    :   List[Any]					=				image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
           UpperCamelCase__    :   str					=				image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

           for model_class in self.all_model_classes:
                 UpperCamelCase__    :   Optional[Any]					=				True
                 self.check_hidden_states_output(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						(padded_height, padded_width)  )

                 # check that output_hidden_states also work using config
                 del inputs_dict["output_hidden_states"]
                 UpperCamelCase__    :   Union[str, Any]					=				True
                 self.check_hidden_states_output(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						(padded_height, padded_width)  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__    :   Tuple					=				self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_image_modeling(*__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[str]:




           """simple docstring"""







           UpperCamelCase__    :   Any					=				self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_image_classification(*__SCREAMING_SNAKE_CASE  )






     @slow
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           for model_name in SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 UpperCamelCase__    :   Optional[Any]					=				SwinvaModel.from_pretrained(__SCREAMING_SNAKE_CASE  )
                 self.assertIsNotNone(__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   int:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   Optional[int]					=				self.model_tester.prepare_config_and_inputs_for_common()

           UpperCamelCase__    :   Union[str, Any]					=				_config_zero_init(__SCREAMING_SNAKE_CASE  )
           for model_class in self.all_model_classes:
                 UpperCamelCase__    :   Optional[Any]					=				model_class(config=__SCREAMING_SNAKE_CASE  )
                 for name, param in model.named_parameters():
                       if "embeddings" not in name and "logit_scale" not in name and param.requires_grad:
                             self.assertIn(
                                 ((param.data.mean() * 1e9).round() / 1e9).item()						,						[0.0, 1.0]						,						msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''						,						)




@require_vision
@require_torch
class 				_lowerCamelCase (      unittest.TestCase ):





     """simple docstring"""







     @cached_property
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           return (
               AutoImageProcessor.from_pretrained('''microsoft/swinv2-tiny-patch4-window8-256'''  )
               if is_vision_available()
               else None
           )






     @slow
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__    :   Any					=				SwinvaForImageClassification.from_pretrained('''microsoft/swinv2-tiny-patch4-window8-256'''  ).to(
               __SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Any					=				self.default_image_processor

           UpperCamelCase__    :   int					=				Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''  )
           UpperCamelCase__    :   int					=				image_processor(images=__SCREAMING_SNAKE_CASE						,						return_tensors='''pt'''  ).to(__SCREAMING_SNAKE_CASE  )

           # forward pass
           with torch.no_grad():
                 UpperCamelCase__    :   str					=				model(**__SCREAMING_SNAKE_CASE  )

           # verify the logits
           UpperCamelCase__    :   str					=				torch.Size((1, 1_0_0_0)  )
           self.assertEqual(outputs.logits.shape						,						__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Tuple					=				torch.tensor([-0.3947, -0.4306, 0.0026]  ).to(__SCREAMING_SNAKE_CASE  )
           self.assertTrue(torch.allclose(outputs.logits[0, :3]						,						__SCREAMING_SNAKE_CASE						,						atol=1e-4  )  )




import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel




class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  '''M-CLIP'''






     def __init__(							self						,						__SCREAMING_SNAKE_CASE=1_0_2_4						,						__SCREAMING_SNAKE_CASE=7_6_8						,						**__SCREAMING_SNAKE_CASE  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   Dict					=				transformerDimSize
           UpperCamelCase__    :   Any					=				imageDimSize
           super().__init__(**__SCREAMING_SNAKE_CASE  )




class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  MCLIPConfig






     def __init__(							self						,						__SCREAMING_SNAKE_CASE						,						*__SCREAMING_SNAKE_CASE						,						**__SCREAMING_SNAKE_CASE  )  ->   Optional[int]:




           """simple docstring"""







           super().__init__(__SCREAMING_SNAKE_CASE						,						*__SCREAMING_SNAKE_CASE						,						**__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Union[str, Any]					=				XLMRobertaModel(__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Tuple					=				torch.nn.Linear(
               in_features=config.transformerDimensions						,						out_features=config.numDims  )






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   List[str]:




           """simple docstring"""







           UpperCamelCase__    :   Tuple					=				self.transformer(input_ids=__SCREAMING_SNAKE_CASE						,						attention_mask=__SCREAMING_SNAKE_CASE  )[0]
           UpperCamelCase__    :   Tuple					=				(embs * attention_mask.unsqueeze(2  )).sum(dim=1  ) / attention_mask.sum(dim=1  )[:, None]
           return self.LinearTransformation(__SCREAMING_SNAKE_CASE  ), embs


