






'''simple docstring'''



import collections
import inspect
import unittest

from transformers import SwinvaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch
      from torch import nn

      from transformers import SwinvaForImageClassification, SwinvaForMaskedImageModeling, SwinvaModel
      from transformers.models.swinva.modeling_swinva import SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor
class UpperCAmelCase_					:




      """simple docstring"""




      def __init__( self			:		Any     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Any=13     ,      UpperCAmelCase			:		Optional[int]=32     ,      UpperCAmelCase			:		Dict=2     ,      UpperCAmelCase			:		int=3     ,      UpperCAmelCase			:		Tuple=16     ,      UpperCAmelCase			:		str=[1, 2, 1]     ,      UpperCAmelCase			:		Optional[Any]=[2, 2, 4]     ,      UpperCAmelCase			:		Optional[int]=2     ,      UpperCAmelCase			:		Union[str, Any]=2.0     ,      UpperCAmelCase			:		List[str]=True     ,      UpperCAmelCase			:		Dict=0.0     ,      UpperCAmelCase			:		Any=0.0     ,      UpperCAmelCase			:		str=0.1     ,      UpperCAmelCase			:		Optional[Any]="gelu"     ,      UpperCAmelCase			:		Union[str, Any]=False     ,      UpperCAmelCase			:		Union[str, Any]=True     ,      UpperCAmelCase			:		List[Any]=0.0_2     ,      UpperCAmelCase			:		str=1e-5     ,      UpperCAmelCase			:		Union[str, Any]=True     ,      UpperCAmelCase			:		str=None     ,      UpperCAmelCase			:		Optional[int]=True     ,      UpperCAmelCase			:		Dict=10     ,      UpperCAmelCase			:		Tuple=8     ,      )  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   Any				  =parent
       lowercase					:   int				  =batch_size
       lowercase					:   List[Any]				  =image_size
       lowercase					:   int				  =patch_size
       lowercase					:   List[Any]				  =num_channels
       lowercase					:   Dict				  =embed_dim
       lowercase					:   List[Any]				  =depths
       lowercase					:   List[Any]				  =num_heads
       lowercase					:   int				  =window_size
       lowercase					:   Optional[Any]				  =mlp_ratio
       lowercase					:   List[str]				  =qkv_bias
       lowercase					:   str				  =hidden_dropout_prob
       lowercase					:   Any				  =attention_probs_dropout_prob
       lowercase					:   List[str]				  =drop_path_rate
       lowercase					:   Tuple				  =hidden_act
       lowercase					:   Optional[int]				  =use_absolute_embeddings
       lowercase					:   Dict				  =patch_norm
       lowercase					:   str				  =layer_norm_eps
       lowercase					:   Tuple				  =initializer_range
       lowercase					:   List[str]				  =is_training
       lowercase					:   List[str]				  =scope
       lowercase					:   Optional[Any]				  =use_labels
       lowercase					:   Optional[Any]				  =type_sequence_label_size
       lowercase					:   str				  =encoder_stride






      def 				A__      ( self			:		Dict		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   List[str]				  =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

       lowercase					:   Union[str, Any]				  =None
       if self.use_labels:
        lowercase					:   Dict				  =ids_tensor([self.batch_size]     ,      self.type_sequence_label_size		)

       lowercase					:   Tuple				  =self.get_config()

       return config, pixel_values, labels






      def 				A__      ( self			:		str		)  ->  Optional[int]:


       '''simple docstring'''

       return SwinvaConfig(
           image_size=self.image_size     ,      patch_size=self.patch_size     ,      num_channels=self.num_channels     ,      embed_dim=self.embed_dim     ,      depths=self.depths     ,      num_heads=self.num_heads     ,      window_size=self.window_size     ,      mlp_ratio=self.mlp_ratio     ,      qkv_bias=self.qkv_bias     ,      hidden_dropout_prob=self.hidden_dropout_prob     ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,      drop_path_rate=self.drop_path_rate     ,      hidden_act=self.hidden_act     ,      use_absolute_embeddings=self.use_absolute_embeddings     ,      path_norm=self.patch_norm     ,      layer_norm_eps=self.layer_norm_eps     ,      initializer_range=self.initializer_range     ,      encoder_stride=self.encoder_stride     ,      )






      def 				A__      ( self			:		Tuple     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		Optional[int]		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   str				  =SwinvaModel(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase					:   Any				  =model(UpperCAmelCase		)

       lowercase					:   Union[str, Any]				  =((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths		) - 1))
       lowercase					:   Any				  =int(config.embed_dim * 2 ** (len(config.depths		) - 1)		)

       self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, expected_seq_len, expected_dim)		)






      def 				A__      ( self			:		Dict     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		str		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   str				  =SwinvaForMaskedImageModeling(config=UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase					:   Tuple				  =model(UpperCAmelCase		)
       self.parent.assertEqual(
           result.logits.shape     ,      (self.batch_size, self.num_channels, self.image_size, self.image_size)		)

       # test greyscale images
       lowercase					:   int				  =1
       lowercase					:   Tuple				  =SwinvaForMaskedImageModeling(UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()

       lowercase					:   Tuple				  =floats_tensor([self.batch_size, 1, self.image_size, self.image_size]		)
       lowercase					:   Any				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, 1, self.image_size, self.image_size)		)






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		Optional[int]		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   int				  =self.type_sequence_label_size
       lowercase					:   str				  =SwinvaForImageClassification(UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()
       lowercase					:   Optional[int]				  =model(UpperCAmelCase     ,      labels=UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.type_sequence_label_size)		)






      def 				A__      ( self			:		int		)  ->  int:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self.prepare_config_and_inputs()
       lowercase  ,		lowercase  ,		lowercase					:   List[Any]				  =config_and_inputs
       lowercase					:   Dict				  ={'''pixel_values''': pixel_values}
       return config, inputs_dict





@require_torch
class UpperCAmelCase_					(    __A				,       __A				,       unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			(
          (SwinvaModel, SwinvaForImageClassification, SwinvaForMaskedImageModeling) if is_torch_available() else ()
      )
      UpperCamelCase_								=			(
          {'''feature-extraction''': SwinvaModel, '''image-classification''': SwinvaForImageClassification}
          if is_torch_available()
          else {}
      )

      UpperCamelCase_								=			False
      UpperCamelCase_								=			False
      UpperCamelCase_								=			False
      UpperCamelCase_								=			False






      def 				A__      ( self			:		Union[str, Any]		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   List[str]				  =SwinvaModelTester(self		)
       lowercase					:   List[str]				  =ConfigTester(self     ,      config_class=UpperCAmelCase     ,      embed_dim=37		)






      def 				A__      ( self			:		Union[str, Any]		)  ->  int:


       '''simple docstring'''

       self.config_tester.create_and_test_config_to_json_string()
       self.config_tester.create_and_test_config_to_json_file()
       self.config_tester.create_and_test_config_from_and_save_pretrained()
       self.config_tester.create_and_test_config_with_num_labels()
       self.config_tester.check_config_can_be_init_without_params()
       self.config_tester.check_config_arguments_init()






      def 				A__      ( self			:		Dict		)  ->  int:


       '''simple docstring'''

       lowercase					:   str				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*UpperCAmelCase		)






      @unittest.skip(reason='''Got `CUDA error: misaligned address` with PyTorch 2.0.0.'''		)
      def 				A__      ( self			:		int		)  ->  Optional[Any]:


       '''simple docstring'''

       pass






      @unittest.skip(reason='''Swinv2 does not use inputs_embeds'''		)
      def 				A__      ( self			:		Dict		)  ->  int:


       '''simple docstring'''

       pass






      def 				A__      ( self			:		List[Any]		)  ->  List[Any]:


       '''simple docstring'''

       lowercase  ,		lowercase					:   int				  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
        lowercase					:   Optional[Any]				  =model_class(UpperCAmelCase		)
        self.assertIsInstance(model.get_input_embeddings()     ,      (nn.Module)		)
        lowercase					:   Dict				  =model.get_output_embeddings()
        self.assertTrue(x is None or isinstance(UpperCAmelCase     ,      nn.Linear		)		)






      def 				A__      ( self			:		Optional[Any]		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase  ,		lowercase					:   List[Any]				  =self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
        lowercase					:   Optional[Any]				  =model_class(UpperCAmelCase		)
        lowercase					:   List[Any]				  =inspect.signature(model.forward		)
        # signature.parameters is an OrderedDict => so arg_names order is deterministic
        lowercase					:   Dict				  =[*signature.parameters.keys()]

        lowercase					:   Optional[int]				  =['''pixel_values''']
        self.assertListEqual(arg_names[:1]     ,      UpperCAmelCase		)






      def 				A__      ( self			:		int		)  ->  List[str]:


       '''simple docstring'''

       lowercase  ,		lowercase					:   int				  =self.model_tester.prepare_config_and_inputs_for_common()
       lowercase					:   int				  =True

       for model_class in self.all_model_classes:
        lowercase					:   List[str]				  =True
        lowercase					:   Tuple				  =False
        lowercase					:   Any				  =True
        lowercase					:   Dict				  =model_class(UpperCAmelCase		)
        model.to(UpperCAmelCase		)
        model.eval()
        with torch.no_grad():
         lowercase					:   Any				  =model(**self._prepare_for_class(UpperCAmelCase     ,      UpperCAmelCase		)		)
        lowercase					:   int				  =outputs.attentions
        lowercase					:   Dict				  =len(self.model_tester.depths		)
        self.assertEqual(len(UpperCAmelCase		)     ,      UpperCAmelCase		)

        # check that output_attentions also work using config
        del inputs_dict["output_attentions"]
        lowercase					:   List[Any]				  =True
        lowercase					:   Union[str, Any]				  =config.window_size**2
        lowercase					:   Tuple				  =model_class(UpperCAmelCase		)
        model.to(UpperCAmelCase		)
        model.eval()
        with torch.no_grad():
         lowercase					:   Any				  =model(**self._prepare_for_class(UpperCAmelCase     ,      UpperCAmelCase		)		)
        lowercase					:   Union[str, Any]				  =outputs.attentions
        self.assertEqual(len(UpperCAmelCase		)     ,      UpperCAmelCase		)

        self.assertListEqual(
            list(attentions[0].shape[-3:]		)     ,      [self.model_tester.num_heads[0], window_size_squared, window_size_squared]     ,      )
        lowercase					:   Optional[int]				  =len(UpperCAmelCase		)

        # Check attention is always last and order is fine
        lowercase					:   int				  =True
        lowercase					:   Union[str, Any]				  =True
        lowercase					:   List[Any]				  =model_class(UpperCAmelCase		)
        model.to(UpperCAmelCase		)
        model.eval()
        with torch.no_grad():
         lowercase					:   Optional[int]				  =model(**self._prepare_for_class(UpperCAmelCase     ,      UpperCAmelCase		)		)

        if hasattr(self.model_tester     ,      '''num_hidden_states_types'''		):
         lowercase					:   Tuple				  =self.model_tester.num_hidden_states_types
        else:
         # also another +1 for reshaped_hidden_states
         lowercase					:   List[Any]				  =2
        self.assertEqual(out_len + added_hidden_states     ,      len(UpperCAmelCase		)		)

        lowercase					:   Optional[Any]				  =outputs.attentions

        self.assertEqual(len(UpperCAmelCase		)     ,      UpperCAmelCase		)

        self.assertListEqual(
            list(self_attentions[0].shape[-3:]		)     ,      [self.model_tester.num_heads[0], window_size_squared, window_size_squared]     ,      )






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Any		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   Dict				  =model_class(UpperCAmelCase		)
       model.to(UpperCAmelCase		)
       model.eval()

       with torch.no_grad():
        lowercase					:   int				  =model(**self._prepare_for_class(UpperCAmelCase     ,      UpperCAmelCase		)		)

       lowercase					:   Tuple				  =outputs.hidden_states

       lowercase					:   List[Any]				  =getattr(
           self.model_tester     ,      '''expected_num_hidden_layers'''     ,      len(self.model_tester.depths		) + 1		)
       self.assertEqual(len(UpperCAmelCase		)     ,      UpperCAmelCase		)

       # Swinv2 has a different seq_length
       lowercase					:   Optional[int]				  =(
           config.patch_size
           if isinstance(config.patch_size     ,      collections.abc.Iterable		)
           else (config.patch_size, config.patch_size)
       )

       lowercase					:   Any				  =(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

       self.assertListEqual(
           list(hidden_states[0].shape[-2:]		)     ,      [num_patches, self.model_tester.embed_dim]     ,      )

       lowercase					:   Union[str, Any]				  =outputs.reshaped_hidden_states
       self.assertEqual(len(UpperCAmelCase		)     ,      UpperCAmelCase		)

       lowercase  ,		lowercase  ,		lowercase  ,		lowercase					:   List[str]				  =reshaped_hidden_states[0].shape
       lowercase					:   List[Any]				  =(
           reshaped_hidden_states[0].view(UpperCAmelCase     ,      UpperCAmelCase     ,      height * width		).permute(0     ,      2     ,      1		)
       )
       self.assertListEqual(
           list(reshaped_hidden_states.shape[-2:]		)     ,      [num_patches, self.model_tester.embed_dim]     ,      )






      def 				A__      ( self			:		str		)  ->  List[Any]:


       '''simple docstring'''

       lowercase  ,		lowercase					:   Tuple				  =self.model_tester.prepare_config_and_inputs_for_common()

       lowercase					:   int				  =(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size     ,      collections.abc.Iterable		)
           else (self.model_tester.image_size, self.model_tester.image_size)
       )

       for model_class in self.all_model_classes:
        lowercase					:   int				  =True
        self.check_hidden_states_output(UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase		)

        # check that output_hidden_states also work using config
        del inputs_dict["output_hidden_states"]
        lowercase					:   Tuple				  =True

        self.check_hidden_states_output(UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase		)






      def 				A__      ( self			:		Any		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase  ,		lowercase					:   Union[str, Any]				  =self.model_tester.prepare_config_and_inputs_for_common()
       lowercase					:   Any				  =3

       lowercase					:   int				  =(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size     ,      collections.abc.Iterable		)
           else (self.model_tester.image_size, self.model_tester.image_size)
       )
       lowercase					:   Optional[int]				  =(
           config.patch_size
           if isinstance(config.patch_size     ,      collections.abc.Iterable		)
           else (config.patch_size, config.patch_size)
       )

       lowercase					:   Dict				  =image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
       lowercase					:   Optional[Any]				  =image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

       for model_class in self.all_model_classes:
        lowercase					:   List[str]				  =True
        self.check_hidden_states_output(UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase     ,      (padded_height, padded_width)		)

        # check that output_hidden_states also work using config
        del inputs_dict["output_hidden_states"]
        lowercase					:   Tuple				  =True
        self.check_hidden_states_output(UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase     ,      (padded_height, padded_width)		)






      def 				A__      ( self			:		Tuple		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_image_modeling(*UpperCAmelCase		)






      def 				A__      ( self			:		int		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*UpperCAmelCase		)






      @slow
      def 				A__      ( self			:		Dict		)  ->  Optional[Any]:


       '''simple docstring'''

       for model_name in SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase					:   int				  =SwinvaModel.from_pretrained(UpperCAmelCase		)
        self.assertIsNotNone(UpperCAmelCase		)






      def 				A__      ( self			:		List[str]		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase  ,		lowercase					:   List[str]				  =self.model_tester.prepare_config_and_inputs_for_common()

       lowercase					:   int				  =_config_zero_init(UpperCAmelCase		)
       for model_class in self.all_model_classes:
        lowercase					:   str				  =model_class(config=UpperCAmelCase		)
        for name, param in model.named_parameters():
         if "embeddings" not in name and "logit_scale" not in name and param.requires_grad:
          self.assertIn(
              ((param.data.mean() * 1e9).round() / 1e9).item()     ,      [0.0, 1.0]     ,      msg=f'Parameter {name} of model {model_class} seems not properly initialized'     ,      )





@require_vision
@require_torch
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      @cached_property
      def 				A__      ( self			:		str		)  ->  Any:


       '''simple docstring'''

       return (
           AutoImageProcessor.from_pretrained('''microsoft/swinv2-tiny-patch4-window8-256'''		)
           if is_vision_available()
           else None
       )






      @slow
      def 				A__      ( self			:		Optional[int]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   List[str]				  =SwinvaForImageClassification.from_pretrained('''microsoft/swinv2-tiny-patch4-window8-256'''		).to(
           UpperCAmelCase		)
       lowercase					:   Any				  =self.default_image_processor

       lowercase					:   List[Any]				  =Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''		)
       lowercase					:   List[str]				  =image_processor(images=UpperCAmelCase     ,      return_tensors='''pt'''		).to(UpperCAmelCase		)

       # forward pass
       with torch.no_grad():
        lowercase					:   Optional[Any]				  =model(**UpperCAmelCase		)

       # verify the logits
       lowercase					:   Union[str, Any]				  =torch.Size((1, 1000)		)
       self.assertEqual(outputs.logits.shape     ,      UpperCAmelCase		)
       lowercase					:   Union[str, Any]				  =torch.tensor([-0.3_9_4_7, -0.4_3_0_6, 0.0_0_2_6]		).to(UpperCAmelCase		)
       self.assertTrue(torch.allclose(outputs.logits[0, :3]     ,      UpperCAmelCase     ,      atol=1e-4		)		)











'''simple docstring'''



from __future__ import annotations

import unittest

from transformers import DistilBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers.models.distilbert.modeling_tf_distilbert import (
          TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFDistilBertForMaskedLM,
          TFDistilBertForMultipleChoice,
          TFDistilBertForQuestionAnswering,
          TFDistilBertForSequenceClassification,
          TFDistilBertForTokenClassification,
          TFDistilBertModel,
      )
class UpperCAmelCase_					:




      """simple docstring"""




      def __init__( self			:		Tuple     ,      UpperCAmelCase			:		List[str]     ,      )  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   int				  =parent
       lowercase					:   Any				  =13
       lowercase					:   Any				  =7
       lowercase					:   Optional[int]				  =True
       lowercase					:   Optional[int]				  =True
       lowercase					:   Tuple				  =False
       lowercase					:   Optional[Any]				  =True
       lowercase					:   Dict				  =99
       lowercase					:   Union[str, Any]				  =32
       lowercase					:   Union[str, Any]				  =2
       lowercase					:   Union[str, Any]				  =4
       lowercase					:   List[str]				  =37
       lowercase					:   str				  ='''gelu'''
       lowercase					:   Dict				  =0.1
       lowercase					:   List[Any]				  =0.1
       lowercase					:   List[str]				  =512
       lowercase					:   Optional[int]				  =16
       lowercase					:   Optional[Any]				  =2
       lowercase					:   List[str]				  =0.0_2
       lowercase					:   Any				  =3
       lowercase					:   Optional[Any]				  =4
       lowercase					:   int				  =None






      def 				A__      ( self			:		List[str]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Tuple				  =ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size		)

       lowercase					:   Any				  =None
       if self.use_input_mask:
        lowercase					:   List[str]				  =random_attention_mask([self.batch_size, self.seq_length]		)

       lowercase					:   Union[str, Any]				  =None
       lowercase					:   Any				  =None
       lowercase					:   str				  =None
       if self.use_labels:
        lowercase					:   Union[str, Any]				  =ids_tensor([self.batch_size]     ,      self.type_sequence_label_size		)
        lowercase					:   Tuple				  =ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels		)
        lowercase					:   List[Any]				  =ids_tensor([self.batch_size]     ,      self.num_choices		)

       lowercase					:   Dict				  =DistilBertConfig(
           vocab_size=self.vocab_size     ,      dim=self.hidden_size     ,      n_layers=self.num_hidden_layers     ,      n_heads=self.num_attention_heads     ,      hidden_dim=self.intermediate_size     ,      hidden_act=self.hidden_act     ,      dropout=self.hidden_dropout_prob     ,      attention_dropout=self.attention_probs_dropout_prob     ,      max_position_embeddings=self.max_position_embeddings     ,      initializer_range=self.initializer_range     ,      )

       return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		Optional[Any]		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   int				  =TFDistilBertModel(config=UpperCAmelCase		)
       lowercase					:   int				  ={'''input_ids''': input_ids, '''attention_mask''': input_mask}

       lowercase					:   List[str]				  =model(UpperCAmelCase		)

       lowercase					:   str				  =[input_ids, input_mask]

       lowercase					:   Tuple				  =model(UpperCAmelCase		)

       self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)		)






      def 				A__      ( self			:		List[Any]     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Optional[Any]		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   List[str]				  =TFDistilBertForMaskedLM(config=UpperCAmelCase		)
       lowercase					:   int				  ={'''input_ids''': input_ids, '''attention_mask''': input_mask}
       lowercase					:   Union[str, Any]				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)		)






      def 				A__      ( self			:		List[Any]     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		Optional[int]		)  ->  Any:


       '''simple docstring'''

       lowercase					:   str				  =TFDistilBertForQuestionAnswering(config=UpperCAmelCase		)
       lowercase					:   int				  ={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
       }
       lowercase					:   List[str]				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.start_logits.shape     ,      (self.batch_size, self.seq_length)		)
       self.parent.assertEqual(result.end_logits.shape     ,      (self.batch_size, self.seq_length)		)






      def 				A__      ( self			:		str     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		Tuple     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Dict		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   Dict				  =self.num_labels
       lowercase					:   Optional[Any]				  =TFDistilBertForSequenceClassification(UpperCAmelCase		)
       lowercase					:   str				  ={'''input_ids''': input_ids, '''attention_mask''': input_mask}
       lowercase					:   Union[str, Any]				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_labels)		)






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Dict		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self.num_choices
       lowercase					:   Optional[int]				  =TFDistilBertForMultipleChoice(UpperCAmelCase		)
       lowercase					:   Optional[Any]				  =tf.tile(tf.expand_dims(UpperCAmelCase     ,      1		)     ,      (1, self.num_choices, 1)		)
       lowercase					:   List[Any]				  =tf.tile(tf.expand_dims(UpperCAmelCase     ,      1		)     ,      (1, self.num_choices, 1)		)
       lowercase					:   Tuple				  ={
           '''input_ids''': multiple_choice_inputs_ids,
           '''attention_mask''': multiple_choice_input_mask,
       }
       lowercase					:   Tuple				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_choices)		)






      def 				A__      ( self			:		List[Any]     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Tuple     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		Union[str, Any]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Dict				  =self.num_labels
       lowercase					:   Tuple				  =TFDistilBertForTokenClassification(UpperCAmelCase		)
       lowercase					:   Optional[Any]				  ={'''input_ids''': input_ids, '''attention_mask''': input_mask}
       lowercase					:   str				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)		)






      def 				A__      ( self			:		List[str]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   int				  =self.prepare_config_and_inputs()
       ((lowercase)  ,		(lowercase)  ,		(lowercase)  ,		(lowercase)  ,		(lowercase)  ,		(lowercase))					:   Union[str, Any]				  =config_and_inputs
       lowercase					:   Tuple				  ={'''input_ids''': input_ids, '''attention_mask''': input_mask}
       return config, inputs_dict





@require_tf
class UpperCAmelCase_					(    __A				,       __A				,       unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			(
          (
              TFDistilBertModel,
              TFDistilBertForMaskedLM,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertForMultipleChoice,
          )
          if is_tf_available()
          else None
      )
      UpperCamelCase_								=			(
          {
              '''feature-extraction''': TFDistilBertModel,
              '''fill-mask''': TFDistilBertForMaskedLM,
              '''question-answering''': TFDistilBertForQuestionAnswering,
              '''text-classification''': TFDistilBertForSequenceClassification,
              '''token-classification''': TFDistilBertForTokenClassification,
              '''zero-shot''': TFDistilBertForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      UpperCamelCase_								=			False
      UpperCamelCase_								=			False






      def 				A__      ( self			:		Dict		)  ->  str:


       '''simple docstring'''

       lowercase					:   str				  =TFDistilBertModelTester(self		)
       lowercase					:   int				  =ConfigTester(self     ,      config_class=UpperCAmelCase     ,      dim=37		)






      def 				A__      ( self			:		Union[str, Any]		)  ->  Dict:


       '''simple docstring'''

       self.config_tester.run_common_tests()






      def 				A__      ( self			:		Optional[Any]		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_model(*UpperCAmelCase		)






      def 				A__      ( self			:		Tuple		)  ->  Any:


       '''simple docstring'''

       lowercase					:   str				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_masked_lm(*UpperCAmelCase		)






      def 				A__      ( self			:		Optional[int]		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   Any				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_question_answering(*UpperCAmelCase		)






      def 				A__      ( self			:		Any		)  ->  str:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_sequence_classification(*UpperCAmelCase		)






      def 				A__      ( self			:		Optional[Any]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   str				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_multiple_choice(*UpperCAmelCase		)






      def 				A__      ( self			:		str		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   str				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_distilbert_for_token_classification(*UpperCAmelCase		)






      @slow
      def 				A__      ( self			:		List[Any]		)  ->  Dict:


       '''simple docstring'''

       for model_name in list(TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]		):
        lowercase					:   Union[str, Any]				  =TFDistilBertModel.from_pretrained(UpperCAmelCase		)
        self.assertIsNotNone(UpperCAmelCase		)





@require_tf
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      @slow
      def 				A__      ( self			:		List[str]		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =TFDistilBertModel.from_pretrained('''distilbert-base-uncased'''		)
       lowercase					:   Tuple				  =tf.constant([[0, 1, 2, 3, 4, 5]]		)
       lowercase					:   List[Any]				  =model(UpperCAmelCase		)[0]

       lowercase					:   str				  =[1, 6, 768]
       self.assertEqual(output.shape     ,      UpperCAmelCase		)

       lowercase					:   Optional[int]				  =tf.constant(
           [
               [
                   [0.1_9_2_6_1_8_8_5, -0.1_3_7_3_2_9_5_5, 0.4_1_1_9_7_9_9],
                   [0.2_2_1_5_0_1_5_6, -0.0_7_4_2_2_6_6_1, 0.3_9_0_3_7_2_0_4],
                   [0.2_2_7_5_6_0_1_8, -0.0_8_9_6_4_1_4, 0.3_7_0_1_4_6_7],
               ]
           ]		)
       tf.debugging.assert_near(output[:, :3, :3]     ,      UpperCAmelCase     ,      atol=1e-4		)



