


import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    SwiftFormerConfig,
    SwiftFormerForImageClassification,
    ViTImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
A   :					int	=      logging.get_logger(__name__)

A   :					Union[str, Any]	=      torch.device("cpu")

def    lowercase_	(	):

      """simple docstring"""




      lowerCamelCase__	:       Union[str, Any]					=						"http://images.cocodataset.org/val2017/000000039769.jpg"
      lowerCamelCase__	:       List[Any]					=						Image.open(requests.get(_A					,      stream=_A		).raw		)
      return im

def    lowercase_	(	_A    :					Tuple		):

      """simple docstring"""




      if swiftformer_name == "swiftformer_xs":
            return torch.tensor([-2.1703E00, 2.1107E00, -2.0811E00, 8.8685E-01, 2.4360E-01]		)

      elif swiftformer_name == "swiftformer_s":
            return torch.tensor([3.9636E-01, 2.3478E-01, -1.6963E00, -1.7381E00, -8.6337E-01]		)

      elif swiftformer_name == "swiftformer_l1":
            return torch.tensor([-4.2768E-01, -4.7429E-01, -1.0897E00, -1.0248E00, 3.5523E-02]		)

      elif swiftformer_name == "swiftformer_l3":
            return torch.tensor([-2.5330E-01, 2.4211E-01, -6.0185E-01, -8.2789E-01, -6.0446E-02]		)

def    lowercase_	(	_A    :					Union[str, Any]					,      _A    :					Optional[Any]					,      _A    :					Union[str, Any]		):

      """simple docstring"""




      lowerCamelCase__	:       int					=						dct.pop(_A		)
      lowerCamelCase__	:       Union[str, Any]					=						val

def    lowercase_	(	_A    :					Optional[int]		):

      """simple docstring"""




      lowerCamelCase__	:       Optional[Any]					=						[]
      for k in state_dict.keys():
            lowerCamelCase__	:       str					=						k
            if ".pwconv" in k:
                  lowerCamelCase__	:       Any					=						k_new.replace(".pwconv"					,      ".point_wise_conv"		)
            if ".dwconv" in k:
                  lowerCamelCase__	:       int					=						k_new.replace(".dwconv"					,      ".depth_wise_conv"		)
            if ".Proj." in k:
                  lowerCamelCase__	:       Dict					=						k_new.replace(".Proj."					,      ".proj."		)
            if "patch_embed" in k_new:
                  lowerCamelCase__	:       Dict					=						k_new.replace("patch_embed"					,      "swiftformer.patch_embed.patch_embedding"		)
            if "network" in k_new:
                  lowerCamelCase__	:       Dict					=						k_new.split("."		)
                  if ls[2].isdigit():
                        lowerCamelCase__	:       List[str]					=						"swiftformer.encoder.network." + ls[1] + ".blocks." + ls[2] + "." + ".".join(ls[3:]		)
                  else:
                        lowerCamelCase__	:       List[Any]					=						k_new.replace("network"					,      "swiftformer.encoder.network"		)
            rename_keys.append((k, k_new)		)
      return rename_keys

@torch.no_grad()
def    lowercase_	(	_A    :					Optional[int]					,      _A    :					int					,      _A    :					Optional[int]		):

      """simple docstring"""




      lowerCamelCase__	:       Dict					=						SwiftFormerConfig()

      # dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
      lowerCamelCase__	:       Optional[Any]					=						1000
      lowerCamelCase__	:       Dict					=						"huggingface/label-files"
      lowerCamelCase__	:       Dict					=						"imagenet-1k-id2label.json"
      lowerCamelCase__	:       List[str]					=						json.load(open(hf_hub_download(_A					,      _A					,      repo_type="dataset"		)					,      "r"		)		)
      lowerCamelCase__	:       Any					=						{int(_A		): v for k, v in idalabel.items()}
      lowerCamelCase__	:       List[str]					=						idalabel
      lowerCamelCase__	:       str					=						{v: k for k, v in idalabel.items()}

      # size of the architecture
      if swiftformer_name == "swiftformer_xs":
            lowerCamelCase__	:       Optional[int]					=						[3, 3, 6, 4]
            lowerCamelCase__	:       str					=						[48, 56, 112, 220]

      elif swiftformer_name == "swiftformer_s":
            lowerCamelCase__	:       Optional[int]					=						[3, 3, 9, 6]
            lowerCamelCase__	:       Tuple					=						[48, 64, 168, 224]

      elif swiftformer_name == "swiftformer_l1":
            lowerCamelCase__	:       str					=						[4, 3, 10, 5]
            lowerCamelCase__	:       str					=						[48, 96, 192, 384]

      elif swiftformer_name == "swiftformer_l3":
            lowerCamelCase__	:       Any					=						[4, 4, 12, 6]
            lowerCamelCase__	:       str					=						[64, 128, 320, 512]

      # load state_dict of original model, remove and rename some keys
      if original_ckpt:
            if original_ckpt.startswith("https"		):
                  lowerCamelCase__	:       List[str]					=						torch.hub.load_state_dict_from_url(_A					,      map_location="cpu"					,      check_hash=_A		)
            else:
                  lowerCamelCase__	:       Tuple					=						torch.load(_A					,      map_location="cpu"		)
      lowerCamelCase__	:       Tuple					=						checkpoint

      lowerCamelCase__	:       Dict					=						create_rename_keys(_A		)
      for rename_key_src, rename_key_dest in rename_keys:
            rename_key(_A					,      _A					,      _A		)

      # load HuggingFace model
      lowerCamelCase__	:       str					=						SwiftFormerForImageClassification(_A		).eval()
      hf_model.load_state_dict(_A		)

      # prepare test inputs
      lowerCamelCase__	:       Optional[Any]					=						prepare_img()
      lowerCamelCase__	:       Any					=						ViTImageProcessor.from_pretrained("preprocessor_config"		)
      lowerCamelCase__	:       int					=						processor(images=_A					,      return_tensors="pt"		)

      # compare outputs from both models
      lowerCamelCase__	:       str					=						get_expected_output(_A		)
      lowerCamelCase__	:       Dict					=						hf_model(inputs["pixel_values"]		).logits

      assert hf_logits.shape == torch.Size([1, 1000]		)
      assert torch.allclose(hf_logits[0, 0:5]					,      _A					,      atol=1E-3		)

      Path(_A		).mkdir(exist_ok=_A		)
      print(F"Saving model {swiftformer_name} to {pytorch_dump_folder_path}"		)
      hf_model.save_pretrained(_A		)


if __name__ == "__main__":
     A   :					Tuple	=      argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         "--swiftformer_name",
         default="swiftformer_xs",
         choices=["swiftformer_xs", "swiftformer_s", "swiftformer_l1", "swiftformer_l3"],
         type=str,
         help="Name of the SwiftFormer model you'd like to convert.",
     )
     parser.add_argument(
         "--pytorch_dump_folder_path",
         default="./converted_outputs/",
         type=str,
         help="Path to the output PyTorch model directory.",
     )
     parser.add_argument("--original_ckpt", default=None, type=str, help="Path to the original model checkpoint.")

     A   :					Any	=      parser.parse_args()
     convert_swiftformer_checkpoint(args.swiftformer_name, args.pytorch_dump_folder_path, args.original_ckpt)






import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        AlbertForMaskedLM,
        AlbertForMultipleChoice,
        AlbertForPreTraining,
        AlbertForQuestionAnswering,
        AlbertForSequenceClassification,
        AlbertForTokenClassification,
        AlbertModel,
    )
    from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class     _lowercase				:


       """simple docstring"""




       def __init__(     self							:    List[str]      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    List[str]=13      ,   __lowerCamelCase							:    Dict=7      ,   __lowerCamelCase							:    List[Any]=True      ,   __lowerCamelCase							:    List[Any]=True      ,   __lowerCamelCase							:    Union[str, Any]=True      ,   __lowerCamelCase							:    Optional[Any]=True      ,   __lowerCamelCase							:    Optional[int]=99      ,   __lowerCamelCase							:    List[Any]=16      ,   __lowerCamelCase							:    Optional[Any]=36      ,   __lowerCamelCase							:    Optional[int]=6      ,   __lowerCamelCase							:    Union[str, Any]=6      ,   __lowerCamelCase							:    Optional[int]=6      ,   __lowerCamelCase							:    Dict=37      ,   __lowerCamelCase							:    List[Any]="gelu"      ,   __lowerCamelCase							:    Tuple=0.1      ,   __lowerCamelCase							:    Optional[Any]=0.1      ,   __lowerCamelCase							:    List[Any]=512      ,   __lowerCamelCase							:    Dict=16      ,   __lowerCamelCase							:    Union[str, Any]=2      ,   __lowerCamelCase							:    Dict=0.0_2      ,   __lowerCamelCase							:    Optional[Any]=3      ,   __lowerCamelCase							:    Dict=4      ,   __lowerCamelCase							:    Dict=None      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       Dict					=						parent
             lowerCamelCase__	:       List[Any]					=						batch_size
             lowerCamelCase__	:       Any					=						seq_length
             lowerCamelCase__	:       List[str]					=						is_training
             lowerCamelCase__	:       int					=						use_input_mask
             lowerCamelCase__	:       List[str]					=						use_token_type_ids
             lowerCamelCase__	:       int					=						use_labels
             lowerCamelCase__	:       Dict					=						vocab_size
             lowerCamelCase__	:       List[Any]					=						embedding_size
             lowerCamelCase__	:       Dict					=						hidden_size
             lowerCamelCase__	:       Any					=						num_hidden_layers
             lowerCamelCase__	:       Optional[Any]					=						num_hidden_groups
             lowerCamelCase__	:       Optional[int]					=						num_attention_heads
             lowerCamelCase__	:       List[str]					=						intermediate_size
             lowerCamelCase__	:       Optional[Any]					=						hidden_act
             lowerCamelCase__	:       str					=						hidden_dropout_prob
             lowerCamelCase__	:       Union[str, Any]					=						attention_probs_dropout_prob
             lowerCamelCase__	:       Optional[int]					=						max_position_embeddings
             lowerCamelCase__	:       List[Any]					=						type_vocab_size
             lowerCamelCase__	:       Optional[Any]					=						type_sequence_label_size
             lowerCamelCase__	:       Optional[int]					=						initializer_range
             lowerCamelCase__	:       str					=						num_labels
             lowerCamelCase__	:       List[Any]					=						num_choices
             lowerCamelCase__	:       Any					=						scope





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[Any]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.vocab_size )

             lowerCamelCase__	:       Optional[int]					=						None
             if self.use_input_mask:
                   lowerCamelCase__	:       Any					=						random_attention_mask([self.batch_size, self.seq_length] )

             lowerCamelCase__	:       Optional[Any]					=						None
             if self.use_token_type_ids:
                   lowerCamelCase__	:       Union[str, Any]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.type_vocab_size )

             lowerCamelCase__	:       Tuple					=						None
             lowerCamelCase__	:       List[str]					=						None
             lowerCamelCase__	:       int					=						None
             if self.use_labels:
                   lowerCamelCase__	:       Tuple					=						ids_tensor([self.batch_size]      ,   self.type_sequence_label_size )
                   lowerCamelCase__	:       Union[str, Any]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.num_labels )
                   lowerCamelCase__	:       str					=						ids_tensor([self.batch_size]      ,   self.num_choices )

             lowerCamelCase__	:       Union[str, Any]					=						self.get_config()

             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





       def 		lowerCAmelCase  (     self							:    str ):


             '''simple docstring'''



             return AlbertConfig(
                 vocab_size=self.vocab_size      ,   hidden_size=self.hidden_size      ,   num_hidden_layers=self.num_hidden_layers      ,   num_attention_heads=self.num_attention_heads      ,   intermediate_size=self.intermediate_size      ,   hidden_act=self.hidden_act      ,   hidden_dropout_prob=self.hidden_dropout_prob      ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,   max_position_embeddings=self.max_position_embeddings      ,   type_vocab_size=self.type_vocab_size      ,   initializer_range=self.initializer_range      ,   num_hidden_groups=self.num_hidden_groups      ,   )





       def 		lowerCAmelCase  (     self							:    Dict      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       int					=						AlbertModel(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       Any					=						model(__lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase )
             lowerCamelCase__	:       Any					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase )
             lowerCamelCase__	:       Optional[int]					=						model(__lowerCamelCase )
             self.parent.assertEqual(result.last_hidden_state.shape      ,   (self.batch_size, self.seq_length, self.hidden_size) )
             self.parent.assertEqual(result.pooler_output.shape      ,   (self.batch_size, self.hidden_size) )





       def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    Tuple ):


             '''simple docstring'''



             lowerCamelCase__	:       Any					=						AlbertForPreTraining(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       Union[str, Any]					=						model(
                 __lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase      ,   sentence_order_label=__lowerCamelCase      ,   )
             self.parent.assertEqual(result.prediction_logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size) )
             self.parent.assertEqual(result.sop_logits.shape      ,   (self.batch_size, config.num_labels) )





       def 		lowerCAmelCase  (     self							:    str      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Tuple ):


             '''simple docstring'''



             lowerCamelCase__	:       Dict					=						AlbertForMaskedLM(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       Tuple					=						model(__lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size) )





       def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    int ):


             '''simple docstring'''



             lowerCamelCase__	:       str					=						AlbertForQuestionAnswering(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       str					=						model(
                 __lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   start_positions=__lowerCamelCase      ,   end_positions=__lowerCamelCase      ,   )
             self.parent.assertEqual(result.start_logits.shape      ,   (self.batch_size, self.seq_length) )
             self.parent.assertEqual(result.end_logits.shape      ,   (self.batch_size, self.seq_length) )





       def 		lowerCAmelCase  (     self							:    Optional[int]      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       int					=						self.num_labels
             lowerCamelCase__	:       Optional[int]					=						AlbertForSequenceClassification(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       Any					=						model(__lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_labels) )





       def 		lowerCAmelCase  (     self							:    Dict      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						self.num_labels
             lowerCamelCase__	:       List[str]					=						AlbertForTokenClassification(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       Tuple					=						model(__lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.num_labels) )





       def 		lowerCAmelCase  (     self							:    Tuple      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Union[str, Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						self.num_choices
             lowerCamelCase__	:       Optional[int]					=						AlbertForMultipleChoice(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()
             lowerCamelCase__	:       Any					=						input_ids.unsqueeze(1 ).expand(-1      ,   self.num_choices      ,   -1 ).contiguous()
             lowerCamelCase__	:       Optional[int]					=						token_type_ids.unsqueeze(1 ).expand(-1      ,   self.num_choices      ,   -1 ).contiguous()
             lowerCamelCase__	:       Dict					=						input_mask.unsqueeze(1 ).expand(-1      ,   self.num_choices      ,   -1 ).contiguous()
             lowerCamelCase__	:       int					=						model(
                 __lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase      ,   )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_choices) )





       def 		lowerCAmelCase  (     self							:    str ):


             '''simple docstring'''



             lowerCamelCase__	:       int					=						self.prepare_config_and_inputs()
             (
                 (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     
             )	:       Union[str, Any]					=						config_and_inputs
             lowerCamelCase__	:       str					=						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
             return config, inputs_dict







@require_torch
class     _lowercase				(							lowercase__							,						lowercase__							,						unittest.TestCase):


       """simple docstring"""




       A__      		=    (
           (
               AlbertModel,
               AlbertForPreTraining,
               AlbertForMaskedLM,
               AlbertForMultipleChoice,
               AlbertForSequenceClassification,
               AlbertForTokenClassification,
               AlbertForQuestionAnswering,
           )
           if is_torch_available()
           else ()
       )
       A__      		=    (
           {
               "feature-extraction": AlbertModel,
               "fill-mask": AlbertForMaskedLM,
               "question-answering": AlbertForQuestionAnswering,
               "text-classification": AlbertForSequenceClassification,
               "token-classification": AlbertForTokenClassification,
               "zero-shot": AlbertForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       A__      		=    True





       def 		lowerCAmelCase  (     self							:    List[str]      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    Dict=False ):


             '''simple docstring'''



             lowerCamelCase__	:       Any					=						super()._prepare_for_class(__lowerCamelCase      ,   __lowerCamelCase      ,   return_labels=__lowerCamelCase )

             if return_labels:
                   if model_class in get_values(__lowerCamelCase ):
                         lowerCamelCase__	:       Union[str, Any]					=						torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.seq_length)      ,   dtype=torch.long      ,   device=__lowerCamelCase )
                         lowerCamelCase__	:       List[str]					=						torch.zeros(
                             self.model_tester.batch_size      ,   dtype=torch.long      ,   device=__lowerCamelCase )
             return inputs_dict





       def 		lowerCAmelCase  (     self							:    Optional[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						AlbertModelTester(self )
             lowerCamelCase__	:       Optional[Any]					=						ConfigTester(self      ,   config_class=__lowerCamelCase      ,   hidden_size=37 )





       def 		lowerCAmelCase  (     self							:    Tuple ):


             '''simple docstring'''



             self.config_tester.run_common_tests()





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Union[str, Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_pretraining(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Any ):


             '''simple docstring'''



             lowerCamelCase__	:       int					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_lm(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Any ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_multiple_choice(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Tuple ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_question_answering(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Union[str, Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       List[str]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_sequence_classification(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Optional[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       List[str]					=						self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                   lowerCamelCase__	:       Dict					=						type
                   self.model_tester.create_and_check_model(*__lowerCamelCase )





       @slow
       def 		lowerCAmelCase  (     self							:    Optional[Any] ):


             '''simple docstring'''



             for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   lowerCamelCase__	:       List[str]					=						AlbertModel.from_pretrained(__lowerCamelCase )
                   self.assertIsNotNone(__lowerCamelCase )







@require_torch
class     _lowercase				(							unittest.TestCase):


       """simple docstring"""




       @slow
       def 		lowerCAmelCase  (     self							:    Union[str, Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						AlbertModel.from_pretrained("albert-base-v2" )
             lowerCamelCase__	:       Any					=						torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]] )
             lowerCamelCase__	:       int					=						torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
             with torch.no_grad():
                   lowerCamelCase__	:       List[Any]					=						model(__lowerCamelCase      ,   attention_mask=__lowerCamelCase )[0]
             lowerCamelCase__	:       Tuple					=						torch.Size((1, 11, 768) )
             self.assertEqual(output.shape      ,   __lowerCamelCase )
             lowerCamelCase__	:       Dict					=						torch.tensor(
                 [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]] )

             self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,   __lowerCamelCase      ,   atol=1E-4 ) )


