


import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
       from transformers.models.esm.modeling_esm import (
           ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
           EsmEmbeddings,
           create_position_ids_from_input_ids,
       )







class  _lowercase   :
      def __init__(     self				,			a				,			a=1_3				,			a=7				,			a=False				,			a=True				,			a=False				,			a=True				,			a=3_3				,			a=3_2				,			a=5				,			a=4				,			a=3_7				,			a="gelu"				,			a=0.1				,			a=0.1				,			a=5_1_2				,			a=1_6				,			a=2				,			a=0.02				,			a=3				,			a=4				,			a=None				,			):
       snake_case__  :    Optional[int]							       =parent
       snake_case__  :    int							       =batch_size
       snake_case__  :    Any							       =seq_length
       snake_case__  :    Union[str, Any]							       =is_training
       snake_case__  :    Dict							       =use_input_mask
       snake_case__  :    str							       =use_token_type_ids
       snake_case__  :    Optional[Any]							       =use_labels
       snake_case__  :    int							       =vocab_size
       snake_case__  :    List[str]							       =hidden_size
       snake_case__  :    List[Any]							       =num_hidden_layers
       snake_case__  :    str							       =num_attention_heads
       snake_case__  :    Any							       =intermediate_size
       snake_case__  :    Any							       =hidden_act
       snake_case__  :    Optional[int]							       =hidden_dropout_prob
       snake_case__  :    Optional[Any]							       =attention_probs_dropout_prob
       snake_case__  :    int							       =max_position_embeddings
       snake_case__  :    Optional[Any]							       =type_vocab_size
       snake_case__  :    Dict							       =type_sequence_label_size
       snake_case__  :    Tuple							       =initializer_range
       snake_case__  :    Tuple							       =num_labels
       snake_case__  :    Optional[int]							       =num_choices
       snake_case__  :    Any							       =scope
      def 						lowercase__					(     self   ):
       snake_case__  :    Union[str, Any]							       =ids_tensor([self.batch_size, self.seq_length]				,			self.vocab_size   )

       snake_case__  :    str							       =None
       if self.use_input_mask:
        snake_case__  :    Dict							       =random_attention_mask([self.batch_size, self.seq_length]   )

       snake_case__  :    Optional[int]							       =None
       snake_case__  :    str							       =None
       snake_case__  :    Tuple							       =None
       if self.use_labels:
        snake_case__  :    Dict							       =ids_tensor([self.batch_size]				,			self.type_sequence_label_size   )
        snake_case__  :    List[str]							       =ids_tensor([self.batch_size, self.seq_length]				,			self.num_labels   )
        snake_case__  :    List[str]							       =ids_tensor([self.batch_size]				,			self.num_choices   )

       snake_case__  :    Tuple							       =self.get_config()

       return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
      def 						lowercase__					(     self   ):
       return EsmConfig(
           vocab_size=self.vocab_size				,			hidden_size=self.hidden_size				,			pad_token_id=1				,			num_hidden_layers=self.num_hidden_layers				,			num_attention_heads=self.num_attention_heads				,			intermediate_size=self.intermediate_size				,			hidden_act=self.hidden_act				,			hidden_dropout_prob=self.hidden_dropout_prob				,			attention_probs_dropout_prob=self.attention_probs_dropout_prob				,			max_position_embeddings=self.max_position_embeddings				,			type_vocab_size=self.type_vocab_size				,			initializer_range=self.initializer_range				,			)
      def 						lowercase__					(     self				,			a				,			a				,			a				,			a				,			a				,			a   ):
       snake_case__  :    int							       =EsmModel(config=a   )
       model.to(a   )
       model.eval()
       snake_case__  :    Tuple							       =model(a				,			attention_mask=a   )
       snake_case__  :    Optional[Any]							       =model(a   )
       snake_case__  :    Optional[Any]							       =model(a   )

       self.parent.assertEqual(result.last_hidden_state.shape				,			(self.batch_size, self.seq_length, self.hidden_size)   )
       self.parent.assertEqual(result.pooler_output.shape				,			(self.batch_size, self.hidden_size)   )
      def 						lowercase__					(     self				,			a				,			a				,			a				,			a				,			a				,			a   ):
       snake_case__  :    Optional[int]							       =EsmForMaskedLM(config=a   )
       model.to(a   )
       model.eval()
       snake_case__  :    Optional[Any]							       =model(a				,			attention_mask=a				,			labels=a   )
       self.parent.assertEqual(result.logits.shape				,			(self.batch_size, self.seq_length, self.vocab_size)   )
      def 						lowercase__					(     self				,			a				,			a				,			a				,			a				,			a				,			a   ):
       snake_case__  :    Optional[int]							       =self.num_labels
       snake_case__  :    str							       =EsmForTokenClassification(config=a   )
       model.to(a   )
       model.eval()
       snake_case__  :    List[str]							       =model(a				,			attention_mask=a				,			labels=a   )
       self.parent.assertEqual(result.logits.shape				,			(self.batch_size, self.seq_length, self.num_labels)   )



      def 						lowercase__					(     self   ):
       snake_case__  :    Dict							       =self.prepare_config_and_inputs()
       (
           snake_case__   
       )  :    Optional[Any]							       =config_and_inputs
       snake_case__  :    Optional[Any]							       ={"""input_ids""": input_ids, """attention_mask""": input_mask}
       return config, inputs_dict







@require_torch
class  _lowercase   (   _A						,				_A						,				unittest.TestCase						):
      _a							:						Optional[Any]										=							False

      _a							:						Optional[int]										=							(
          (
              EsmForMaskedLM,
              EsmModel,
              EsmForSequenceClassification,
              EsmForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _a							:						Tuple										=							()
      _a							:						str										=							(
          {
              'feature-extraction': EsmModel,
              'fill-mask': EsmForMaskedLM,
              'text-classification': EsmForSequenceClassification,
              'token-classification': EsmForTokenClassification,
              'zero-shot': EsmForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      _a							:						int										=							True
      def 						lowercase__					(     self   ):
       snake_case__  :    int							       =EsmModelTester(self   )
       snake_case__  :    List[Any]							       =ConfigTester(self				,			config_class=a				,			hidden_size=3_7   )
      def 						lowercase__					(     self   ):
       self.config_tester.run_common_tests()
      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[Any]							       =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*a   )
      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[int]							       =self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
        snake_case__  :    Optional[Any]							       =type
        self.model_tester.create_and_check_model(*a   )
      def 						lowercase__					(     self   ):
       snake_case__  :    List[str]							       =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*a   )
      def 						lowercase__					(     self   ):
       snake_case__  :    Union[str, Any]							       =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*a   )
      @slow
      def 						lowercase__					(     self   ):
       for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        snake_case__  :    Any							       =EsmModel.from_pretrained(a   )
        self.assertIsNotNone(a   )
      def 						lowercase__					(     self   ):
       snake_case__  :    List[Any]							       =self.model_tester.prepare_config_and_inputs()[0]
       snake_case__  :    int							       =EsmEmbeddings(config=a   )

       snake_case__  :    Union[str, Any]							       =torch.as_tensor([[1_2, 3_1, 1_3, model.padding_idx]]   )
       snake_case__  :    List[str]							       =torch.as_tensor(
           [
               [
                   0 + model.padding_idx + 1,
                   1 + model.padding_idx + 1,
                   2 + model.padding_idx + 1,
                   model.padding_idx,
               ]
           ]   )
       snake_case__  :    Optional[Any]							       =create_position_ids_from_input_ids(a				,			model.padding_idx   )
       self.assertEqual(position_ids.shape				,			expected_positions.shape   )
       self.assertTrue(torch.all(torch.eq(a				,			a   )   )   )
      def 						lowercase__					(     self   ):
       snake_case__  :    Union[str, Any]							       =self.model_tester.prepare_config_and_inputs()[0]
       snake_case__  :    Any							       =EsmEmbeddings(config=a   )

       snake_case__  :    str							       =torch.empty(2				,			4				,			3_0   )
       snake_case__  :    str							       =[
           0 + embeddings.padding_idx + 1,
           1 + embeddings.padding_idx + 1,
           2 + embeddings.padding_idx + 1,
           3 + embeddings.padding_idx + 1,
       ]
       snake_case__  :    List[str]							       =torch.as_tensor([expected_single_positions, expected_single_positions]   )
       snake_case__  :    int							       =embeddings.create_position_ids_from_inputs_embeds(a   )
       self.assertEqual(position_ids.shape				,			expected_positions.shape   )
       self.assertTrue(torch.all(torch.eq(a				,			a   )   )   )
      @unittest.skip("""Esm does not support embedding resizing"""   )
      def 						lowercase__					(     self   ):
       pass
      @unittest.skip("""Esm does not support embedding resizing"""   )
      def 						lowercase__					(     self   ):
       pass



      @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""   )
      def 						lowercase__					(     self   ):
       pass







@require_torch
class  _lowercase   (   _A						):
      @slow
      def 						lowercase__					(     self   ):
       with torch.no_grad():
        snake_case__  :    Optional[int]							       =EsmForMaskedLM.from_pretrained("""facebook/esm2_t6_8M_UR50D"""   )
        model.eval()
        snake_case__  :    Any							       =torch.tensor([[0, 1, 2, 3, 4, 5]]   )
        snake_case__  :    int							       =model(a   )[0]

        snake_case__  :    str							       =3_3

        snake_case__  :    Tuple							       =torch.Size((1, 6, vocab_size)   )
        self.assertEqual(output.shape				,			a   )

        snake_case__  :    List[Any]							       =torch.tensor(
            [[[8.9215, -10.5898, -6.4671], [-6.3967, -13.9114, -1.1212], [-7.7812, -13.9516, -3.7406]]]   )
        self.assertTrue(torch.allclose(output[:, :3, :3]				,			a				,			atol=1e-4   )   )



      @slow
      def 						lowercase__					(     self   ):
       with torch.no_grad():
        snake_case__  :    Optional[int]							       =EsmModel.from_pretrained("""facebook/esm2_t6_8M_UR50D"""   )
        model.eval()

        snake_case__  :    Dict							       =torch.tensor([[0, 6, 4, 1_3, 5, 4, 1_6, 1_2, 1_1, 7, 2]]   )
        snake_case__  :    Optional[int]							       =model(a   )[0]
        # compare the actual values for a slice.
        snake_case__  :    Tuple							       =torch.tensor(
            [[[0.1444, 0.5413, 0.3248], [0.3034, 0.0053, 0.3108], [0.3228, -0.2499, 0.3415]]]   )
        self.assertTrue(torch.allclose(output[:, :3, :3]				,			a				,			atol=1e-4   )   )













from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCamelCase	:		Dict										=   logging.get_logger(__name__)

__lowerCamelCase	:		Union[str, Any]										=   {
    """xlm-mlm-en-2048""": """https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json""",
    """xlm-mlm-ende-1024""": """https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-enfr-1024""": """https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json""",
    """xlm-mlm-enro-1024""": """https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json""",
    """xlm-mlm-tlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json""",
    """xlm-mlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json""",
    """xlm-clm-enfr-1024""": """https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json""",
    """xlm-clm-ende-1024""": """https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-17-1280""": """https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json""",
    """xlm-mlm-100-1280""": """https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json""",
}







class  _lowercase   (   _A						):
      _a							:						int										=							'xlm'
      _a							:						List[Any]										=							{
          'hidden_size': 'emb_dim',
          'num_attention_heads': 'n_heads',
          'num_hidden_layers': 'n_layers',
          'n_words': 'vocab_size',  # For backward compatibility
      }
      def __init__(     self				,			a=3_0_1_4_5				,			a=2_0_4_8				,			a=1_2				,			a=1_6				,			a=0.1				,			a=0.1				,			a=True				,			a=False				,			a=False				,			a=False				,			a=1				,			a=True				,			a=5_1_2				,			a=2_0_4_8**-0.5				,			a=1e-12				,			a=0.02				,			a=0				,			a=1				,			a=2				,			a=3				,			a=5				,			a=True				,			a="first"				,			a=True				,			a=None				,			a=True				,			a=0.1				,			a=5				,			a=5				,			a=0				,			a=0				,			a=2				,			a=0				,			**a				,			):
       snake_case__  :    str							       =vocab_size
       snake_case__  :    List[Any]							       =emb_dim
       snake_case__  :    List[str]							       =n_layers
       snake_case__  :    Any							       =n_heads
       snake_case__  :    Any							       =dropout
       snake_case__  :    Optional[int]							       =attention_dropout
       snake_case__  :    Dict							       =gelu_activation
       snake_case__  :    Tuple							       =sinusoidal_embeddings
       snake_case__  :    Optional[int]							       =causal
       snake_case__  :    Optional[int]							       =asm
       snake_case__  :    Optional[int]							       =n_langs
       snake_case__  :    Union[str, Any]							       =use_lang_emb
       snake_case__  :    Optional[Any]							       =layer_norm_eps
       snake_case__  :    Union[str, Any]							       =bos_index
       snake_case__  :    List[Any]							       =eos_index
       snake_case__  :    Optional[Any]							       =pad_index
       snake_case__  :    Union[str, Any]							       =unk_index
       snake_case__  :    Dict							       =mask_index
       snake_case__  :    str							       =is_encoder
       snake_case__  :    Optional[Any]							       =max_position_embeddings
       snake_case__  :    Any							       =embed_init_std
       snake_case__  :    Optional[Any]							       =init_std
       snake_case__  :    str							       =summary_type
       snake_case__  :    List[Any]							       =summary_use_proj
       snake_case__  :    str							       =summary_activation
       snake_case__  :    Optional[Any]							       =summary_proj_to_labels
       snake_case__  :    Optional[int]							       =summary_first_dropout
       snake_case__  :    Optional[Any]							       =start_n_top
       snake_case__  :    List[Any]							       =end_n_top
       snake_case__  :    Union[str, Any]							       =mask_token_id
       snake_case__  :    Any							       =lang_id

       if "n_words" in kwargs:
        snake_case__  :    int							       =kwargs["""n_words"""]

       super().__init__(pad_token_id=a				,			bos_token_id=a				,			**a   )







class  _lowercase   (   _A						):
      @property
      def 						lowercase__					(     self   ):
       if self.task == "multiple-choice":
        snake_case__  :    Any							       ={0: """batch""", 1: """choice""", 2: """sequence"""}
       else:
        snake_case__  :    Union[str, Any]							       ={0: """batch""", 1: """sequence"""}
       return OrderedDict(
           [
               ("""input_ids""", dynamic_axis),
               ("""attention_mask""", dynamic_axis),
               ("""token_type_ids""", dynamic_axis),
           ]   )





