





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          AlbertForMaskedLM,
          AlbertForMultipleChoice,
          AlbertForPreTraining,
          AlbertForQuestionAnswering,
          AlbertForSequenceClassification,
          AlbertForTokenClassification,
          AlbertModel,
      )
      from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class      lowercase__    :







  def __init__(	self    ,						_lowercase    ,						_lowercase=13    ,						_lowercase=7    ,						_lowercase=True    ,						_lowercase=True    ,						_lowercase=True    ,						_lowercase=True    ,						_lowercase=99    ,						_lowercase=16    ,						_lowercase=36    ,						_lowercase=6    ,						_lowercase=6    ,						_lowercase=6    ,						_lowercase=37    ,						_lowercase="gelu"    ,						_lowercase=0.1    ,						_lowercase=0.1    ,						_lowercase=512    ,						_lowercase=16    ,						_lowercase=2    ,						_lowercase=0.02    ,						_lowercase=3    ,						_lowercase=4    ,						_lowercase=None    ,						):
       lowerCAmelCase_					:       Dict    =		parent
       lowerCAmelCase_					:       List[Any]    =		batch_size
       lowerCAmelCase_					:       int    =		seq_length
       lowerCAmelCase_					:       int    =		is_training
       lowerCAmelCase_					:       Union[str, Any]    =		use_input_mask
       lowerCAmelCase_					:       Dict    =		use_token_type_ids
       lowerCAmelCase_					:       str    =		use_labels
       lowerCAmelCase_					:       Optional[int]    =		vocab_size
       lowerCAmelCase_					:       str    =		embedding_size
       lowerCAmelCase_					:       Dict    =		hidden_size
       lowerCAmelCase_					:       List[Any]    =		num_hidden_layers
       lowerCAmelCase_					:       List[str]    =		num_hidden_groups
       lowerCAmelCase_					:       Dict    =		num_attention_heads
       lowerCAmelCase_					:       int    =		intermediate_size
       lowerCAmelCase_					:       int    =		hidden_act
       lowerCAmelCase_					:       Any    =		hidden_dropout_prob
       lowerCAmelCase_					:       Optional[int]    =		attention_probs_dropout_prob
       lowerCAmelCase_					:       Union[str, Any]    =		max_position_embeddings
       lowerCAmelCase_					:       List[Any]    =		type_vocab_size
       lowerCAmelCase_					:       Any    =		type_sequence_label_size
       lowerCAmelCase_					:       Tuple    =		initializer_range
       lowerCAmelCase_					:       List[str]    =		num_labels
       lowerCAmelCase_					:       Dict    =		num_choices
       lowerCAmelCase_					:       Dict    =		scope







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       str    =		ids_tensor([self.batch_size, self.seq_length]    ,						self.vocab_size						)

       lowerCAmelCase_					:       Optional[Any]    =		None
       if self.use_input_mask:
            lowerCAmelCase_					:       List[str]    =		random_attention_mask([self.batch_size, self.seq_length]						)

       lowerCAmelCase_					:       Optional[int]    =		None
       if self.use_token_type_ids:
            lowerCAmelCase_					:       int    =		ids_tensor([self.batch_size, self.seq_length]    ,						self.type_vocab_size						)

       lowerCAmelCase_					:       Union[str, Any]    =		None
       lowerCAmelCase_					:       List[str]    =		None
       lowerCAmelCase_					:       Optional[Any]    =		None
       if self.use_labels:
            lowerCAmelCase_					:       Dict    =		ids_tensor([self.batch_size]    ,						self.type_sequence_label_size						)
            lowerCAmelCase_					:       Optional[Any]    =		ids_tensor([self.batch_size, self.seq_length]    ,						self.num_labels						)
            lowerCAmelCase_					:       List[str]    =		ids_tensor([self.batch_size]    ,						self.num_choices						)

       lowerCAmelCase_					:       Optional[Any]    =		self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def 	UpperCAmelCase__		(	self						):
       return AlbertConfig(
           vocab_size=self.vocab_size    ,						hidden_size=self.hidden_size    ,						num_hidden_layers=self.num_hidden_layers    ,						num_attention_heads=self.num_attention_heads    ,						intermediate_size=self.intermediate_size    ,						hidden_act=self.hidden_act    ,						hidden_dropout_prob=self.hidden_dropout_prob    ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,						max_position_embeddings=self.max_position_embeddings    ,						type_vocab_size=self.type_vocab_size    ,						initializer_range=self.initializer_range    ,						num_hidden_groups=self.num_hidden_groups    ,						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Tuple    =		AlbertModel(config=_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       Union[str, Any]    =		model(_lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase						)
       lowerCAmelCase_					:       List[Any]    =		model(_lowercase    ,						token_type_ids=_lowercase						)
       lowerCAmelCase_					:       List[str]    =		model(_lowercase						)
       self.parent.assertEqual(result.last_hidden_state.shape    ,						(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape    ,						(self.batch_size, self.hidden_size)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       int    =		AlbertForPreTraining(config=_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       int    =		model(
           _lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase    ,						labels=_lowercase    ,						sentence_order_label=_lowercase    ,						)
       self.parent.assertEqual(result.prediction_logits.shape    ,						(self.batch_size, self.seq_length, self.vocab_size)						)
       self.parent.assertEqual(result.sop_logits.shape    ,						(self.batch_size, config.num_labels)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       List[str]    =		AlbertForMaskedLM(config=_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       int    =		model(_lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase    ,						labels=_lowercase						)
       self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.seq_length, self.vocab_size)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[int]    =		AlbertForQuestionAnswering(config=_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       Union[str, Any]    =		model(
           _lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase    ,						start_positions=_lowercase    ,						end_positions=_lowercase    ,						)
       self.parent.assertEqual(result.start_logits.shape    ,						(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape    ,						(self.batch_size, self.seq_length)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       List[str]    =		self.num_labels
       lowerCAmelCase_					:       str    =		AlbertForSequenceClassification(_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       Tuple    =		model(_lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase    ,						labels=_lowercase						)
       self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.num_labels)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[int]    =		self.num_labels
       lowerCAmelCase_					:       Any    =		AlbertForTokenClassification(config=_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       List[Any]    =		model(_lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase    ,						labels=_lowercase						)
       self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.seq_length, self.num_labels)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       List[Any]    =		self.num_choices
       lowerCAmelCase_					:       Dict    =		AlbertForMultipleChoice(config=_lowercase						)
       model.to(_lowercase						)
       model.eval()
       lowerCAmelCase_					:       List[str]    =		input_ids.unsqueeze(1						).expand(-1    ,						self.num_choices    ,						-1						).contiguous()
       lowerCAmelCase_					:       Any    =		token_type_ids.unsqueeze(1						).expand(-1    ,						self.num_choices    ,						-1						).contiguous()
       lowerCAmelCase_					:       List[Any]    =		input_mask.unsqueeze(1						).expand(-1    ,						self.num_choices    ,						-1						).contiguous()
       lowerCAmelCase_					:       Optional[int]    =		model(
           _lowercase    ,						attention_mask=_lowercase    ,						token_type_ids=_lowercase    ,						labels=_lowercase    ,						)
       self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.num_choices)						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		self.prepare_config_and_inputs()
       (
           (
           lowerCAmelCase_
       )   ,  (
           lowerCAmelCase_
       )   ,  (
           lowerCAmelCase_
       )   ,  (
           lowerCAmelCase_
       )   ,  (
           lowerCAmelCase_
       )   ,  (
           lowerCAmelCase_
       )   ,  (
           lowerCAmelCase_
       )   ,  
       )					:       int    =		config_and_inputs
       lowerCAmelCase_					:       Optional[Any]    =		{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
       return config, inputs_dict





@require_torch
class      lowercase__    (							__A     ,					__A     ,					unittest.TestCase  ):
  __UpperCamelCase	     =  (
      (
          AlbertModel,
          AlbertForPreTraining,
          AlbertForMaskedLM,
          AlbertForMultipleChoice,
          AlbertForSequenceClassification,
          AlbertForTokenClassification,
          AlbertForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  __UpperCamelCase	     =  (
      {
          """feature-extraction""": AlbertModel,
          """fill-mask""": AlbertForMaskedLM,
          """question-answering""": AlbertForQuestionAnswering,
          """text-classification""": AlbertForSequenceClassification,
          """token-classification""": AlbertForTokenClassification,
          """zero-shot""": AlbertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  __UpperCamelCase	     =  True







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase=False						):
       lowerCAmelCase_					:       Dict    =		super()._prepare_for_class(_lowercase    ,						_lowercase    ,						return_labels=_lowercase						)

       if return_labels:
            if model_class in get_values(_lowercase						):
                 lowerCAmelCase_					:       Tuple    =		torch.zeros(
                     (self.model_tester.batch_size, self.model_tester.seq_length)    ,						dtype=torch.long    ,						device=_lowercase						)
                 lowerCAmelCase_					:       int    =		torch.zeros(
                     self.model_tester.batch_size    ,						dtype=torch.long    ,						device=_lowercase						)
       return inputs_dict







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[Any]    =		AlbertModelTester(self						)
       lowerCAmelCase_					:       int    =		ConfigTester(self    ,						config_class=_lowercase    ,						hidden_size=37						)







  def 	UpperCAmelCase__		(	self						):
       self.config_tester.run_common_tests()







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       str    =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       List[Any]    =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[int]    =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Union[str, Any]    =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Any    =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_lowercase						)







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
            lowerCAmelCase_					:       Optional[Any]    =		type
            self.model_tester.create_and_check_model(*_lowercase						)







  @slow
  def 	UpperCAmelCase__		(	self						):
       for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            lowerCAmelCase_					:       Optional[Any]    =		AlbertModel.from_pretrained(_lowercase						)
            self.assertIsNotNone(_lowercase						)





@require_torch
class      lowercase__    (							unittest.TestCase  ):







  @slow
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       str    =		AlbertModel.from_pretrained("""albert-base-v2"""						)
       lowerCAmelCase_					:       Optional[int]    =		torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]						)
       lowerCAmelCase_					:       List[Any]    =		torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]						)
       with torch.no_grad():
            lowerCAmelCase_					:       Optional[Any]    =		model(_lowercase    ,						attention_mask=_lowercase						)[0]
       lowerCAmelCase_					:       Dict    =		torch.Size((1, 11, 768)						)
       self.assertEqual(output.shape    ,						_lowercase						)
       lowerCAmelCase_					:       Dict    =		torch.tensor(
           [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]						)

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,						_lowercase    ,						atol=1e-4						)						)







from collections import OrderedDict
from typing import Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...feature_extraction_utils import FeatureExtractionMixin
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...tokenization_utils_base import PreTrainedTokenizerBase
from ...utils import TensorType, logging


UpperCAmelCase_					:   Optional[Any] 							=						logging.get_logger(__name__)

UpperCAmelCase_					:   Any 							=						{
    """deepmind/language-perceiver""": """https://huggingface.co/deepmind/language-perceiver/resolve/main/config.json""",
    # See all Perceiver models at https://huggingface.co/models?filter=perceiver
}





class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  """perceiver"""







  def __init__(	self    ,						_lowercase=256    ,						_lowercase=1_280    ,						_lowercase=768    ,						_lowercase=1    ,						_lowercase=26    ,						_lowercase=8    ,						_lowercase=8    ,						_lowercase=None    ,						_lowercase=None    ,						_lowercase="kv"    ,						_lowercase=1    ,						_lowercase=1    ,						_lowercase="gelu"    ,						_lowercase=0.1    ,						_lowercase=0.02    ,						_lowercase=1e-12    ,						_lowercase=True    ,						_lowercase=262    ,						_lowercase=2_048    ,						_lowercase=56    ,						_lowercase=[368, 496]    ,						_lowercase=16    ,						_lowercase=1_920    ,						_lowercase=16    ,						_lowercase=[1, 16, 224, 224]    ,						**_lowercase    ,						):
       super().__init__(**_lowercase						)

       lowerCAmelCase_					:       Optional[int]    =		num_latents
       lowerCAmelCase_					:       List[str]    =		d_latents
       lowerCAmelCase_					:       int    =		d_model
       lowerCAmelCase_					:       Dict    =		num_blocks
       lowerCAmelCase_					:       Union[str, Any]    =		num_self_attends_per_block
       lowerCAmelCase_					:       List[str]    =		num_self_attention_heads
       lowerCAmelCase_					:       List[str]    =		num_cross_attention_heads
       lowerCAmelCase_					:       List[Any]    =		qk_channels
       lowerCAmelCase_					:       Optional[Any]    =		v_channels
       lowerCAmelCase_					:       Optional[Any]    =		cross_attention_shape_for_attention
       lowerCAmelCase_					:       Optional[int]    =		self_attention_widening_factor
       lowerCAmelCase_					:       int    =		cross_attention_widening_factor
       lowerCAmelCase_					:       Union[str, Any]    =		hidden_act
       lowerCAmelCase_					:       int    =		attention_probs_dropout_prob
       lowerCAmelCase_					:       int    =		initializer_range
       lowerCAmelCase_					:       Optional[Any]    =		layer_norm_eps
       lowerCAmelCase_					:       Optional[Any]    =		use_query_residual
       # masked language modeling attributes
       lowerCAmelCase_					:       Tuple    =		vocab_size
       lowerCAmelCase_					:       List[Any]    =		max_position_embeddings
       # image classification attributes
       lowerCAmelCase_					:       List[Any]    =		image_size
       # flow attributes
       lowerCAmelCase_					:       Dict    =		train_size
       # multimodal autoencoding attributes
       lowerCAmelCase_					:       Optional[Any]    =		num_frames
       lowerCAmelCase_					:       int    =		audio_samples_per_frame
       lowerCAmelCase_					:       Any    =		samples_per_patch
       lowerCAmelCase_					:       Any    =		output_shape





class      lowercase__    (							__A  ):







  @property
  def 	UpperCAmelCase__		(	self						):
       if self.task == "multiple-choice":
            lowerCAmelCase_					:       Optional[Any]    =		{0: """batch""", 1: """choice""", 2: """sequence"""}
       else:
            lowerCAmelCase_					:       Optional[int]    =		{0: """batch""", 1: """sequence"""}
       return OrderedDict(
           [
               ("""inputs""", dynamic_axis),
               ("""attention_mask""", dynamic_axis),
           ]						)







  @property
  def 	UpperCAmelCase__		(	self						):
       return 1e-4







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = -1    ,						_lowercase = -1    ,						_lowercase = -1    ,						_lowercase = False    ,						_lowercase = None    ,						_lowercase = 3    ,						_lowercase = 40    ,						_lowercase = 40    ,						):
       # copied from `transformers.onnx.config.OnnxConfig` and slightly altered/simplified

       if isinstance(_lowercase    ,						_lowercase						):
            # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
            lowerCAmelCase_					:       Union[str, Any]    =		compute_effective_axis_dimension(
                _lowercase    ,						fixed_dimension=OnnxConfig.default_fixed_batch    ,						num_token_to_add=0						)
            # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
            lowerCAmelCase_					:       int    =		preprocessor.num_special_tokens_to_add(_lowercase						)
            lowerCAmelCase_					:       int    =		compute_effective_axis_dimension(
                _lowercase    ,						fixed_dimension=OnnxConfig.default_fixed_sequence    ,						num_token_to_add=_lowercase						)
            # Generate dummy inputs according to compute batch and sequence
            lowerCAmelCase_					:       Optional[int]    =		[""" """.join(["""a"""]						) * seq_length] * batch_size
            lowerCAmelCase_					:       Optional[Any]    =		dict(preprocessor(_lowercase    ,						return_tensors=_lowercase						)						)
            lowerCAmelCase_					:       Optional[Any]    =		inputs.pop("""input_ids"""						)
            return inputs
       elif isinstance(_lowercase    ,						_lowercase						) and preprocessor.model_input_names[0] == "pixel_values":
            # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
            lowerCAmelCase_					:       Any    =		compute_effective_axis_dimension(_lowercase    ,						fixed_dimension=OnnxConfig.default_fixed_batch						)
            lowerCAmelCase_					:       str    =		self._generate_dummy_images(_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						)
            lowerCAmelCase_					:       List[str]    =		dict(preprocessor(images=_lowercase    ,						return_tensors=_lowercase						)						)
            lowerCAmelCase_					:       Tuple    =		inputs.pop("""pixel_values"""						)
            return inputs
       else:
            raise ValueError(
                """Unable to generate dummy inputs for the model. Please provide a tokenizer or a preprocessor."""						)
