



import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          MegatronBertForCausalLM,
          MegatronBertForMaskedLM,
          MegatronBertForMultipleChoice,
          MegatronBertForNextSentencePrediction,
          MegatronBertForPreTraining,
          MegatronBertForQuestionAnswering,
          MegatronBertForSequenceClassification,
          MegatronBertForTokenClassification,
          MegatronBertModel,
      )

class       _SCREAMING_SNAKE_CASE	:





     def __init__(       self :							str     ,  UpperCamelCase :							str     ,  UpperCamelCase :							Dict=1_3     ,  UpperCamelCase :							Dict=7     ,  UpperCamelCase :							Any=True     ,  UpperCamelCase :							Any=True     ,  UpperCamelCase :							Optional[int]=True     ,  UpperCamelCase :							Any=True     ,  UpperCamelCase :							Dict=9_9     ,  UpperCamelCase :							int=6_4     ,  UpperCamelCase :							Dict=3_2     ,  UpperCamelCase :							int=5     ,  UpperCamelCase :							Union[str, Any]=4     ,  UpperCamelCase :							Optional[int]=3_7     ,  UpperCamelCase :							List[Any]="gelu"     ,  UpperCamelCase :							Any=0.1     ,  UpperCamelCase :							Dict=0.1     ,  UpperCamelCase :							Union[str, Any]=5_1_2     ,  UpperCamelCase :							List[str]=1_6     ,  UpperCamelCase :							str=2     ,  UpperCamelCase :							Optional[int]=0.0_2     ,  UpperCamelCase :							Optional[int]=3     ,  UpperCamelCase :							Optional[int]=4     ,  UpperCamelCase :							Dict=None     ,  )->int:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	parent
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	batch_size
      __SCREAMING_SNAKE_CASE       :    str       =	seq_length
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	is_training
      __SCREAMING_SNAKE_CASE       :    Tuple       =	use_input_mask
      __SCREAMING_SNAKE_CASE       :    int       =	use_token_type_ids
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	use_labels
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	vocab_size
      __SCREAMING_SNAKE_CASE       :    Any       =	hidden_size
      __SCREAMING_SNAKE_CASE       :    Tuple       =	embedding_size
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	num_hidden_layers
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	num_attention_heads
      __SCREAMING_SNAKE_CASE       :    List[str]       =	intermediate_size
      __SCREAMING_SNAKE_CASE       :    List[str]       =	hidden_act
      __SCREAMING_SNAKE_CASE       :    Dict       =	hidden_dropout_prob
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE       :    Tuple       =	max_position_embeddings
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	type_vocab_size
      __SCREAMING_SNAKE_CASE       :    List[str]       =	type_sequence_label_size
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	initializer_range
      __SCREAMING_SNAKE_CASE       :    Any       =	num_labels
      __SCREAMING_SNAKE_CASE       :    Any       =	num_choices
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	scope





     def 	__snake_case    (       self :							Any				)->List[str]:
      __SCREAMING_SNAKE_CASE       :    List[str]       =	ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size				)

      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	None
      if self.use_input_mask:
       __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	random_attention_mask([self.batch_size, self.seq_length]				)

      __SCREAMING_SNAKE_CASE       :    Dict       =	None
      if self.use_token_type_ids:
       __SCREAMING_SNAKE_CASE       :    Optional[int]       =	ids_tensor([self.batch_size, self.seq_length]     ,  self.type_vocab_size				)

      __SCREAMING_SNAKE_CASE       :    Dict       =	None
      __SCREAMING_SNAKE_CASE       :    Tuple       =	None
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	None
      if self.use_labels:
       __SCREAMING_SNAKE_CASE       :    str       =	ids_tensor([self.batch_size]     ,  self.type_sequence_label_size				)
       __SCREAMING_SNAKE_CASE       :    str       =	ids_tensor([self.batch_size, self.seq_length]     ,  self.num_labels				)
       __SCREAMING_SNAKE_CASE       :    int       =	ids_tensor([self.batch_size]     ,  self.num_choices				)

      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	self.get_config()

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





     def 	__snake_case    (       self :							str				)->List[str]:
      return MegatronBertConfig(
          vocab_size=self.vocab_size     ,  hidden_size=self.hidden_size     ,  num_hidden_layers=self.num_hidden_layers     ,  num_attention_heads=self.num_attention_heads     ,  intermediate_size=self.intermediate_size     ,  embedding_size=self.embedding_size     ,  hidden_act=self.hidden_act     ,  hidden_dropout_prob=self.hidden_dropout_prob     ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,  max_position_embeddings=self.max_position_embeddings     ,  type_vocab_size=self.type_vocab_size     ,  is_decoder=__snake_case     ,  initializer_range=self.initializer_range     ,  )





     def 	__snake_case    (       self :							int     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							int     ,  UpperCamelCase :							Tuple     ,  UpperCamelCase :							int     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							Optional[int]				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	MegatronBertModel(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    Tuple       =	model(__snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case				)
      __SCREAMING_SNAKE_CASE       :    str       =	model(__snake_case     ,  token_type_ids=__snake_case				)
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	model(__snake_case				)

      self.parent.assertEqual(result.last_hidden_state.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)				)
      self.parent.assertEqual(result.pooler_output.shape     ,  (self.batch_size, self.hidden_size)				)





     def 	__snake_case    (       self :							int     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							Any     ,  UpperCamelCase :							int     ,  UpperCamelCase :							Dict     ,  UpperCamelCase :							Union[str, Any]				)->List[Any]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	MegatronBertForMaskedLM(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	model(__snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case				)
      self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)				)





     def 	__snake_case    (       self :							Tuple     ,  UpperCamelCase :							int     ,  UpperCamelCase :							str     ,  UpperCamelCase :							Any     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							Any     ,  UpperCamelCase :							List[Any]				)->Optional[int]:
      __SCREAMING_SNAKE_CASE       :    Any       =	MegatronBertForCausalLM(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    str       =	model(__snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case				)
      self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)				)





     def 	__snake_case    (       self :							str     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							str				)->Dict:
      __SCREAMING_SNAKE_CASE       :    Dict       =	MegatronBertForNextSentencePrediction(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    int       =	model(
          __snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case     ,  )
      self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, 2)				)





     def 	__snake_case    (       self :							List[str]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							int     ,  UpperCamelCase :							Tuple     ,  UpperCamelCase :							Dict     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							Optional[int]				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	MegatronBertForPreTraining(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	model(
          __snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case     ,  next_sentence_label=__snake_case     ,  )
      self.parent.assertEqual(result.prediction_logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)				)
      self.parent.assertEqual(result.seq_relationship_logits.shape     ,  (self.batch_size, 2)				)





     def 	__snake_case    (       self :							Optional[int]     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							Tuple     ,  UpperCamelCase :							List[Any]     ,  UpperCamelCase :							Optional[int]				)->Optional[Any]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	MegatronBertForQuestionAnswering(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    Tuple       =	model(
          __snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  start_positions=__snake_case     ,  end_positions=__snake_case     ,  )
      self.parent.assertEqual(result.start_logits.shape     ,  (self.batch_size, self.seq_length)				)
      self.parent.assertEqual(result.end_logits.shape     ,  (self.batch_size, self.seq_length)				)





     def 	__snake_case    (       self :							Dict     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							Any     ,  UpperCamelCase :							int     ,  UpperCamelCase :							int     ,  UpperCamelCase :							Tuple     ,  UpperCamelCase :							Tuple				)->Any:
      __SCREAMING_SNAKE_CASE       :    int       =	self.num_labels
      __SCREAMING_SNAKE_CASE       :    List[str]       =	MegatronBertForSequenceClassification(__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    List[str]       =	model(__snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case				)
      self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_labels)				)





     def 	__snake_case    (       self :							List[str]     ,  UpperCamelCase :							Optional[int]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							int     ,  UpperCamelCase :							Dict     ,  UpperCamelCase :							Tuple     ,  UpperCamelCase :							List[Any]     ,  UpperCamelCase :							Dict				)->str:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	self.num_labels
      __SCREAMING_SNAKE_CASE       :    int       =	MegatronBertForTokenClassification(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    Any       =	model(__snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case				)
      self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.num_labels)				)





     def 	__snake_case    (       self :							Optional[Any]     ,  UpperCamelCase :							Optional[Any]     ,  UpperCamelCase :							List[str]     ,  UpperCamelCase :							Dict     ,  UpperCamelCase :							Union[str, Any]     ,  UpperCamelCase :							Optional[Any]     ,  UpperCamelCase :							Tuple     ,  UpperCamelCase :							Union[str, Any]				)->Optional[int]:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	self.num_choices
      __SCREAMING_SNAKE_CASE       :    Tuple       =	MegatronBertForMultipleChoice(config=__snake_case				)
      model.to(__snake_case				)
      model.eval()
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	input_ids.unsqueeze(1				).expand(-1     ,  self.num_choices     ,  -1				).contiguous()
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	token_type_ids.unsqueeze(1				).expand(-1     ,  self.num_choices     ,  -1				).contiguous()
      __SCREAMING_SNAKE_CASE       :    List[str]       =	input_mask.unsqueeze(1				).expand(-1     ,  self.num_choices     ,  -1				).contiguous()
      __SCREAMING_SNAKE_CASE       :    Dict       =	model(
          __snake_case     ,  attention_mask=__snake_case     ,  token_type_ids=__snake_case     ,  labels=__snake_case     ,  )
      self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_choices)				)






     def 	__snake_case    (       self :							Dict				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    List[str]       =	self.prepare_config_and_inputs()
      (
          __SCREAMING_SNAKE_CASE    
      )       :    Optional[Any]       =	config_and_inputs
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
      return config, inputs_dict






@require_torch
class       _SCREAMING_SNAKE_CASE	(__SCREAMING_SNAKE_CASE			,    __SCREAMING_SNAKE_CASE			,    unittest.TestCase ):
     lowerCAmelCase					=			(
         (
             MegatronBertModel,
             MegatronBertForMaskedLM,
             MegatronBertForCausalLM,
             MegatronBertForMultipleChoice,
             MegatronBertForNextSentencePrediction,
             MegatronBertForPreTraining,
             MegatronBertForQuestionAnswering,
             MegatronBertForSequenceClassification,
             MegatronBertForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     lowerCAmelCase					=			(
         {
             """feature-extraction""": MegatronBertModel,
             """fill-mask""": MegatronBertForMaskedLM,
             """question-answering""": MegatronBertForQuestionAnswering,
             """text-classification""": MegatronBertForSequenceClassification,
             """text-generation""": MegatronBertForCausalLM,
             """token-classification""": MegatronBertForTokenClassification,
             """zero-shot""": MegatronBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     lowerCAmelCase					=			True
     # test_resize_embeddings = False
     lowerCAmelCase					=			False





     def 	__snake_case    (       self :							Any     ,  UpperCamelCase :							Optional[Any]     ,  UpperCamelCase :							str     ,  UpperCamelCase :							List[Any]=False				)->Dict:
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	super()._prepare_for_class(__snake_case     ,  __snake_case     ,  return_labels=__snake_case				)

      if return_labels:
       if model_class in get_values(__snake_case				):
        __SCREAMING_SNAKE_CASE       :    int       =	torch.zeros(
            (self.model_tester.batch_size, self.model_tester.seq_length)     ,  dtype=torch.long     ,  device=__snake_case				)
        __SCREAMING_SNAKE_CASE       :    str       =	torch.zeros(
            self.model_tester.batch_size     ,  dtype=torch.long     ,  device=__snake_case				)
      return inputs_dict





     def 	__snake_case    (       self :							Optional[Any]				)->str:
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	MegatronBertModelTester(self				)
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	ConfigTester(self     ,  config_class=__snake_case     ,  hidden_size=3_7				)





     def 	__snake_case    (       self :							Dict				)->str:
      self.config_tester.run_common_tests()





     def 	__snake_case    (       self :							Any				)->str:
      __SCREAMING_SNAKE_CASE       :    int       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_model(*__snake_case				)





     def 	__snake_case    (       self :							List[str]				)->Optional[Any]:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__snake_case				)





     def 	__snake_case    (       self :							Any				)->Optional[Any]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__snake_case				)





     def 	__snake_case    (       self :							int				)->int:
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__snake_case				)





     def 	__snake_case    (       self :							List[Any]				)->Optional[Any]:
      __SCREAMING_SNAKE_CASE       :    Tuple       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_pretraining(*__snake_case				)





     def 	__snake_case    (       self :							Union[str, Any]				)->Optional[int]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_question_answering(*__snake_case				)





     def 	__snake_case    (       self :							int				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    Dict       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__snake_case				)






     def 	__snake_case    (       self :							Optional[int]				)->List[Any]:
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_megatron_bert_for_token_classification(*__snake_case				)






def 	_lowerCAmelCase						(				__lowerCamelCase   :     Optional[Any]     ):



 """simple docstring"""





 return torch.tensor(
     a_		,	dtype=torch.long		,	device=a_		,	)


_lowerCamelCase						    = 1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class       _SCREAMING_SNAKE_CASE	(unittest.TestCase ):





     @slow
     @unittest.skip("Model is not available."				)
     def 	__snake_case    (       self :							Union[str, Any]				)->List[Any]:
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	'''nvidia/megatron-bert-uncased-345m'''
      if "MYDIR" in os.environ:
       __SCREAMING_SNAKE_CASE       :    Any       =	os.path.join(os.environ["MYDIR"]     ,  __snake_case				)
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	MegatronBertModel.from_pretrained(__snake_case				)
      model.to(__snake_case				)
      model.half()
      __SCREAMING_SNAKE_CASE       :    List[str]       =	_long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]				)
      with torch.no_grad():
       __SCREAMING_SNAKE_CASE       :    Any       =	model(__snake_case				)[0]
      __SCREAMING_SNAKE_CASE       :    Tuple       =	torch.Size((1, 9, 1_0_2_4)				)
      self.assertEqual(output.shape     ,  __snake_case				)

      __SCREAMING_SNAKE_CASE       :    str       =	[-0.6_0_4_0, -0.2_5_1_7, -0.1_0_2_5, 0.3_4_2_0, -0.6_7_5_8, -0.0_0_1_7, -0.1_0_8_9, -0.1_9_9_0, 0.5_7_2_8]
      for ii in range(3				):
       for jj in range(3				):
        __SCREAMING_SNAKE_CASE       :    int       =	output[0, ii, jj]
        __SCREAMING_SNAKE_CASE       :    str       =	expected[3 * ii + jj]
        __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	'''ii={} jj={} a={} b={}'''.format(__snake_case     ,  __snake_case     ,  __snake_case     ,  __snake_case				)
        self.assertTrue(math.isclose(__snake_case     ,  __snake_case     ,  rel_tol=__snake_case     ,  abs_tol=__snake_case				)     ,  msg=__snake_case				)











import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase						    =  logging.get_logger(__name__)

_lowerCamelCase						    =  {
    """BAAI/AltCLIP""": """https://huggingface.co/BAAI/AltCLIP/resolve/main/config.json""",
    # See all AltCLIP models at https://huggingface.co/models?filter=altclip
}

class       _SCREAMING_SNAKE_CASE	(UpperCamelCase ):
     lowerCAmelCase					=			"""altclip_text_model"""





     def __init__(       self :							Union[str, Any]     ,  UpperCamelCase :							List[Any]=2_5_0_0_0_2     ,  UpperCamelCase :							int=1_0_2_4     ,  UpperCamelCase :							Dict=2_4     ,  UpperCamelCase :							Dict=1_6     ,  UpperCamelCase :							Dict=4_0_9_6     ,  UpperCamelCase :							List[str]="gelu"     ,  UpperCamelCase :							Any=0.1     ,  UpperCamelCase :							List[str]=0.1     ,  UpperCamelCase :							Union[str, Any]=5_1_4     ,  UpperCamelCase :							List[Any]=1     ,  UpperCamelCase :							Any=0.0_2     ,  UpperCamelCase :							Any=0.0_2     ,  UpperCamelCase :							Union[str, Any]=1E-05     ,  UpperCamelCase :							List[str]=1     ,  UpperCamelCase :							Dict=0     ,  UpperCamelCase :							Tuple=2     ,  UpperCamelCase :							Union[str, Any]="absolute"     ,  UpperCamelCase :							Optional[int]=True     ,  UpperCamelCase :							Any=7_6_8     ,  **UpperCamelCase :							str     ,  )->Any:
      super().__init__(pad_token_id=UpperCamelCase     ,  bos_token_id=UpperCamelCase     ,  eos_token_id=UpperCamelCase     ,  **UpperCamelCase				)

      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	vocab_size
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	hidden_size
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	num_hidden_layers
      __SCREAMING_SNAKE_CASE       :    Any       =	num_attention_heads
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	hidden_act
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	intermediate_size
      __SCREAMING_SNAKE_CASE       :    int       =	hidden_dropout_prob
      __SCREAMING_SNAKE_CASE       :    List[str]       =	attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE       :    Dict       =	max_position_embeddings
      __SCREAMING_SNAKE_CASE       :    List[str]       =	type_vocab_size
      __SCREAMING_SNAKE_CASE       :    Any       =	initializer_range
      __SCREAMING_SNAKE_CASE       :    int       =	initializer_factor
      __SCREAMING_SNAKE_CASE       :    List[str]       =	layer_norm_eps
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	position_embedding_type
      __SCREAMING_SNAKE_CASE       :    List[str]       =	use_cache
      __SCREAMING_SNAKE_CASE       :    Any       =	project_dim






class       _SCREAMING_SNAKE_CASE	(UpperCamelCase ):
     lowerCAmelCase					=			"""altclip_vision_model"""





     def __init__(       self :							Dict     ,  UpperCamelCase :							Tuple=7_6_8     ,  UpperCamelCase :							Optional[int]=3_0_7_2     ,  UpperCamelCase :							Optional[Any]=5_1_2     ,  UpperCamelCase :							int=1_2     ,  UpperCamelCase :							List[Any]=1_2     ,  UpperCamelCase :							List[str]=3     ,  UpperCamelCase :							List[Any]=2_2_4     ,  UpperCamelCase :							int=3_2     ,  UpperCamelCase :							Optional[Any]="quick_gelu"     ,  UpperCamelCase :							List[Any]=1E-5     ,  UpperCamelCase :							List[str]=0.0     ,  UpperCamelCase :							Tuple=0.0_2     ,  UpperCamelCase :							str=1.0     ,  **UpperCamelCase :							Tuple     ,  )->Optional[int]:
      super().__init__(**UpperCamelCase				)

      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	hidden_size
      __SCREAMING_SNAKE_CASE       :    Dict       =	intermediate_size
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	projection_dim
      __SCREAMING_SNAKE_CASE       :    int       =	num_hidden_layers
      __SCREAMING_SNAKE_CASE       :    str       =	num_attention_heads
      __SCREAMING_SNAKE_CASE       :    Tuple       =	num_channels
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	patch_size
      __SCREAMING_SNAKE_CASE       :    str       =	image_size
      __SCREAMING_SNAKE_CASE       :    List[str]       =	initializer_range
      __SCREAMING_SNAKE_CASE       :    int       =	initializer_factor
      __SCREAMING_SNAKE_CASE       :    Tuple       =	attention_dropout
      __SCREAMING_SNAKE_CASE       :    List[str]       =	layer_norm_eps
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	hidden_act






     @classmethod
     def 	__snake_case    (       cls :							Union[str, Any]     ,  UpperCamelCase :							Union[str, os.PathLike]     ,  **UpperCamelCase :							Optional[Any]				)->"PretrainedConfig":
      cls._set_token_in_kwargs(UpperCamelCase				)

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       :    Optional[int]       =	cls.get_config_dict(UpperCamelCase     ,  **UpperCamelCase				)

      # get the vision config dict if we are loading from AltCLIPConfig
      if config_dict.get("model_type"				) == "altclip":
       __SCREAMING_SNAKE_CASE       :    List[str]       =	config_dict["vision_config"]

      if "model_type" in config_dict and hasattr(cls     ,  "model_type"				) and config_dict["model_type"] != cls.model_type:
       logger.warning(
           F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
           F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""				)

      return cls.from_dict(UpperCamelCase     ,  **UpperCamelCase				)






class       _SCREAMING_SNAKE_CASE	(UpperCamelCase ):
     lowerCAmelCase					=			"""altclip"""
     lowerCAmelCase					=			True





     def __init__(       self :							Optional[Any]     ,  UpperCamelCase :							List[str]=None     ,  UpperCamelCase :							List[str]=None     ,  UpperCamelCase :							Union[str, Any]=7_6_8     ,  UpperCamelCase :							Optional[Any]=2.6_5_9_2     ,  **UpperCamelCase :							int				)->Any:
      # If `_config_dict` exist, we use them for the backward compatibility.
      # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
      # of confusion!).
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	kwargs.pop("text_config_dict"     ,  UpperCamelCase				)
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	kwargs.pop("vision_config_dict"     ,  UpperCamelCase				)

      super().__init__(**UpperCamelCase				)

      # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
      # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
      # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
      if text_config_dict is not None:
       if text_config is None:
        __SCREAMING_SNAKE_CASE       :    Tuple       =	{}

       # This is the complete result when using `text_config_dict`.
       __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	AltCLIPTextConfig(**UpperCamelCase				).to_dict()

       # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
       for key, value in _text_config_dict.items():
        if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
         # If specified in `text_config_dict`
         if key in text_config_dict:
          __SCREAMING_SNAKE_CASE       :    List[str]       =	(
              F"""`{key}` is found in both `text_config_dict` and `text_config` but with different values. """
              F"""The value `text_config_dict[\"{key}\"]` will be used instead."""
          )
         # If inferred from default argument values (just to be super careful)
         else:
          __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	(
              F"""`text_config_dict` is provided which will be used to initialize `AltCLIPTextConfig`. The """
              F"""value `text_config[\"{key}\"]` will be overriden."""
          )
         logger.warning(UpperCamelCase				)

            # Update all values in `text_config` with the ones in `_text_config_dict`.
       text_config.update(_text_config_dict				)

      if vision_config_dict is not None:
       if vision_config is None:
        __SCREAMING_SNAKE_CASE       :    int       =	{}

       # This is the complete result when using `vision_config_dict`.
       __SCREAMING_SNAKE_CASE       :    Optional[int]       =	AltCLIPVisionConfig(**UpperCamelCase				).to_dict()
       # convert keys to string instead of integer
       if "id2label" in _vision_config_dict:
        __SCREAMING_SNAKE_CASE       :    List[Any]       =	{
            str(UpperCamelCase				): value for key, value in _vision_config_dict["id2label"].items()
        }

       # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
       for key, value in _vision_config_dict.items():
        if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
         # If specified in `vision_config_dict`
         if key in vision_config_dict:
          __SCREAMING_SNAKE_CASE       :    Dict       =	(
              F"""`{key}` is found in both `vision_config_dict` and `vision_config` but with different """
              F"""values. The value `vision_config_dict[\"{key}\"]` will be used instead."""
          )
         # If inferred from default argument values (just to be super careful)
         else:
          __SCREAMING_SNAKE_CASE       :    Dict       =	(
              F"""`vision_config_dict` is provided which will be used to initialize `AltCLIPVisionConfig`. """
              F"""The value `vision_config[\"{key}\"]` will be overriden."""
          )
         logger.warning(UpperCamelCase				)

            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
       vision_config.update(_vision_config_dict				)

      if text_config is None:
       __SCREAMING_SNAKE_CASE       :    List[Any]       =	{}
       logger.info("`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values."				)

      if vision_config is None:
       __SCREAMING_SNAKE_CASE       :    List[str]       =	{}
       logger.info("`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values."				)

      __SCREAMING_SNAKE_CASE       :    Tuple       =	AltCLIPTextConfig(**UpperCamelCase				)
      __SCREAMING_SNAKE_CASE       :    Tuple       =	AltCLIPVisionConfig(**UpperCamelCase				)

      __SCREAMING_SNAKE_CASE       :    List[Any]       =	projection_dim
      __SCREAMING_SNAKE_CASE       :    int       =	logit_scale_init_value
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	1.0





     @classmethod
     def 	__snake_case    (       cls :							Dict     ,  UpperCamelCase :							AltCLIPTextConfig     ,  UpperCamelCase :							AltCLIPVisionConfig     ,  **UpperCamelCase :							int				)->Dict:
      return cls(text_config=text_config.to_dict()     ,  vision_config=vision_config.to_dict()     ,  **UpperCamelCase				)






     def 	__snake_case    (       self :							str				)->List[str]:
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	copy.deepcopy(self.__dict__				)
      __SCREAMING_SNAKE_CASE       :    Dict       =	self.text_config.to_dict()
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	self.vision_config.to_dict()
      __SCREAMING_SNAKE_CASE       :    int       =	self.__class__.model_type
      return output






