




'''simple docstring'''

import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       MegatronBertForCausalLM,
       MegatronBertForMaskedLM,
       MegatronBertForMultipleChoice,
       MegatronBertForNextSentencePrediction,
       MegatronBertForPreTraining,
       MegatronBertForQuestionAnswering,
       MegatronBertForSequenceClassification,
       MegatronBertForTokenClassification,
       MegatronBertModel,
   )





class lowerCamelCase						:







  def __init__(		self     ,     lowercase__     ,     lowercase__=13     ,     lowercase__=7     ,     lowercase__=True     ,     lowercase__=True     ,     lowercase__=True     ,     lowercase__=True     ,     lowercase__=99     ,     lowercase__=64     ,     lowercase__=32     ,     lowercase__=5     ,     lowercase__=4     ,     lowercase__=37     ,     lowercase__="gelu"     ,     lowercase__=0.1     ,     lowercase__=0.1     ,     lowercase__=512     ,     lowercase__=16     ,     lowercase__=2     ,     lowercase__=0.02     ,     lowercase__=3     ,     lowercase__=4     ,     lowercase__=None     ,     )		->						Optional[int]:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     parent
        _snake_case		:	int					     =     batch_size
        _snake_case		:	Any					     =     seq_length
        _snake_case		:	str					     =     is_training
        _snake_case		:	Any					     =     use_input_mask
        _snake_case		:	List[Any]					     =     use_token_type_ids
        _snake_case		:	int					     =     use_labels
        _snake_case		:	Optional[int]					     =     vocab_size
        _snake_case		:	List[Any]					     =     hidden_size
        _snake_case		:	List[str]					     =     embedding_size
        _snake_case		:	List[str]					     =     num_hidden_layers
        _snake_case		:	str					     =     num_attention_heads
        _snake_case		:	Any					     =     intermediate_size
        _snake_case		:	List[str]					     =     hidden_act
        _snake_case		:	Any					     =     hidden_dropout_prob
        _snake_case		:	Any					     =     attention_probs_dropout_prob
        _snake_case		:	Union[str, Any]					     =     max_position_embeddings
        _snake_case		:	Dict					     =     type_vocab_size
        _snake_case		:	List[Any]					     =     type_sequence_label_size
        _snake_case		:	Dict					     =     initializer_range
        _snake_case		:	Union[str, Any]					     =     num_labels
        _snake_case		:	Dict					     =     num_choices
        _snake_case		:	Optional[Any]					     =     scope







  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        _snake_case		:	Tuple					     =     ids_tensor([self.batch_size, self.seq_length]     ,     self.vocab_size		)

        _snake_case		:	int					     =     None
        if self.use_input_mask:
              _snake_case		:	Any					     =     random_attention_mask([self.batch_size, self.seq_length]		)

        _snake_case		:	int					     =     None
        if self.use_token_type_ids:
              _snake_case		:	Optional[int]					     =     ids_tensor([self.batch_size, self.seq_length]     ,     self.type_vocab_size		)

        _snake_case		:	Tuple					     =     None
        _snake_case		:	Tuple					     =     None
        _snake_case		:	Optional[int]					     =     None
        if self.use_labels:
              _snake_case		:	List[str]					     =     ids_tensor([self.batch_size]     ,     self.type_sequence_label_size		)
              _snake_case		:	Dict					     =     ids_tensor([self.batch_size, self.seq_length]     ,     self.num_labels		)
              _snake_case		:	Tuple					     =     ids_tensor([self.batch_size]     ,     self.num_choices		)

        _snake_case		:	List[Any]					     =     self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        return MegatronBertConfig(
            vocab_size=self.vocab_size     ,     hidden_size=self.hidden_size     ,     num_hidden_layers=self.num_hidden_layers     ,     num_attention_heads=self.num_attention_heads     ,     intermediate_size=self.intermediate_size     ,     embedding_size=self.embedding_size     ,     hidden_act=self.hidden_act     ,     hidden_dropout_prob=self.hidden_dropout_prob     ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,     max_position_embeddings=self.max_position_embeddings     ,     type_vocab_size=self.type_vocab_size     ,     is_decoder=lowerCamelCase_     ,     initializer_range=self.initializer_range     ,     )







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						Tuple:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     MegatronBertModel(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	str					     =     model(lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_		)
        _snake_case		:	Optional[int]					     =     model(lowerCamelCase_     ,     token_type_ids=lowerCamelCase_		)
        _snake_case		:	Any					     =     model(lowerCamelCase_		)

        self.parent.assertEqual(result.last_hidden_state.shape     ,     (self.batch_size, self.seq_length, self.hidden_size)		)
        self.parent.assertEqual(result.pooler_output.shape     ,     (self.batch_size, self.hidden_size)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	int					     =     MegatronBertForMaskedLM(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	List[Any]					     =     model(lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_		)
        self.parent.assertEqual(result.logits.shape     ,     (self.batch_size, self.seq_length, self.vocab_size)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						Any:






        """simple docstring"""





        _snake_case		:	Optional[Any]					     =     MegatronBertForCausalLM(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	List[Any]					     =     model(lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_		)
        self.parent.assertEqual(result.logits.shape     ,     (self.batch_size, self.seq_length, self.vocab_size)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						List[str]:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     MegatronBertForNextSentencePrediction(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	Dict					     =     model(
            lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_     ,     )
        self.parent.assertEqual(result.logits.shape     ,     (self.batch_size, 2)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						Optional[int]:






        """simple docstring"""





        _snake_case		:	Dict					     =     MegatronBertForPreTraining(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	Union[str, Any]					     =     model(
            lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_     ,     next_sentence_label=lowerCamelCase_     ,     )
        self.parent.assertEqual(result.prediction_logits.shape     ,     (self.batch_size, self.seq_length, self.vocab_size)		)
        self.parent.assertEqual(result.seq_relationship_logits.shape     ,     (self.batch_size, 2)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	Tuple					     =     MegatronBertForQuestionAnswering(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	int					     =     model(
            lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     start_positions=lowerCamelCase_     ,     end_positions=lowerCamelCase_     ,     )
        self.parent.assertEqual(result.start_logits.shape     ,     (self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape     ,     (self.batch_size, self.seq_length)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						Optional[int]:






        """simple docstring"""





        _snake_case		:	Any					     =     self.num_labels
        _snake_case		:	Any					     =     MegatronBertForSequenceClassification(lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	int					     =     model(lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_		)
        self.parent.assertEqual(result.logits.shape     ,     (self.batch_size, self.num_labels)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	str					     =     self.num_labels
        _snake_case		:	Tuple					     =     MegatronBertForTokenClassification(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	Dict					     =     model(lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_		)
        self.parent.assertEqual(result.logits.shape     ,     (self.batch_size, self.seq_length, self.num_labels)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__     ,     lowercase__		)		->						List[str]:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     self.num_choices
        _snake_case		:	str					     =     MegatronBertForMultipleChoice(config=lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.eval()
        _snake_case		:	Union[str, Any]					     =     input_ids.unsqueeze(1		).expand(-1     ,     self.num_choices     ,     -1		).contiguous()
        _snake_case		:	List[Any]					     =     token_type_ids.unsqueeze(1		).expand(-1     ,     self.num_choices     ,     -1		).contiguous()
        _snake_case		:	List[Any]					     =     input_mask.unsqueeze(1		).expand(-1     ,     self.num_choices     ,     -1		).contiguous()
        _snake_case		:	List[str]					     =     model(
            lowerCamelCase_     ,     attention_mask=lowerCamelCase_     ,     token_type_ids=lowerCamelCase_     ,     labels=lowerCamelCase_     ,     )
        self.parent.assertEqual(result.logits.shape     ,     (self.batch_size, self.num_choices)		)







  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     self.prepare_config_and_inputs()
        (
            _snake_case					
        )		:	Optional[int]					     =     config_and_inputs
        _snake_case		:	List[Any]					     =     {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict





@require_torch
class lowerCamelCase						(_UpperCAmelCase						,      _UpperCAmelCase						,      unittest.TestCase					):
  _lowercase  :	Dict       					=				(
      (
          MegatronBertModel,
          MegatronBertForMaskedLM,
          MegatronBertForCausalLM,
          MegatronBertForMultipleChoice,
          MegatronBertForNextSentencePrediction,
          MegatronBertForPreTraining,
          MegatronBertForQuestionAnswering,
          MegatronBertForSequenceClassification,
          MegatronBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  _lowercase  :	Optional[Any]       					=				(
      {
          """feature-extraction""": MegatronBertModel,
          """fill-mask""": MegatronBertForMaskedLM,
          """question-answering""": MegatronBertForQuestionAnswering,
          """text-classification""": MegatronBertForSequenceClassification,
          """text-generation""": MegatronBertForCausalLM,
          """token-classification""": MegatronBertForTokenClassification,
          """zero-shot""": MegatronBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  _lowercase  :	Dict       					=				True
  # test_resize_embeddings = False
  _lowercase  :	Any       					=				False







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__     ,     lowercase__=False		)		->						Optional[int]:






        """simple docstring"""





        _snake_case		:	Tuple					     =     super()._prepare_for_class(lowerCamelCase_     ,     lowerCamelCase_     ,     return_labels=lowerCamelCase_		)

        if return_labels:
              if model_class in get_values(lowerCamelCase_		):
                    _snake_case		:	str					     =     torch.zeros(
                        (self.model_tester.batch_size, self.model_tester.seq_length)     ,     dtype=torch.long     ,     device=lowerCamelCase_		)
                    _snake_case		:	int					     =     torch.zeros(
                        self.model_tester.batch_size     ,     dtype=torch.long     ,     device=lowerCamelCase_		)
        return inputs_dict







  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	Optional[Any]					     =     MegatronBertModelTester(self		)
        _snake_case		:	Tuple					     =     ConfigTester(self     ,     config_class=lowerCamelCase_     ,     hidden_size=37		)







  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        self.config_tester.run_common_tests()







  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        _snake_case		:	Tuple					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_model(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_masked_lm(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        _snake_case		:	int					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        _snake_case		:	int					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_pretraining(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        _snake_case		:	Any					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_question_answering(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        _snake_case		:	Any					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*lowerCamelCase_		)







  def 				UpperCAmelCase_				(		self		)		->						Optional[int]:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_token_classification(*lowerCamelCase_		)




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      return torch.tensor(
          a_			,      dtype=torch.long			,      device=a_			,      )


UpperCAmelCase					:		str		=							1E-4





@require_torch
@require_sentencepiece
@require_tokenizers
class lowerCamelCase						(unittest.TestCase					):







  @slow
  @unittest.skip('''Model is not available.'''		)
  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        _snake_case		:	Tuple					     =     '''nvidia/megatron-bert-uncased-345m'''
        if "MYDIR" in os.environ:
              _snake_case		:	Tuple					     =     os.path.join(os.environ['''MYDIR''']     ,     lowerCamelCase_		)
        _snake_case		:	Dict					     =     MegatronBertModel.from_pretrained(lowerCamelCase_		)
        model.to(lowerCamelCase_		)
        model.half()
        _snake_case		:	List[str]					     =     _long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]		)
        with torch.no_grad():
              _snake_case		:	List[str]					     =     model(lowerCamelCase_		)[0]
        _snake_case		:	Any					     =     torch.Size((1, 9, 1_024)		)
        self.assertEqual(output.shape     ,     lowerCamelCase_		)

        _snake_case		:	Union[str, Any]					     =     [-0.6_040, -0.2_517, -0.1_025, 0.3_420, -0.6_758, -0.0_017, -0.1_089, -0.1_990, 0.5_728]
        for ii in range(3		):
              for jj in range(3		):
                    _snake_case		:	Optional[Any]					     =     output[0, ii, jj]
                    _snake_case		:	Optional[Any]					     =     expected[3 * ii + jj]
                    _snake_case		:	int					     =     '''ii={} jj={} a={} b={}'''.format(lowerCamelCase_     ,     lowerCamelCase_     ,     lowerCamelCase_     ,     lowerCamelCase_		)
                    self.assertTrue(math.isclose(lowerCamelCase_     ,     lowerCamelCase_     ,     rel_tol=lowerCamelCase_     ,     abs_tol=lowerCamelCase_		)     ,     msg=lowerCamelCase_		)









'''simple docstring'''

import math

from numpy import inf
from scipy.integrate import quad




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      if num <= 0:
            raise ValueError('''math domain error'''	)

      return quad(lowerCAmelCase_			,      0			,      lowerCAmelCase_			,      args=(lowerCAmelCase_)	)[0]




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      return math.pow(lowerCAmelCase_			,      z - 1	) * math.exp(-x	)


if __name__ == "__main__":
   from doctest import testmod

   testmod()


