import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MODEL_FOR_PRETRAINING_MAPPING,
						    MegatronBertForCausalLM,
						    MegatronBertForMaskedLM,
						    MegatronBertForMultipleChoice,
						    MegatronBertForNextSentencePrediction,
						    MegatronBertForPreTraining,
						    MegatronBertForQuestionAnswering,
						    MegatronBertForSequenceClassification,
						    MegatronBertForTokenClassification,
						    MegatronBertModel,
						)






class      __lowercase					:

						def __init__(	self				:      Dict							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      List[Any]=13							,							__lowerCamelCase				:      int=7							,							__lowerCamelCase				:      Tuple=True							,							__lowerCamelCase				:      Union[str, Any]=True							,							__lowerCamelCase				:      int=True							,							__lowerCamelCase				:      Union[str, Any]=True							,							__lowerCamelCase				:      str=99							,							__lowerCamelCase				:      Optional[Any]=64							,							__lowerCamelCase				:      Optional[Any]=32							,							__lowerCamelCase				:      Union[str, Any]=5							,							__lowerCamelCase				:      Optional[int]=4							,							__lowerCamelCase				:      List[Any]=37							,							__lowerCamelCase				:      str="gelu"							,							__lowerCamelCase				:      Any=0.1							,							__lowerCamelCase				:      Dict=0.1							,							__lowerCamelCase				:      Tuple=5_12							,							__lowerCamelCase				:      Optional[Any]=16							,							__lowerCamelCase				:      Optional[Any]=2							,							__lowerCamelCase				:      str=0.02							,							__lowerCamelCase				:      List[str]=3							,							__lowerCamelCase				:      Optional[Any]=4							,							__lowerCamelCase				:      Tuple=None							,							)							->		Optional[int]:


									'''simple docstring'''

									lowercase							=		parent
									lowercase							=		batch_size
									lowercase							=		seq_length
									lowercase							=		is_training
									lowercase							=		use_input_mask
									lowercase							=		use_token_type_ids
									lowercase							=		use_labels
									lowercase							=		vocab_size
									lowercase							=		hidden_size
									lowercase							=		embedding_size
									lowercase							=		num_hidden_layers
									lowercase							=		num_attention_heads
									lowercase							=		intermediate_size
									lowercase							=		hidden_act
									lowercase							=		hidden_dropout_prob
									lowercase							=		attention_probs_dropout_prob
									lowercase							=		max_position_embeddings
									lowercase							=		type_vocab_size
									lowercase							=		type_sequence_label_size
									lowercase							=		initializer_range
									lowercase							=		num_labels
									lowercase							=		num_choices
									lowercase							=		scope

						def        __a		(	self				:      Tuple   )							->		List[str]:


									'''simple docstring'''

									lowercase							=		ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size   )

									lowercase							=		None
									if self.use_input_mask:
												lowercase							=		random_attention_mask([self.batch_size, self.seq_length]   )

									lowercase							=		None
									if self.use_token_type_ids:
												lowercase							=		ids_tensor([self.batch_size, self.seq_length]							,							self.type_vocab_size   )

									lowercase							=		None
									lowercase							=		None
									lowercase							=		None
									if self.use_labels:
												lowercase							=		ids_tensor([self.batch_size]							,							self.type_sequence_label_size   )
												lowercase							=		ids_tensor([self.batch_size, self.seq_length]							,							self.num_labels   )
												lowercase							=		ids_tensor([self.batch_size]							,							self.num_choices   )

									lowercase							=		self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

						def        __a		(	self				:      str   )							->		Union[str, Any]:


									'''simple docstring'''

									return MegatronBertConfig(
									    vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							embedding_size=self.embedding_size							,							hidden_act=self.hidden_act							,							hidden_dropout_prob=self.hidden_dropout_prob							,							attention_probs_dropout_prob=self.attention_probs_dropout_prob							,							max_position_embeddings=self.max_position_embeddings							,							type_vocab_size=self.type_vocab_size							,							is_decoder=lowerCamelCase_							,							initializer_range=self.initializer_range							,							)

						def        __a		(	self				:      Tuple							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      int							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      int   )							->		Tuple:


									'''simple docstring'''

									lowercase							=		MegatronBertModel(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_   )
									lowercase							=		model(lowerCamelCase_							,							token_type_ids=lowerCamelCase_   )
									lowercase							=		model(lowerCamelCase_   )

									self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size)   )
									self.parent.assertEqual(result.pooler_output.shape							,							(self.batch_size, self.hidden_size)   )

						def        __a		(	self				:      str							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      Optional[Any]							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      Dict							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      int							,							__lowerCamelCase				:      Union[str, Any]   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		MegatronBertForMaskedLM(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_   )
									self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)   )

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      Dict							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Any   )							->		Any:


									'''simple docstring'''

									lowercase							=		MegatronBertForCausalLM(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_   )
									self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)   )

						def        __a		(	self				:      List[str]							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      int							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      List[str]   )							->		List[str]:


									'''simple docstring'''

									lowercase							=		MegatronBertForNextSentencePrediction(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(
									    lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_							,							)
									self.parent.assertEqual(result.logits.shape							,							(self.batch_size, 2)   )

						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      Optional[Any]							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      List[Any]   )							->		Optional[int]:


									'''simple docstring'''

									lowercase							=		MegatronBertForPreTraining(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(
									    lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_							,							next_sentence_label=lowerCamelCase_							,							)
									self.parent.assertEqual(result.prediction_logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)   )
									self.parent.assertEqual(result.seq_relationship_logits.shape							,							(self.batch_size, 2)   )

						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      str   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		MegatronBertForQuestionAnswering(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(
									    lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							start_positions=lowerCamelCase_							,							end_positions=lowerCamelCase_							,							)
									self.parent.assertEqual(result.start_logits.shape							,							(self.batch_size, self.seq_length)   )
									self.parent.assertEqual(result.end_logits.shape							,							(self.batch_size, self.seq_length)   )

						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      Dict							,							__lowerCamelCase				:      int							,							__lowerCamelCase				:      int							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Dict   )							->		Optional[int]:


									'''simple docstring'''

									lowercase							=		self.num_labels
									lowercase							=		MegatronBertForSequenceClassification(lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_   )
									self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_labels)   )

						def        __a		(	self				:      int							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Dict							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Optional[int]							,							__lowerCamelCase				:      Union[str, Any]   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		self.num_labels
									lowercase							=		MegatronBertForTokenClassification(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		model(lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_   )
									self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.num_labels)   )

						def        __a		(	self				:      Any							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Optional[Any]							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      Dict							,							__lowerCamelCase				:      Optional[int]   )							->		List[str]:


									'''simple docstring'''

									lowercase							=		self.num_choices
									lowercase							=		MegatronBertForMultipleChoice(config=lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.eval()
									lowercase							=		input_ids.unsqueeze(1   ).expand(-1							,							self.num_choices							,							-1   ).contiguous()
									lowercase							=		token_type_ids.unsqueeze(1   ).expand(-1							,							self.num_choices							,							-1   ).contiguous()
									lowercase							=		input_mask.unsqueeze(1   ).expand(-1							,							self.num_choices							,							-1   ).contiguous()
									lowercase							=		model(
									    lowerCamelCase_							,							attention_mask=lowerCamelCase_							,							token_type_ids=lowerCamelCase_							,							labels=lowerCamelCase_							,							)
									self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_choices)   )



						def        __a		(	self				:      str   )							->		Any:


									'''simple docstring'''

									lowercase							=		self.prepare_config_and_inputs()
									(
									    lowercase					
									)							=		config_and_inputs
									lowercase							=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
									return config, inputs_dict






@require_torch
class      __lowercase					(    _UpperCAmelCase       ,     _UpperCAmelCase       ,     unittest.TestCase    ):
						lowercase					 =     (
						    (
						        MegatronBertModel,
						        MegatronBertForMaskedLM,
						        MegatronBertForCausalLM,
						        MegatronBertForMultipleChoice,
						        MegatronBertForNextSentencePrediction,
						        MegatronBertForPreTraining,
						        MegatronBertForQuestionAnswering,
						        MegatronBertForSequenceClassification,
						        MegatronBertForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						lowercase					 =     (
						    {
						        'feature-extraction': MegatronBertModel,
						        'fill-mask': MegatronBertForMaskedLM,
						        'question-answering': MegatronBertForQuestionAnswering,
						        'text-classification': MegatronBertForSequenceClassification,
						        'text-generation': MegatronBertForCausalLM,
						        'token-classification': MegatronBertForTokenClassification,
						        'zero-shot': MegatronBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						lowercase					 =     True
						# test_resize_embeddings = False
						lowercase					 =     False

						def        __a		(	self				:      Any							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      Optional[int]=False   )							->		Optional[int]:


									'''simple docstring'''

									lowercase							=		super()._prepare_for_class(lowerCamelCase_							,							lowerCamelCase_							,							return_labels=lowerCamelCase_   )

									if return_labels:
												if model_class in get_values(lowerCamelCase_   ):
															lowercase							=		torch.zeros(
															    (self.model_tester.batch_size, self.model_tester.seq_length)							,							dtype=torch.long							,							device=lowerCamelCase_   )
															lowercase							=		torch.zeros(
															    self.model_tester.batch_size							,							dtype=torch.long							,							device=lowerCamelCase_   )
									return inputs_dict

						def        __a		(	self				:      Dict   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		MegatronBertModelTester(self   )
									lowercase							=		ConfigTester(self							,							config_class=lowerCamelCase_							,							hidden_size=37   )

						def        __a		(	self				:      Tuple   )							->		Any:


									'''simple docstring'''

									self.config_tester.run_common_tests()

						def        __a		(	self				:      Any   )							->		Any:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_model(*lowerCamelCase_   )

						def        __a		(	self				:      Optional[int]   )							->		str:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_masked_lm(*lowerCamelCase_   )

						def        __a		(	self				:      Tuple   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*lowerCamelCase_   )

						def        __a		(	self				:      List[Any]   )							->		Dict:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*lowerCamelCase_   )

						def        __a		(	self				:      Optional[Any]   )							->		Optional[Any]:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_pretraining(*lowerCamelCase_   )

						def        __a		(	self				:      Union[str, Any]   )							->		Dict:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_question_answering(*lowerCamelCase_   )

						def        __a		(	self				:      Optional[int]   )							->		Any:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*lowerCamelCase_   )



						def        __a		(	self				:      Tuple   )							->		Optional[int]:


									'''simple docstring'''

									lowercase							=		self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_token_classification(*lowerCamelCase_   )






def        __UpperCAmelCase    (       UpperCAmelCase       )->       Dict:



			"""simple docstring"""







			return torch.tensor(
			    a_,		dtype=torch.long,		device=a_,		)


A_      =      1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class      __lowercase					(    unittest.TestCase    ):

						@slow
						@unittest.skip('''Model is not available.'''   )
						def        __a		(	self				:      Any   )							->		Optional[Any]:


									'''simple docstring'''

									lowercase							=		'''nvidia/megatron-bert-uncased-345m'''
									if "MYDIR" in os.environ:
												lowercase							=		os.path.join(os.environ['''MYDIR''']							,							lowerCamelCase_   )
									lowercase							=		MegatronBertModel.from_pretrained(lowerCamelCase_   )
									model.to(lowerCamelCase_   )
									model.half()
									lowercase							=		_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]   )
									with torch.no_grad():
												lowercase							=		model(lowerCamelCase_   )[0]
									lowercase							=		torch.Size((1, 9, 10_24)   )
									self.assertEqual(output.shape							,							lowerCamelCase_   )

									lowercase							=		[-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
									for ii in range(3   ):
												for jj in range(3   ):
															lowercase							=		output[0, ii, jj]
															lowercase							=		expected[3 * ii + jj]
															lowercase							=		'''ii={} jj={} a={} b={}'''.format(lowerCamelCase_							,							lowerCamelCase_							,							lowerCamelCase_							,							lowerCamelCase_   )
															self.assertTrue(math.isclose(lowerCamelCase_							,							lowerCamelCase_							,							rel_tol=lowerCamelCase_							,							abs_tol=lowerCamelCase_   )							,							msg=lowerCamelCase_   )






import os
import random
import sys

from . import cryptomath_module as cryptomath
from . import rabin_miller

A_      =      3






def        __UpperCAmelCase    (       UpperCAmelCase       )->       int:



			"""simple docstring"""







			print('''Generating primitive root of p'''       )
			while True:
						lowercase							=		random.randrange(3,		UpperCAmelCase       )
						if pow(UpperCAmelCase,		2,		UpperCAmelCase       ) == 1:
									continue
						if pow(UpperCAmelCase,		UpperCAmelCase,		UpperCAmelCase       ) == 1:
									continue
						return g






def        __UpperCAmelCase    (       UpperCAmelCase       )->       tuple[tuple[int, int, int, int], tuple[int, int]]:



			"""simple docstring"""







			print('''Generating prime p...'''       )
			lowercase							=		rabin_miller.generate_large_prime(UpperCAmelCase       )  # select large prime number.
			lowercase							=		primitive_root(UpperCAmelCase       )  # one primitive root on modulo p.
			lowercase							=		random.randrange(3,		UpperCAmelCase       )  # private_key -> have to be greater than 2 for safety.
			lowercase							=		cryptomath.find_mod_inverse(pow(UpperCAmelCase,		UpperCAmelCase,		UpperCAmelCase       ),		UpperCAmelCase       )

			lowercase							=		(key_size, e_a, e_a, p)
			lowercase							=		(key_size, d)

			return public_key, private_key






def        __UpperCAmelCase    (       UpperCAmelCase,		UpperCAmelCase       )->       None:



			"""simple docstring"""







			if os.path.exists(f'{name}_pubkey.txt'       ) or os.path.exists(f'{name}_privkey.txt'       ):
						print('''\nWARNING:'''       )
						print(
						    f'"{name}_pubkey.txt" or "{name}_privkey.txt" already exists. \n'
						    '''Use a different name or delete these files and re-run this program.'''       )
						sys.exit()

			lowercase		,lowercase							=		generate_key(UpperCAmelCase       )
			print(f'\nWriting public key to file {name}_pubkey.txt...'       )
			with open(f'{name}_pubkey.txt',		'''w'''       ) as fo:
						fo.write(f'{public_key[0]},{public_key[1]},{public_key[2]},{public_key[3]}'       )

			print(f'Writing private key to file {name}_privkey.txt...'       )
			with open(f'{name}_privkey.txt',		'''w'''       ) as fo:
						fo.write(f'{private_key[0]},{private_key[1]}'       )






def        __UpperCAmelCase    (       )->       None:



			"""simple docstring"""







			print('''Making key files...'''       )
			make_key_files('''elgamal''',		2048       )
			print('''Key files generation successful'''       )


if __name__ == "__main__":
						main()





