






import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    MegatronBertForCausalLM,
				    MegatronBertForMaskedLM,
				    MegatronBertForMultipleChoice,
				    MegatronBertForNextSentencePrediction,
				    MegatronBertForPreTraining,
				    MegatronBertForQuestionAnswering,
				    MegatronBertForSequenceClassification,
				    MegatronBertForTokenClassification,
				    MegatronBertModel,
				)





class 						SCREAMING_SNAKE_CASE				:





							"""simple docstring"""

							def __init__(  self:   List[Any]		,     __A:   Any		,     __A:   List[str]=13		,     __A:   Tuple=7		,     __A:   Any=True		,     __A:   Union[str, Any]=True		,     __A:   str=True		,     __A:   List[str]=True		,     __A:   Any=99		,     __A:   str=64		,     __A:   str=32		,     __A:   Tuple=5		,     __A:   Dict=4		,     __A:   Optional[int]=37		,     __A:   List[str]="gelu"		,     __A:   Dict=0.1		,     __A:   Optional[int]=0.1		,     __A:   Optional[Any]=5_12		,     __A:   List[Any]=16		,     __A:   Any=2		,     __A:   List[str]=0.02		,     __A:   Union[str, Any]=3		,     __A:   Dict=4		,     __A:   Optional[Any]=None		,     )    ->				str:
												_A								= parent
												_A								= batch_size
												_A								= seq_length
												_A								= is_training
												_A								= use_input_mask
												_A								= use_token_type_ids
												_A								= use_labels
												_A								= vocab_size
												_A								= hidden_size
												_A								= embedding_size
												_A								= num_hidden_layers
												_A								= num_attention_heads
												_A								= intermediate_size
												_A								= hidden_act
												_A								= hidden_dropout_prob
												_A								= attention_probs_dropout_prob
												_A								= max_position_embeddings
												_A								= type_vocab_size
												_A								= type_sequence_label_size
												_A								= initializer_range
												_A								= num_labels
												_A								= num_choices
												_A								= scope

							def       __A (  self:   List[str]   )    ->				Optional[int]:
												_A								= ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size   )

												_A								= None
												if self.use_input_mask:
																	_A								= random_attention_mask([self.batch_size, self.seq_length]   )

												_A								= None
												if self.use_token_type_ids:
																	_A								= ids_tensor([self.batch_size, self.seq_length]		,     self.type_vocab_size   )

												_A								= None
												_A								= None
												_A								= None
												if self.use_labels:
																	_A								= ids_tensor([self.batch_size]		,     self.type_sequence_label_size   )
																	_A								= ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels   )
																	_A								= ids_tensor([self.batch_size]		,     self.num_choices   )

												_A								= self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

							def       __A (  self:   List[str]   )    ->				Optional[Any]:
												return MegatronBertConfig(
												    vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     intermediate_size=self.intermediate_size		,     embedding_size=self.embedding_size		,     hidden_act=self.hidden_act		,     hidden_dropout_prob=self.hidden_dropout_prob		,     attention_probs_dropout_prob=self.attention_probs_dropout_prob		,     max_position_embeddings=self.max_position_embeddings		,     type_vocab_size=self.type_vocab_size		,     is_decoder=__A		,     initializer_range=self.initializer_range		,     )

							def       __A (  self:   Optional[int]		,     __A:   int		,     __A:   Dict		,     __A:   Tuple		,     __A:   List[Any]		,     __A:   int		,     __A:   Optional[Any]		,     __A:   List[Any]   )    ->				Union[str, Any]:
												_A								= MegatronBertModel(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(__A		,     attention_mask=__A		,     token_type_ids=__A   )
												_A								= model(__A		,     token_type_ids=__A   )
												_A								= model(__A   )

												self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)   )
												self.parent.assertEqual(result.pooler_output.shape		,     (self.batch_size, self.hidden_size)   )

							def       __A (  self:   int		,     __A:   Dict		,     __A:   Optional[int]		,     __A:   Dict		,     __A:   Union[str, Any]		,     __A:   int		,     __A:   int		,     __A:   List[str]   )    ->				Optional[int]:
												_A								= MegatronBertForMaskedLM(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(__A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A   )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)   )

							def       __A (  self:   Any		,     __A:   Optional[Any]		,     __A:   Optional[int]		,     __A:   int		,     __A:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   Optional[int]		,     __A:   Optional[Any]   )    ->				int:
												_A								= MegatronBertForCausalLM(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(__A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A   )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)   )

							def       __A (  self:   str		,     __A:   Tuple		,     __A:   List[str]		,     __A:   Tuple		,     __A:   List[str]		,     __A:   Union[str, Any]		,     __A:   Tuple		,     __A:   List[Any]   )    ->				str:
												_A								= MegatronBertForNextSentencePrediction(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(
												    __A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A		,     )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, 2)   )

							def       __A (  self:   List[Any]		,     __A:   Optional[int]		,     __A:   Union[str, Any]		,     __A:   Any		,     __A:   Any		,     __A:   int		,     __A:   Optional[int]		,     __A:   Any   )    ->				List[str]:
												_A								= MegatronBertForPreTraining(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(
												    __A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A		,     next_sentence_label=__A		,     )
												self.parent.assertEqual(result.prediction_logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)   )
												self.parent.assertEqual(result.seq_relationship_logits.shape		,     (self.batch_size, 2)   )

							def       __A (  self:   Optional[Any]		,     __A:   List[str]		,     __A:   List[str]		,     __A:   Tuple		,     __A:   Dict		,     __A:   Optional[Any]		,     __A:   Dict		,     __A:   Tuple   )    ->				Union[str, Any]:
												_A								= MegatronBertForQuestionAnswering(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(
												    __A		,     attention_mask=__A		,     token_type_ids=__A		,     start_positions=__A		,     end_positions=__A		,     )
												self.parent.assertEqual(result.start_logits.shape		,     (self.batch_size, self.seq_length)   )
												self.parent.assertEqual(result.end_logits.shape		,     (self.batch_size, self.seq_length)   )

							def       __A (  self:   Union[str, Any]		,     __A:   Union[str, Any]		,     __A:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   List[str]		,     __A:   List[str]		,     __A:   Dict		,     __A:   Optional[Any]   )    ->				Union[str, Any]:
												_A								= self.num_labels
												_A								= MegatronBertForSequenceClassification(__A   )
												model.to(__A   )
												model.eval()
												_A								= model(__A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A   )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_labels)   )

							def       __A (  self:   Union[str, Any]		,     __A:   List[Any]		,     __A:   Tuple		,     __A:   Optional[int]		,     __A:   List[str]		,     __A:   Any		,     __A:   str		,     __A:   int   )    ->				Union[str, Any]:
												_A								= self.num_labels
												_A								= MegatronBertForTokenClassification(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= model(__A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A   )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.num_labels)   )

							def       __A (  self:   str		,     __A:   Optional[int]		,     __A:   List[str]		,     __A:   List[str]		,     __A:   Dict		,     __A:   int		,     __A:   str		,     __A:   str   )    ->				Dict:
												_A								= self.num_choices
												_A								= MegatronBertForMultipleChoice(config=__A   )
												model.to(__A   )
												model.eval()
												_A								= input_ids.unsqueeze(1   ).expand(-1		,     self.num_choices		,     -1   ).contiguous()
												_A								= token_type_ids.unsqueeze(1   ).expand(-1		,     self.num_choices		,     -1   ).contiguous()
												_A								= input_mask.unsqueeze(1   ).expand(-1		,     self.num_choices		,     -1   ).contiguous()
												_A								= model(
												    __A		,     attention_mask=__A		,     token_type_ids=__A		,     labels=__A		,     )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_choices)   )



							def       __A (  self:   List[Any]   )    ->				Any:
												_A								= self.prepare_config_and_inputs()
												(
												    (
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,(
												    _A
												)							,
												)								= config_and_inputs
												_A								= {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
												return config, inputs_dict





@require_torch
class 						SCREAMING_SNAKE_CASE				( snake_case  ,	snake_case  ,	unittest.TestCase					):





							"""simple docstring"""

							A_						  =					(
							    (
							        MegatronBertModel,
							        MegatronBertForMaskedLM,
							        MegatronBertForCausalLM,
							        MegatronBertForMultipleChoice,
							        MegatronBertForNextSentencePrediction,
							        MegatronBertForPreTraining,
							        MegatronBertForQuestionAnswering,
							        MegatronBertForSequenceClassification,
							        MegatronBertForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							A_						  =					(
							    {
							        "feature-extraction": MegatronBertModel,
							        "fill-mask": MegatronBertForMaskedLM,
							        "question-answering": MegatronBertForQuestionAnswering,
							        "text-classification": MegatronBertForSequenceClassification,
							        "text-generation": MegatronBertForCausalLM,
							        "token-classification": MegatronBertForTokenClassification,
							        "zero-shot": MegatronBertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							A_						  =					True
							# test_resize_embeddings = False
							A_						  =					False

							def       __A (  self:   List[Any]		,     __A:   Optional[Any]		,     __A:   Union[str, Any]		,     __A:   Any=False   )    ->				Dict:
												_A								= super()._prepare_for_class(__A		,     __A		,     return_labels=__A   )

												if return_labels:
																	if model_class in get_values(__A   ):
																						_A								= torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.seq_length)		,     dtype=torch.long		,     device=__A   )
																						_A								= torch.zeros(
																						    self.model_tester.batch_size		,     dtype=torch.long		,     device=__A   )
												return inputs_dict

							def       __A (  self:   List[str]   )    ->				str:
												_A								= MegatronBertModelTester(self   )
												_A								= ConfigTester(self		,     config_class=__A		,     hidden_size=37   )

							def       __A (  self:   int   )    ->				Any:
												self.config_tester.run_common_tests()

							def       __A (  self:   List[str]   )    ->				List[Any]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_model(*__A   )

							def       __A (  self:   int   )    ->				Union[str, Any]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__A   )

							def       __A (  self:   Tuple   )    ->				Optional[Any]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__A   )

							def       __A (  self:   List[Any]   )    ->				Optional[Any]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__A   )

							def       __A (  self:   Union[str, Any]   )    ->				Optional[int]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_pretraining(*__A   )

							def       __A (  self:   Optional[Any]   )    ->				List[str]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_question_answering(*__A   )

							def       __A (  self:   Union[str, Any]   )    ->				Optional[int]:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__A   )



							def       __A (  self:   int   )    ->				Dict:
												_A								= self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_megatron_bert_for_token_classification(*__A   )




def      __A					(     _lowercase ):

					'''simple docstring'''







					return torch.tensor(
					    _lowercase					,			dtype=torch.long					,			device=_lowercase					,			)


__A      =				1e-4





@require_torch
@require_sentencepiece
@require_tokenizers
class 						SCREAMING_SNAKE_CASE				( unittest.TestCase					):





							"""simple docstring"""

							@slow
							@unittest.skip('''Model is not available.'''   )
							def       __A (  self:   List[str]   )    ->				Dict:
												_A								= '''nvidia/megatron-bert-uncased-345m'''
												if "MYDIR" in os.environ:
																	_A								= os.path.join(os.environ['''MYDIR''']		,     __A   )
												_A								= MegatronBertModel.from_pretrained(__A   )
												model.to(__A   )
												model.half()
												_A								= _long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]   )
												with torch.no_grad():
																	_A								= model(__A   )[0]
												_A								= torch.Size((1, 9, 10_24)   )
												self.assertEqual(output.shape		,     __A   )

												_A								= [-0.6_040, -0.2_517, -0.1_025, 0.3_420, -0.6_758, -0.0_017, -0.1_089, -0.1_990, 0.5_728]
												for ii in range(3   ):
																	for jj in range(3   ):
																						_A								= output[0, ii, jj]
																						_A								= expected[3 * ii + jj]
																						_A								= '''ii={} jj={} a={} b={}'''.format(__A		,     __A		,     __A		,     __A   )
																						self.assertTrue(math.isclose(__A		,     __A		,     rel_tol=__A		,     abs_tol=__A   )		,     msg=__A   )











import itertools
import string
from collections.abc import Generator, Iterable




def      __A					(     _lowercase					,			_lowercase ):

					'''simple docstring'''







					_A								= iter(_lowercase )
					while True:
										_A								= tuple(itertools.islice(_lowercase					,			_lowercase ) )
										if not chunk:
															return
										yield chunk




def      __A					(     _lowercase ):

					'''simple docstring'''







					_A								= ''''''.join([c.upper() for c in dirty if c in string.ascii_letters] )
					_A								= ''''''

					if len(_lowercase ) < 2:
										return dirty

					for i in range(len(_lowercase ) - 1 ):
										clean += dirty[i]

										if dirty[i] == dirty[i + 1]:
															clean += "X"

					clean += dirty[-1]

					if len(_lowercase ) & 1:
										clean += "X"

					return clean




def      __A					(     _lowercase ):

					'''simple docstring'''







					_A								= '''ABCDEFGHIKLMNOPQRSTUVWXYZ'''
					# we're using a list instead of a '2d' array because it makes the math
					# for setting up the table and doing the actual encoding/decoding simpler
					_A								= []

					# copy key chars into the table if they are in `alphabet` ignoring duplicates
					for char in key.upper():
										if char not in table and char in alphabet:
															table.append(_lowercase )

    # fill the rest of the table in with the remaining alphabet chars
					for char in alphabet:
										if char not in table:
															table.append(_lowercase )

					return table




def      __A					(     _lowercase					,			_lowercase ):

					'''simple docstring'''







					_A								= generate_table(_lowercase )
					_A								= prepare_input(_lowercase )
					_A								= ''''''

					# https://en.wikipedia.org/wiki/Playfair_cipher#Description
					for chara, chara in chunker(_lowercase					,			2 ):
										_A							,_A								= divmod(table.index(_lowercase )					,			5 )
										_A							,_A								= divmod(table.index(_lowercase )					,			5 )

										if rowa == rowa:
															ciphertext += table[rowa * 5 + (cola + 1) % 5]
															ciphertext += table[rowa * 5 + (cola + 1) % 5]
										elif cola == cola:
															ciphertext += table[((rowa + 1) % 5) * 5 + cola]
															ciphertext += table[((rowa + 1) % 5) * 5 + cola]
										else:  # rectangle
															ciphertext += table[rowa * 5 + cola]
															ciphertext += table[rowa * 5 + cola]

					return ciphertext




def      __A					(     _lowercase					,			_lowercase ):

					'''simple docstring'''







					_A								= generate_table(_lowercase )
					_A								= ''''''

					# https://en.wikipedia.org/wiki/Playfair_cipher#Description
					for chara, chara in chunker(_lowercase					,			2 ):
										_A							,_A								= divmod(table.index(_lowercase )					,			5 )
										_A							,_A								= divmod(table.index(_lowercase )					,			5 )

										if rowa == rowa:
															plaintext += table[rowa * 5 + (cola - 1) % 5]
															plaintext += table[rowa * 5 + (cola - 1) % 5]
										elif cola == cola:
															plaintext += table[((rowa - 1) % 5) * 5 + cola]
															plaintext += table[((rowa - 1) % 5) * 5 + cola]
										else:  # rectangle
															plaintext += table[rowa * 5 + cola]
															plaintext += table[rowa * 5 + cola]

					return plaintext



