

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
					import torch

					from transformers import XLMRobertaModel

@require_sentencepiece
@require_tokenizers
@require_torch
class 			a  (	unittest.TestCase				):


						@slow
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    XLMRobertaModel.from_pretrained("xlm-roberta-base"	)
										_UpperCAmelCase       :		Dict 		=    torch.tensor([[0, 581, 10269, 83, 99942, 136, 60742, 23, 70, 80583, 18276, 2]]	)
										# The dog is cute and lives in the garden house

										_UpperCAmelCase       :		str 		=    torch.Size((1, 12, 768)	)  # batch_size, sequence_length, embedding_vector_dim
										_UpperCAmelCase       :		Any 		=    torch.tensor(
										    [[-0.01_01, 0.12_18, -0.08_03, 0.08_01, 0.13_27, 0.07_76, -0.12_15, 0.23_83, 0.33_38, 0.31_06, 0.03_00, 0.02_52]]	)
										#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
										#  xlmr.eval()
										#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
										with torch.no_grad():
														_UpperCAmelCase       :		Union[str, Any] 		=    model(lowercase__	)["""last_hidden_state"""].detach()
										self.assertEqual(output.shape						,					lowercase__	)
										# compare the actual values for a slice of last dim
										self.assertTrue(torch.allclose(output[:, :, -1]						,					lowercase__						,					atol=1e-3	)	)





						@slow
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    XLMRobertaModel.from_pretrained("xlm-roberta-large"	)
										_UpperCAmelCase       :		Tuple 		=    torch.tensor([[0, 581, 10269, 83, 99942, 136, 60742, 23, 70, 80583, 18276, 2]]	)
										# The dog is cute and lives in the garden house

										_UpperCAmelCase       :		List[Any] 		=    torch.Size((1, 12, 1024)	)  # batch_size, sequence_length, embedding_vector_dim
										_UpperCAmelCase       :		List[str] 		=    torch.tensor(
										    [[-0.06_99, -0.03_18, 0.07_05, -0.12_41, 0.09_99, -0.05_20, 0.10_04, -0.18_38, -0.47_04, 0.14_37, 0.08_21, 0.01_26]]	)
										#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
										#  xlmr.eval()
										#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
										with torch.no_grad():
														_UpperCAmelCase       :		Optional[int] 		=    model(lowercase__	)["""last_hidden_state"""].detach()
										self.assertEqual(output.shape						,					lowercase__	)
										# compare the actual values for a slice of last dim
										self.assertTrue(torch.allclose(output[:, :, -1]						,					lowercase__						,					atol=1e-3	)	)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_        =      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_        =      {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}

class 				a  (	UpperCAmelCase				):
						_lowercase    						=			"megatron-bert"


						def __init__(       self						,					A_=29056						,					A_=1024						,					A_=24						,					A_=16						,					A_=4096						,					A_="gelu"						,					A_=0.1						,					A_=0.1						,					A_=512						,					A_=2						,					A_=0.02						,					A_=1e-12						,					A_=0						,					A_="absolute"						,					A_=True						,					**A_						,					):







										'''simple docstring'''


										super().__init__(pad_token_id=A_						,					**A_	)

										_UpperCAmelCase       :		Optional[Any] 		=    vocab_size
										_UpperCAmelCase       :		List[str] 		=    hidden_size
										_UpperCAmelCase       :		List[str] 		=    num_hidden_layers
										_UpperCAmelCase       :		Optional[Any] 		=    num_attention_heads
										_UpperCAmelCase       :		Any 		=    hidden_act
										_UpperCAmelCase       :		Optional[Any] 		=    intermediate_size
										_UpperCAmelCase       :		int 		=    hidden_dropout_prob
										_UpperCAmelCase       :		Optional[Any] 		=    attention_probs_dropout_prob
										_UpperCAmelCase       :		Any 		=    max_position_embeddings
										_UpperCAmelCase       :		str 		=    type_vocab_size
										_UpperCAmelCase       :		Optional[int] 		=    initializer_range
										_UpperCAmelCase       :		Dict 		=    layer_norm_eps
										_UpperCAmelCase       :		Tuple 		=    position_embedding_type
										_UpperCAmelCase       :		Tuple 		=    use_cache


