




import json
import logging
import os
import sys
from time import time
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, require_torch_tpu


logging.basicConfig(level=logging.DEBUG)

__A						:							Dict							=  logging.getLogger()
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			SCREAMING_SNAKE_CASE           =					{}
			SCREAMING_SNAKE_CASE           =					os.path.join(SCREAMING_SNAKE_CASE					,	"""all_results.json"""   )
			if os.path.exists(SCREAMING_SNAKE_CASE   ):
						with open(SCREAMING_SNAKE_CASE					,	"""r"""   ) as f:
									SCREAMING_SNAKE_CASE           =					json.load(SCREAMING_SNAKE_CASE   )
			else:
						raise ValueError(f"""can't find {path}"""   )
			return results


__A						:							List[Any]							=  logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)




@require_torch_tpu
class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							def    _UpperCAmelCase     (					self       :	str			)			->					Union[str, Any]:
										import xla_spawn

										SCREAMING_SNAKE_CASE           =					self.get_auto_remove_tmp_dir()
										SCREAMING_SNAKE_CASE           =					f"""
            ./examples/pytorch/text-classification/run_glue.py
            --num_cores=8
            ./examples/pytorch/text-classification/run_glue.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --do_train
            --do_eval
            --debug tpu_metrics_debug
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --max_steps=10
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            """.split()

										with patch.object(a       ,     """argv"""       ,     a			):
													SCREAMING_SNAKE_CASE           =					time()
													xla_spawn.main()
													SCREAMING_SNAKE_CASE           =					time()

													SCREAMING_SNAKE_CASE           =					get_results(a			)
													self.assertGreaterEqual(result["""eval_accuracy"""]       ,     0.75			)

													# Assert that the script takes less than 500 seconds to make sure it doesn't hang.
													self.assertLess(end - start       ,     500			)







							def    _UpperCAmelCase     (					self       :	List[str]			)			->					List[str]:
										import xla_spawn

										SCREAMING_SNAKE_CASE           =					"""
            ./tests/test_trainer_tpu.py
            --num_cores=8
            ./tests/test_trainer_tpu.py
            """.split()
										with patch.object(a       ,     """argv"""       ,     a			):
													xla_spawn.main()












from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A						:							Any							=  logging.get_logger(__name__)

__A						:							int							=  {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     '''megatron-bert'''





							def __init__(					self       :	str       ,     a       :	Union[str, Any]=29_056       ,     a       :	Optional[int]=1_024       ,     a       :	str=24       ,     a       :	str=16       ,     a       :	str=4_096       ,     a       :	Any="gelu"       ,     a       :	Any=0.1       ,     a       :	Optional[int]=0.1       ,     a       :	Optional[int]=512       ,     a       :	Optional[Any]=2       ,     a       :	List[str]=0.02       ,     a       :	List[Any]=1E-1_2       ,     a       :	Any=0       ,     a       :	List[Any]="absolute"       ,     a       :	List[str]=True       ,     **a       :	Optional[Any]       ,     )			->					List[str]:
										super().__init__(pad_token_id=a       ,     **a			)

										SCREAMING_SNAKE_CASE           =					vocab_size
										SCREAMING_SNAKE_CASE           =					hidden_size
										SCREAMING_SNAKE_CASE           =					num_hidden_layers
										SCREAMING_SNAKE_CASE           =					num_attention_heads
										SCREAMING_SNAKE_CASE           =					hidden_act
										SCREAMING_SNAKE_CASE           =					intermediate_size
										SCREAMING_SNAKE_CASE           =					hidden_dropout_prob
										SCREAMING_SNAKE_CASE           =					attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE           =					max_position_embeddings
										SCREAMING_SNAKE_CASE           =					type_vocab_size
										SCREAMING_SNAKE_CASE           =					initializer_range
										SCREAMING_SNAKE_CASE           =					layer_norm_eps
										SCREAMING_SNAKE_CASE           =					position_embedding_type
										SCREAMING_SNAKE_CASE           =					use_cache






