



'''simple docstring'''

import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow



class 		lowerCamelCase__      :




					'''simple docstring'''



					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	int     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	Dict					)  ->       List[str]:



						'''simple docstring'''





						return None






class 		lowerCamelCase__      :




					'''simple docstring'''



					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	Optional[Any]					)  ->       Union[str, Any]:



						'''simple docstring'''





						return None






class 		lowerCamelCase__      (						unittest.TestCase     ):




					'''simple docstring'''



					A_						    =						[
					    # (model_name, model_kwargs)
					    ("""bert-base-cased""", {}),
					    ("""gpt2""", {"""use_cache""": False}),  # We don't support exporting GPT2 past keys anymore
					]




					@require_tf
					@slow
					def 						__UpperCAmelCase     (  self				:	Optional[Any]					)  ->       List[Any]:



						'''simple docstring'''





						for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
							self._test_export(UpperCamelCase_     ,  'tf'     ,  12     ,  **UpperCamelCase_					)




					@require_torch
					@slow
					def 						__UpperCAmelCase     (  self				:	Any					)  ->       List[Any]:



						'''simple docstring'''





						for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
							self._test_export(UpperCamelCase_     ,  'pt'     ,  12     ,  **UpperCamelCase_					)




					@require_torch
					@slow
					def 						__UpperCAmelCase     (  self				:	Optional[Any]					)  ->       Any:



						'''simple docstring'''





						from transformers import BertModel

						_lowercase					:       Optional[Any]       =		['[UNK]', '[SEP]', '[CLS]', '[PAD]', '[MASK]', 'some', 'other', 'words']
						with NamedTemporaryFile(mode='w+t'					) as vocab_file:
							vocab_file.write('\n'.join(UpperCamelCase_					)					)
							vocab_file.flush()
							_lowercase					:       Optional[Any]       =		BertTokenizerFast(vocab_file.name					)

						with TemporaryDirectory() as bert_save_dir:
							_lowercase					:       Any       =		BertModel(BertConfig(vocab_size=len(UpperCamelCase_					)					)					)
							model.save_pretrained(UpperCamelCase_					)
							self._test_export(UpperCamelCase_     ,  'pt'     ,  12     ,  UpperCamelCase_					)




					@require_tf
					@slow
					def 						__UpperCAmelCase     (  self				:	int					)  ->       str:



						'''simple docstring'''





						for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
							_lowercase					:       Dict       =		self._test_export(UpperCamelCase_     ,  'tf'     ,  12     ,  **UpperCamelCase_					)
							_lowercase					:       str       =		quantize(Path(UpperCamelCase_					)					)

							# Ensure the actual quantized model is not bigger than the original one
							if quantized_path.stat().st_size >= Path(UpperCamelCase_					).stat().st_size:
								self.fail('Quantized model is bigger than initial ONNX model'					)




					@require_torch
					@slow
					def 						__UpperCAmelCase     (  self				:	Dict					)  ->       int:



						'''simple docstring'''





						for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
							_lowercase					:       int       =		self._test_export(UpperCamelCase_     ,  'pt'     ,  12     ,  **UpperCamelCase_					)
							_lowercase					:       List[Any]       =		quantize(UpperCamelCase_					)

							# Ensure the actual quantized model is not bigger than the original one
							if quantized_path.stat().st_size >= Path(UpperCamelCase_					).stat().st_size:
								self.fail('Quantized model is bigger than initial ONNX model'					)




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	Dict     ,  UpperCamelCase_				:	Any     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Any=None     ,  **UpperCamelCase_				:	Optional[Any]					)  ->       List[Any]:



						'''simple docstring'''





						try:
							# Compute path
							with TemporaryDirectory() as tempdir:
								_lowercase					:       int       =		Path(UpperCamelCase_					).joinpath('model.onnx'					)

							# Remove folder if exists
							if path.parent.exists():
								path.parent.rmdir()

							# Export
							convert(UpperCamelCase_     ,  UpperCamelCase_     ,  UpperCamelCase_     ,  UpperCamelCase_     ,  UpperCamelCase_     ,  **UpperCamelCase_					)

							return path
						except Exception as e:
							self.fail(UpperCamelCase_					)




					@require_torch
					@require_tokenizers
					@slow
					def 						__UpperCAmelCase     (  self				:	Any					)  ->       Tuple:



						'''simple docstring'''





						from transformers import BertModel

						_lowercase					:       List[str]       =		BertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'					)					)
						_lowercase					:       List[str]       =		BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'					)
						self._test_infer_dynamic_axis(UpperCamelCase_     ,  UpperCamelCase_     ,  'pt'					)




					@require_tf
					@require_tokenizers
					@slow
					def 						__UpperCAmelCase     (  self				:	str					)  ->       List[str]:



						'''simple docstring'''





						from transformers import TFBertModel

						_lowercase					:       Any       =		TFBertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'					)					)
						_lowercase					:       Union[str, Any]       =		BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'					)
						self._test_infer_dynamic_axis(UpperCamelCase_     ,  UpperCamelCase_     ,  'tf'					)




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Any					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       str       =		FeatureExtractionPipeline(UpperCamelCase_     ,  UpperCamelCase_					)

						_lowercase					:       str       =		['input_ids', 'token_type_ids', 'attention_mask', 'output_0', 'output_1']
						_lowercase  ,    _lowercase  ,    _lowercase  ,    _lowercase					:       Dict       =		infer_shapes(UpperCamelCase_     ,  UpperCamelCase_					)

						# Assert all variables are present
						self.assertEqual(len(UpperCamelCase_					)     ,  len(UpperCamelCase_					)					)
						self.assertTrue(all(var_name in shapes for var_name in variable_names					)					)
						self.assertSequenceEqual(variable_names[:3]     ,  UpperCamelCase_					)
						self.assertSequenceEqual(variable_names[3:]     ,  UpperCamelCase_					)

						# Assert inputs are {0: batch, 1: sequence}
						for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
							self.assertDictEqual(shapes[var_name]     ,  {0: 'batch', 1: 'sequence'}					)

						# Assert outputs are {0: batch, 1: sequence} and {0: batch}
						self.assertDictEqual(shapes['output_0']     ,  {0: 'batch', 1: 'sequence'}					)
						self.assertDictEqual(shapes['output_1']     ,  {0: 'batch'}					)




					def 						__UpperCAmelCase     (  self				:	int					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Tuple       =		['input_ids', 'attention_mask', 'token_type_ids']
						_lowercase					:       str       =		{'input_ids': [1, 2, 3, 4], 'attention_mask': [0, 0, 0, 0], 'token_type_ids': [1, 1, 1, 1]}
						_lowercase  ,    _lowercase					:       str       =		ensure_valid_input(FuncContiguousArgs()     ,  UpperCamelCase_     ,  UpperCamelCase_					)

						# Should have exactly the same number of args (all are valid)
						self.assertEqual(len(UpperCamelCase_					)     ,  3					)

						# Should have exactly the same input names
						self.assertEqual(set(UpperCamelCase_					)     ,  set(UpperCamelCase_					)					)

						# Parameter should be reordered according to their respective place in the function:
						# (input_ids, token_type_ids, attention_mask)
						self.assertEqual(UpperCamelCase_     ,  (tokens['input_ids'], tokens['token_type_ids'], tokens['attention_mask'])					)

						# Generated args are interleaved with another args (for instance parameter "past" in GPT2)
						_lowercase  ,    _lowercase					:       str       =		ensure_valid_input(FuncNonContiguousArgs()     ,  UpperCamelCase_     ,  UpperCamelCase_					)

						# Should have exactly the one arg (all before the one not provided "some_other_args")
						self.assertEqual(len(UpperCamelCase_					)     ,  1					)
						self.assertEqual(len(UpperCamelCase_					)     ,  1					)

						# Should have only "input_ids"
						self.assertEqual(inputs_args[0]     ,  tokens['input_ids']					)
						self.assertEqual(ordered_input_names[0]     ,  'input_ids'					)




					def 						__UpperCAmelCase     (  self				:	Optional[Any]					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		generate_identified_filename(Path('/home/something/my_fake_model.onnx'					)     ,  '-test'					)
						self.assertEqual('/home/something/my_fake_model-test.onnx'     ,  generated.as_posix()					)





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A				:    Dict     						=logging.get_logger(__name__)

_A				:    Dict     						={
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						"""megatron-bert"""




					def __init__(  self				:	int     ,  UpperCamelCase_				:	int=2_9056     ,  UpperCamelCase_				:	Optional[int]=1024     ,  UpperCamelCase_				:	Optional[Any]=24     ,  UpperCamelCase_				:	List[Any]=16     ,  UpperCamelCase_				:	Optional[int]=4096     ,  UpperCamelCase_				:	Optional[Any]="gelu"     ,  UpperCamelCase_				:	List[Any]=0.1     ,  UpperCamelCase_				:	Union[str, Any]=0.1     ,  UpperCamelCase_				:	int=512     ,  UpperCamelCase_				:	Dict=2     ,  UpperCamelCase_				:	Union[str, Any]=0.02     ,  UpperCamelCase_				:	Any=1E-12     ,  UpperCamelCase_				:	Tuple=0     ,  UpperCamelCase_				:	Optional[int]="absolute"     ,  UpperCamelCase_				:	Optional[Any]=True     ,  **UpperCamelCase_				:	Any     ,  )  ->       List[Any]:



						'''simple docstring'''





						super().__init__(pad_token_id=UpperCamelCase_     ,  **UpperCamelCase_					)

						_lowercase					:       Dict       =		vocab_size
						_lowercase					:       Any       =		hidden_size
						_lowercase					:       Union[str, Any]       =		num_hidden_layers
						_lowercase					:       Dict       =		num_attention_heads
						_lowercase					:       Dict       =		hidden_act
						_lowercase					:       Optional[Any]       =		intermediate_size
						_lowercase					:       Optional[int]       =		hidden_dropout_prob
						_lowercase					:       Optional[Any]       =		attention_probs_dropout_prob
						_lowercase					:       Any       =		max_position_embeddings
						_lowercase					:       str       =		type_vocab_size
						_lowercase					:       Optional[Any]       =		initializer_range
						_lowercase					:       List[str]       =		layer_norm_eps
						_lowercase					:       List[Any]       =		position_embedding_type
						_lowercase					:       Optional[Any]       =		use_cache
