






from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFRoFormerForCausalLM,
				    TFRoFormerForMaskedLM,
				    TFRoFormerForMultipleChoice,
				    TFRoFormerForQuestionAnswering,
				    TFRoFormerForSequenceClassification,
				    TFRoFormerForTokenClassification,
				    TFRoFormerModel,
				)
				from transformers.models.roformer.modeling_tf_roformer import (
				    TFRoFormerSelfAttention,
				    TFRoFormerSinusoidalPositionalEmbedding,
				)






class 		_snake_case  :






			def __init__(       self		,					a		,					a=13		,					a=7		,					a=True		,					a=True		,					a=True		,					a=True		,					a=99		,					a=32		,					a=2		,					a=4		,					a=37		,					a="gelu"		,					a=0.1		,					a=0.1		,					a=512		,					a=16		,					a=2		,					a=0.02		,					a=3		,					a=4		,					a=None		,					)      ->       int:
						SCREAMING_SNAKE_CASE						= parent
						SCREAMING_SNAKE_CASE						= 13
						SCREAMING_SNAKE_CASE						= 7
						SCREAMING_SNAKE_CASE						= True
						SCREAMING_SNAKE_CASE						= True
						SCREAMING_SNAKE_CASE						= True
						SCREAMING_SNAKE_CASE						= True
						SCREAMING_SNAKE_CASE						= 99
						SCREAMING_SNAKE_CASE						= 32
						SCREAMING_SNAKE_CASE						= 2
						SCREAMING_SNAKE_CASE						= 4
						SCREAMING_SNAKE_CASE						= 37
						SCREAMING_SNAKE_CASE						= 'gelu'
						SCREAMING_SNAKE_CASE						= 0.1
						SCREAMING_SNAKE_CASE						= 0.1
						SCREAMING_SNAKE_CASE						= 512
						SCREAMING_SNAKE_CASE						= 16
						SCREAMING_SNAKE_CASE						= 2
						SCREAMING_SNAKE_CASE						= 0.02
						SCREAMING_SNAKE_CASE						= 3
						SCREAMING_SNAKE_CASE						= 4
						SCREAMING_SNAKE_CASE						= None






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Tuple:
						SCREAMING_SNAKE_CASE						= ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size)

						SCREAMING_SNAKE_CASE						= None
						if self.use_input_mask:
									SCREAMING_SNAKE_CASE						= random_attention_mask([self.batch_size, self.seq_length])

						SCREAMING_SNAKE_CASE						= None
						if self.use_token_type_ids:
									SCREAMING_SNAKE_CASE						= ids_tensor([self.batch_size, self.seq_length]		,					self.type_vocab_size)

						SCREAMING_SNAKE_CASE						= None
						SCREAMING_SNAKE_CASE						= None
						SCREAMING_SNAKE_CASE						= None
						if self.use_labels:
									SCREAMING_SNAKE_CASE						= ids_tensor([self.batch_size]		,					self.type_sequence_label_size)
									SCREAMING_SNAKE_CASE						= ids_tensor([self.batch_size, self.seq_length]		,					self.num_labels)
									SCREAMING_SNAKE_CASE						= ids_tensor([self.batch_size]		,					self.num_choices)

						SCREAMING_SNAKE_CASE						= RoFormerConfig(
						    vocab_size=self.vocab_size		,					hidden_size=self.hidden_size		,					num_hidden_layers=self.num_hidden_layers		,					num_attention_heads=self.num_attention_heads		,					intermediate_size=self.intermediate_size		,					hidden_act=self.hidden_act		,					hidden_dropout_prob=self.hidden_dropout_prob		,					attention_probs_dropout_prob=self.attention_probs_dropout_prob		,					max_position_embeddings=self.max_position_embeddings		,					type_vocab_size=self.type_vocab_size		,					initializer_range=self.initializer_range		,					return_dict=A__		,					)

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       Union[str, Any]:
						SCREAMING_SNAKE_CASE						= TFRoFormerModel(config=A__)
						SCREAMING_SNAKE_CASE						= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

						SCREAMING_SNAKE_CASE						= [input_ids, input_mask]
						SCREAMING_SNAKE_CASE						= model(A__)

						SCREAMING_SNAKE_CASE						= model(A__)

						self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size))






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       str:
						SCREAMING_SNAKE_CASE						= True
						SCREAMING_SNAKE_CASE						= TFRoFormerForCausalLM(config=A__)
						SCREAMING_SNAKE_CASE						= {
						    'input_ids': input_ids,
						    'attention_mask': input_mask,
						    'token_type_ids': token_type_ids,
						}
						SCREAMING_SNAKE_CASE						= model(A__)['logits']
						self.parent.assertListEqual(
						    list(prediction_scores.numpy().shape)		,					[self.batch_size, self.seq_length, self.vocab_size])






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       Dict:
						SCREAMING_SNAKE_CASE						= TFRoFormerForMaskedLM(config=A__)
						SCREAMING_SNAKE_CASE						= {
						    'input_ids': input_ids,
						    'attention_mask': input_mask,
						    'token_type_ids': token_type_ids,
						}
						SCREAMING_SNAKE_CASE						= model(A__)
						self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size))






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       List[str]:
						SCREAMING_SNAKE_CASE						= self.num_labels
						SCREAMING_SNAKE_CASE						= TFRoFormerForSequenceClassification(config=A__)
						SCREAMING_SNAKE_CASE						= {
						    'input_ids': input_ids,
						    'attention_mask': input_mask,
						    'token_type_ids': token_type_ids,
						}

						SCREAMING_SNAKE_CASE						= model(A__)
						self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_labels))






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       Any:
						SCREAMING_SNAKE_CASE						= self.num_choices
						SCREAMING_SNAKE_CASE						= TFRoFormerForMultipleChoice(config=A__)
						SCREAMING_SNAKE_CASE						= tf.tile(tf.expand_dims(A__		,					1)		,					(1, self.num_choices, 1))
						SCREAMING_SNAKE_CASE						= tf.tile(tf.expand_dims(A__		,					1)		,					(1, self.num_choices, 1))
						SCREAMING_SNAKE_CASE						= tf.tile(tf.expand_dims(A__		,					1)		,					(1, self.num_choices, 1))
						SCREAMING_SNAKE_CASE						= {
						    'input_ids': multiple_choice_inputs_ids,
						    'attention_mask': multiple_choice_input_mask,
						    'token_type_ids': multiple_choice_token_type_ids,
						}
						SCREAMING_SNAKE_CASE						= model(A__)
						self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_choices))






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       List[str]:
						SCREAMING_SNAKE_CASE						= self.num_labels
						SCREAMING_SNAKE_CASE						= TFRoFormerForTokenClassification(config=A__)
						SCREAMING_SNAKE_CASE						= {
						    'input_ids': input_ids,
						    'attention_mask': input_mask,
						    'token_type_ids': token_type_ids,
						}
						SCREAMING_SNAKE_CASE						= model(A__)
						self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.num_labels))






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a		,					a		,					a)      ->       Tuple:
						SCREAMING_SNAKE_CASE						= TFRoFormerForQuestionAnswering(config=A__)
						SCREAMING_SNAKE_CASE						= {
						    'input_ids': input_ids,
						    'attention_mask': input_mask,
						    'token_type_ids': token_type_ids,
						}

						SCREAMING_SNAKE_CASE						= model(A__)
						self.parent.assertEqual(result.start_logits.shape		,					(self.batch_size, self.seq_length))
						self.parent.assertEqual(result.end_logits.shape		,					(self.batch_size, self.seq_length))







			def    SCREAMING_SNAKE_CASE__    (       self)      ->       List[str]:
						SCREAMING_SNAKE_CASE						= self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE							
						)				,						(
						    SCREAMING_SNAKE_CASE							
						)				,						(
						    SCREAMING_SNAKE_CASE							
						)				,						(
						    SCREAMING_SNAKE_CASE							
						)				,						(
						    SCREAMING_SNAKE_CASE							
						)				,						(
						    SCREAMING_SNAKE_CASE							
						)				,						(
						    SCREAMING_SNAKE_CASE							
						)				,													
						)						= config_and_inputs
						SCREAMING_SNAKE_CASE						= {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
						return config, inputs_dict






@require_tf
class 		_snake_case  (  _lowerCamelCase   ,    _lowerCamelCase   ,    unittest.TestCase					):
			_lowercase		:    List[Any]			     =							(
			    (
			        TFRoFormerModel,
			        TFRoFormerForCausalLM,
			        TFRoFormerForMaskedLM,
			        TFRoFormerForQuestionAnswering,
			        TFRoFormerForSequenceClassification,
			        TFRoFormerForTokenClassification,
			        TFRoFormerForMultipleChoice,
			    )
			    if is_tf_available()
			    else ()
			)
			_lowercase		:    Any			     =							(
			    {
			        '''feature-extraction''': TFRoFormerModel,
			        '''fill-mask''': TFRoFormerForMaskedLM,
			        '''question-answering''': TFRoFormerForQuestionAnswering,
			        '''text-classification''': TFRoFormerForSequenceClassification,
			        '''text-generation''': TFRoFormerForCausalLM,
			        '''token-classification''': TFRoFormerForTokenClassification,
			        '''zero-shot''': TFRoFormerForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)

			_lowercase		:    Any			     =							False
			_lowercase		:    int			     =							False






			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					a)      ->       str:
						if pipeline_test_casse_name == "TextGenerationPipelineTests":
									return True

						return False






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Union[str, Any]:
						SCREAMING_SNAKE_CASE						= TFRoFormerModelTester(self)
						SCREAMING_SNAKE_CASE						= ConfigTester(self		,					config_class=A__		,					hidden_size=37)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Optional[Any]:
						self.config_tester.run_common_tests()






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       List[Any]:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*A__)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       str:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*A__)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       int:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_lm_head(*A__)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Dict:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*A__)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Union[str, Any]:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*A__)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Tuple:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*A__)






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       List[Any]:
						SCREAMING_SNAKE_CASE						= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*A__)







			@slow
			def    SCREAMING_SNAKE_CASE__    (       self)      ->       str:
						SCREAMING_SNAKE_CASE						= TFRoFormerModel.from_pretrained('junnyu/roformer_chinese_base')
						self.assertIsNotNone(A__)






@require_tf
class 		_snake_case  (  unittest.TestCase					):






			@slow
			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Optional[Any]:
						SCREAMING_SNAKE_CASE						= TFRoFormerForMaskedLM.from_pretrained('junnyu/roformer_chinese_base')
						SCREAMING_SNAKE_CASE						= tf.constant([[0, 1, 2, 3, 4, 5]])
						SCREAMING_SNAKE_CASE						= model(A__)[0]

						# TODO Replace vocab size
						SCREAMING_SNAKE_CASE						= 5_0000

						SCREAMING_SNAKE_CASE						= [1, 6, vocab_size]
						self.assertEqual(output.shape		,					A__)

						print(output[:, :3, :3])

						# TODO Replace values below with what was printed above.
						SCREAMING_SNAKE_CASE						= tf.constant(
						    [
						        [
						            [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
						            [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
						            [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
						        ]
						    ])
						tf.debugging.assert_near(output[:, :3, :3]		,					A__		,					atol=1E-4)






@require_tf
class 		_snake_case  (  unittest.TestCase					):
			_lowercase		:    str			     =							1e-4






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       List[Any]:
						SCREAMING_SNAKE_CASE						= tf.constant([[4, 10]])
						SCREAMING_SNAKE_CASE						= TFRoFormerSinusoidalPositionalEmbedding(num_positions=6		,					embedding_dim=6)

						SCREAMING_SNAKE_CASE						= emba(input_ids.shape)
						SCREAMING_SNAKE_CASE						= tf.constant(
						    [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]])

						tf.debugging.assert_near(A__		,					A__		,					atol=self.tolerance)







			def    SCREAMING_SNAKE_CASE__    (       self)      ->       List[Any]:
						SCREAMING_SNAKE_CASE						= tf.constant(
						    [
						        [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
						        [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
						        [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
						    ])
						SCREAMING_SNAKE_CASE						= TFRoFormerSinusoidalPositionalEmbedding(num_positions=512		,					embedding_dim=512)
						emba([2, 16, 512])
						SCREAMING_SNAKE_CASE						= emba.weight[:3, :5]

						tf.debugging.assert_near(A__		,					A__		,					atol=self.tolerance)






@require_tf
class 		_snake_case  (  unittest.TestCase					):
			_lowercase		:    List[str]			     =							1e-4






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       int:
						# 2,12,16,64
						SCREAMING_SNAKE_CASE						= tf.reshape(tf.range(2 * 12 * 16 * 64		,					dtype=tf.floataa)		,					shape=(2, 12, 16, 64)) / 100

						SCREAMING_SNAKE_CASE						= -tf.reshape(tf.range(2 * 12 * 16 * 64		,					dtype=tf.floataa)		,					shape=(2, 12, 16, 64)) / 100

						SCREAMING_SNAKE_CASE						= TFRoFormerSinusoidalPositionalEmbedding(num_positions=32		,					embedding_dim=64)
						SCREAMING_SNAKE_CASE						= embed_positions([2, 16, 768])[None, None, :, :]

						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= TFRoFormerSelfAttention.apply_rotary_position_embeddings(
						    A__		,					A__		,					A__)

						SCREAMING_SNAKE_CASE						= tf.constant(
						    [
						        [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
						        [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
						        [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
						        [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
						        [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
						        [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
						    ])
						SCREAMING_SNAKE_CASE						= tf.constant(
						    [
						        [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
						        [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
						        [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
						        [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
						        [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
						        [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
						    ])

						tf.debugging.assert_near(query_layer[0, 0, :6, :8]		,					A__		,					atol=self.tolerance)
						tf.debugging.assert_near(key_layer[0, 0, :6, :8]		,					A__		,					atol=self.tolerance)












import os
from collections.abc import Iterator






def 				lowerCamelCase__				(_UpperCAmelCase = "."):
			for dir_path, dir_names, filenames in os.walk(_UpperCAmelCase):
						SCREAMING_SNAKE_CASE						= [d for d in dir_names if d != 'scripts' and d[0] not in '._']
						for filename in filenames:
									if filename == "__init__.py":
												continue
									if os.path.splitext(_UpperCAmelCase)[1] in (".py", ".ipynb"):
												yield os.path.join(_UpperCAmelCase							,       _UpperCAmelCase).lstrip('./')






def 				lowerCamelCase__				(_UpperCAmelCase):
			return F'''{i * '  '}*''' if i else "\n##"






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			SCREAMING_SNAKE_CASE						= old_path.split(os.sep)
			for i, new_part in enumerate(new_path.split(os.sep)):
						if (i + 1 > len(_UpperCAmelCase) or old_parts[i] != new_part) and new_part:
									print(F'''{md_prefix(_UpperCAmelCase)} {new_part.replace('_'							,       ' ').title()}''')
			return new_path






def 				lowerCamelCase__				(_UpperCAmelCase = "."):
			SCREAMING_SNAKE_CASE						= ''
			for filepath in sorted(good_file_paths(_UpperCAmelCase)):
						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= os.path.split(_UpperCAmelCase)
						if filepath != old_path:
									SCREAMING_SNAKE_CASE						= print_path(_UpperCAmelCase							,       _UpperCAmelCase)
						SCREAMING_SNAKE_CASE						= (filepath.count(os.sep) + 1) if filepath else 0
						SCREAMING_SNAKE_CASE						= F'''{filepath}/{filename}'''.replace(' '							,       '%20')
						SCREAMING_SNAKE_CASE						= os.path.splitext(filename.replace('_'							,       ' ').title())[0]
						print(F'''{md_prefix(_UpperCAmelCase)} [{filename}]({url})''')


if __name__ == "__main__":
				print_directory_md('.')




