
'''simple docstring'''



import numpy as np
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     np.ndarray:




   """simple docstring"""
   return 1 / (1 + np.exp(-vector						))
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     np.ndarray:




   """simple docstring"""
   return vector * sigmoid(SCREAMING_SNAKE_CASE_						)


if __name__ == "__main__":
     import doctest

     doctest.testmod()








'''simple docstring'''



from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFRoFormerForCausalLM,
         TFRoFormerForMaskedLM,
         TFRoFormerForMultipleChoice,
         TFRoFormerForQuestionAnswering,
         TFRoFormerForSequenceClassification,
         TFRoFormerForTokenClassification,
         TFRoFormerModel,
     )
     from transformers.models.roformer.modeling_tf_roformer import (
         TFRoFormerSelfAttention,
         TFRoFormerSinusoidalPositionalEmbedding,
     )

class    _a				:





       """simple docstring"""




       def __init__(							self       ,							A__       ,							A__=13       ,							A__=7       ,							A__=True       ,							A__=True       ,							A__=True       ,							A__=True       ,							A__=99       ,							A__=32       ,							A__=2       ,							A__=4       ,							A__=37       ,							A__="gelu"       ,							A__=0.1       ,							A__=0.1       ,							A__=5_12       ,							A__=16       ,							A__=2       ,							A__=0.02       ,							A__=3       ,							A__=4       ,							A__=None       ,							)    ->			int:
          _SCREAMING_SNAKE_CASE      =				parent
          _SCREAMING_SNAKE_CASE      =				13
          _SCREAMING_SNAKE_CASE      =				7
          _SCREAMING_SNAKE_CASE      =				True
          _SCREAMING_SNAKE_CASE      =				True
          _SCREAMING_SNAKE_CASE      =				True
          _SCREAMING_SNAKE_CASE      =				True
          _SCREAMING_SNAKE_CASE      =				99
          _SCREAMING_SNAKE_CASE      =				32
          _SCREAMING_SNAKE_CASE      =				2
          _SCREAMING_SNAKE_CASE      =				4
          _SCREAMING_SNAKE_CASE      =				37
          _SCREAMING_SNAKE_CASE      =				"""gelu"""
          _SCREAMING_SNAKE_CASE      =				0.1
          _SCREAMING_SNAKE_CASE      =				0.1
          _SCREAMING_SNAKE_CASE      =				5_12
          _SCREAMING_SNAKE_CASE      =				16
          _SCREAMING_SNAKE_CASE      =				2
          _SCREAMING_SNAKE_CASE      =				0.02
          _SCREAMING_SNAKE_CASE      =				3
          _SCREAMING_SNAKE_CASE      =				4
          _SCREAMING_SNAKE_CASE      =				None






       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size      )

          _SCREAMING_SNAKE_CASE      =				None
          if self.use_input_mask:
             _SCREAMING_SNAKE_CASE      =				random_attention_mask([self.batch_size, self.seq_length]      )

          _SCREAMING_SNAKE_CASE      =				None
          if self.use_token_type_ids:
             _SCREAMING_SNAKE_CASE      =				ids_tensor([self.batch_size, self.seq_length]       ,							self.type_vocab_size      )

          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				None
          if self.use_labels:
             _SCREAMING_SNAKE_CASE      =				ids_tensor([self.batch_size]       ,							self.type_sequence_label_size      )
             _SCREAMING_SNAKE_CASE      =				ids_tensor([self.batch_size, self.seq_length]       ,							self.num_labels      )
             _SCREAMING_SNAKE_CASE      =				ids_tensor([self.batch_size]       ,							self.num_choices      )

          _SCREAMING_SNAKE_CASE      =				RoFormerConfig(
              vocab_size=self.vocab_size       ,							hidden_size=self.hidden_size       ,							num_hidden_layers=self.num_hidden_layers       ,							num_attention_heads=self.num_attention_heads       ,							intermediate_size=self.intermediate_size       ,							hidden_act=self.hidden_act       ,							hidden_dropout_prob=self.hidden_dropout_prob       ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,							max_position_embeddings=self.max_position_embeddings       ,							type_vocab_size=self.type_vocab_size       ,							initializer_range=self.initializer_range       ,							return_dict=A__       ,							)

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				TFRoFormerModel(config=A__      )
          _SCREAMING_SNAKE_CASE      =				{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

          _SCREAMING_SNAKE_CASE      =				[input_ids, input_mask]
          _SCREAMING_SNAKE_CASE      =				model(A__      )

          _SCREAMING_SNAKE_CASE      =				model(A__      )

          self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				True
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForCausalLM(config=A__      )
          _SCREAMING_SNAKE_CASE      =				{
              """input_ids""": input_ids,
              """attention_mask""": input_mask,
              """token_type_ids""": token_type_ids,
          }
          _SCREAMING_SNAKE_CASE      =				model(A__      )["""logits"""]
          self.parent.assertListEqual(
              list(prediction_scores.numpy().shape      )       ,							[self.batch_size, self.seq_length, self.vocab_size]      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			Dict:
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForMaskedLM(config=A__      )
          _SCREAMING_SNAKE_CASE      =				{
              """input_ids""": input_ids,
              """attention_mask""": input_mask,
              """token_type_ids""": token_type_ids,
          }
          _SCREAMING_SNAKE_CASE      =				model(A__      )
          self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.vocab_size)      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				self.num_labels
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForSequenceClassification(config=A__      )
          _SCREAMING_SNAKE_CASE      =				{
              """input_ids""": input_ids,
              """attention_mask""": input_mask,
              """token_type_ids""": token_type_ids,
          }

          _SCREAMING_SNAKE_CASE      =				model(A__      )
          self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_labels)      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				self.num_choices
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForMultipleChoice(config=A__      )
          _SCREAMING_SNAKE_CASE      =				tf.tile(tf.expand_dims(A__       ,							1      )       ,							(1, self.num_choices, 1)      )
          _SCREAMING_SNAKE_CASE      =				tf.tile(tf.expand_dims(A__       ,							1      )       ,							(1, self.num_choices, 1)      )
          _SCREAMING_SNAKE_CASE      =				tf.tile(tf.expand_dims(A__       ,							1      )       ,							(1, self.num_choices, 1)      )
          _SCREAMING_SNAKE_CASE      =				{
              """input_ids""": multiple_choice_inputs_ids,
              """attention_mask""": multiple_choice_input_mask,
              """token_type_ids""": multiple_choice_token_type_ids,
          }
          _SCREAMING_SNAKE_CASE      =				model(A__      )
          self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_choices)      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				self.num_labels
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForTokenClassification(config=A__      )
          _SCREAMING_SNAKE_CASE      =				{
              """input_ids""": input_ids,
              """attention_mask""": input_mask,
              """token_type_ids""": token_type_ids,
          }
          _SCREAMING_SNAKE_CASE      =				model(A__      )
          self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.num_labels)      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForQuestionAnswering(config=A__      )
          _SCREAMING_SNAKE_CASE      =				{
              """input_ids""": input_ids,
              """attention_mask""": input_mask,
              """token_type_ids""": token_type_ids,
          }

          _SCREAMING_SNAKE_CASE      =				model(A__      )
          self.parent.assertEqual(result.start_logits.shape       ,							(self.batch_size, self.seq_length)      )
          self.parent.assertEqual(result.end_logits.shape       ,							(self.batch_size, self.seq_length)      )






       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				self.prepare_config_and_inputs()
          (
              (
              _SCREAMING_SNAKE_CASE
          ) ,							(
              _SCREAMING_SNAKE_CASE
          ) ,							(
              _SCREAMING_SNAKE_CASE
          ) ,							(
              _SCREAMING_SNAKE_CASE
          ) ,							(
              _SCREAMING_SNAKE_CASE
          ) ,							(
              _SCREAMING_SNAKE_CASE
          ) ,							(
              _SCREAMING_SNAKE_CASE
          ) ,							
          )      =				config_and_inputs
          _SCREAMING_SNAKE_CASE      =				{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
          return config, inputs_dict






@require_tf
class    _a				(_lowerCamelCase  ,					_lowerCamelCase  ,					unittest.TestCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  (
           (
               TFRoFormerModel,
               TFRoFormerForCausalLM,
               TFRoFormerForMaskedLM,
               TFRoFormerForQuestionAnswering,
               TFRoFormerForSequenceClassification,
               TFRoFormerForTokenClassification,
               TFRoFormerForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       SCREAMING_SNAKE_CASE      =  (
           {
               'feature-extraction': TFRoFormerModel,
               'fill-mask': TFRoFormerForMaskedLM,
               'question-answering': TFRoFormerForQuestionAnswering,
               'text-classification': TFRoFormerForSequenceClassification,
               'text-generation': TFRoFormerForCausalLM,
               'token-classification': TFRoFormerForTokenClassification,
               'zero-shot': TFRoFormerForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )

       SCREAMING_SNAKE_CASE      =  False
       SCREAMING_SNAKE_CASE      =  False






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			str:
          if pipeline_test_casse_name == "TextGenerationPipelineTests":
             return True

          return False






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				TFRoFormerModelTester(self      )
          _SCREAMING_SNAKE_CASE      =				ConfigTester(self       ,							config_class=A__       ,							hidden_size=37      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          self.config_tester.run_common_tests()






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*A__      )






       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*A__      )






       def        UpperCamelCase				(							self      )    ->			int:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_lm_head(*A__      )






       def        UpperCamelCase				(							self      )    ->			Dict:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*A__      )






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*A__      )






       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*A__      )






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*A__      )






       @slow
       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				TFRoFormerModel.from_pretrained("""junnyu/roformer_chinese_base"""      )
          self.assertIsNotNone(A__      )






@require_tf
class    _a				(unittest.TestCase):





       """simple docstring"""




       @slow
       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				TFRoFormerForMaskedLM.from_pretrained("""junnyu/roformer_chinese_base"""      )
          _SCREAMING_SNAKE_CASE      =				tf.constant([[0, 1, 2, 3, 4, 5]]      )
          _SCREAMING_SNAKE_CASE      =				model(A__      )[0]

          # TODO Replace vocab size
          _SCREAMING_SNAKE_CASE      =				5_00_00

          _SCREAMING_SNAKE_CASE      =				[1, 6, vocab_size]
          self.assertEqual(output.shape       ,							A__      )

          print(output[:, :3, :3]      )

          # TODO Replace values below with what was printed above.
          _SCREAMING_SNAKE_CASE      =				tf.constant(
              [
                  [
                      [-0.1205_3341, -1.026_4901, 0.2922_1946],
                      [-1.513_3783, 0.19_7433, 0.1519_0607],
                      [-5.013_5403, -3.90_0256, -0.8403_8764],
                  ]
              ]      )
          tf.debugging.assert_near(output[:, :3, :3]       ,							A__       ,							atol=1E-4      )






@require_tf
class    _a				(unittest.TestCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  1E-4






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				tf.constant([[4, 10]]      )
          _SCREAMING_SNAKE_CASE      =				TFRoFormerSinusoidalPositionalEmbedding(num_positions=6       ,							embedding_dim=6      )

          _SCREAMING_SNAKE_CASE      =				emba(input_ids.shape      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(
              [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]      )

          tf.debugging.assert_near(A__       ,							A__       ,							atol=self.tolerance      )






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				tf.constant(
              [
                  [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
                  [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
                  [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
              ]      )
          _SCREAMING_SNAKE_CASE      =				TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_12       ,							embedding_dim=5_12      )
          emba([2, 16, 5_12]      )
          _SCREAMING_SNAKE_CASE      =				emba.weight[:3, :5]

          tf.debugging.assert_near(A__       ,							A__       ,							atol=self.tolerance      )






@require_tf
class    _a				(unittest.TestCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  1E-4






       def        UpperCamelCase				(							self      )    ->			int:
          # 2,12,16,64
          _SCREAMING_SNAKE_CASE      =				tf.reshape(tf.range(2 * 12 * 16 * 64       ,							dtype=tf.floataa      )       ,							shape=(2, 12, 16, 64)      ) / 1_00

          _SCREAMING_SNAKE_CASE      =				-tf.reshape(tf.range(2 * 12 * 16 * 64       ,							dtype=tf.floataa      )       ,							shape=(2, 12, 16, 64)      ) / 1_00

          _SCREAMING_SNAKE_CASE      =				TFRoFormerSinusoidalPositionalEmbedding(num_positions=32       ,							embedding_dim=64      )
          _SCREAMING_SNAKE_CASE      =				embed_positions([2, 16, 7_68]      )[None, None, :, :]

          _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				TFRoFormerSelfAttention.apply_rotary_position_embeddings(
              A__       ,							A__       ,							A__      )

          _SCREAMING_SNAKE_CASE      =				tf.constant(
              [
                  [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
                  [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
                  [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
                  [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
                  [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
                  [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
              ]      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(
              [
                  [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
                  [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
                  [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
                  [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
                  [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
                  [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
              ]      )

          tf.debugging.assert_near(query_layer[0, 0, :6, :8]       ,							A__       ,							atol=self.tolerance      )
          tf.debugging.assert_near(key_layer[0, 0, :6, :8]       ,							A__       ,							atol=self.tolerance      )






