






'''simple docstring'''



from manim import *
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      def 				A__      ( self			:		List[Any]		)  ->  int:


       '''simple docstring'''

       lowercase					:   Tuple				  =Rectangle(height=0.5     ,      width=0.5		)
       lowercase					:   str				  =Rectangle(height=0.4_6     ,      width=0.4_6		).set_stroke(width=0		)

       lowercase					:   Any				  =Rectangle(height=0.2_5     ,      width=0.2_5		)

       lowercase					:   Tuple				  =[mem.copy() for i in range(6		)]
       lowercase					:   Tuple				  =[mem.copy() for i in range(6		)]
       lowercase					:   Optional[int]				  =VGroup(*UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   List[str]				  =VGroup(*UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   Dict				  =VGroup(UpperCAmelCase     ,      UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   Any				  =Text('''CPU'''     ,      font_size=24		)
       lowercase					:   Tuple				  =Group(UpperCAmelCase     ,      UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0.5     ,      aligned_edge=UpperCAmelCase		)
       cpu.move_to([-2.5, -0.5, 0]		)
       self.add(UpperCAmelCase		)

       lowercase					:   Tuple				  =[mem.copy() for i in range(4		)]
       lowercase					:   Dict				  =VGroup(*UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   List[Any]				  =Text('''GPU'''     ,      font_size=24		)
       lowercase					:   Union[str, Any]				  =Group(UpperCAmelCase     ,      UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0.5     ,      aligned_edge=UpperCAmelCase		)
       gpu.move_to([-1, -1, 0]		)
       self.add(UpperCAmelCase		)

       lowercase					:   List[str]				  =[mem.copy() for i in range(6		)]
       lowercase					:   str				  =VGroup(*UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)

       lowercase					:   Any				  =Text('''Model'''     ,      font_size=24		)
       lowercase					:   Dict				  =Group(UpperCAmelCase     ,      UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0.5     ,      aligned_edge=UpperCAmelCase		)
       model.move_to([3, -1.0, 0]		)
       self.add(UpperCAmelCase		)

       lowercase					:   List[str]				  =[]
       lowercase					:   int				  =[]

       for i, rect in enumerate(UpperCAmelCase		):
        lowercase					:   List[Any]				  =fill.copy().set_fill(UpperCAmelCase     ,      opacity=0.8		)
        target.move_to(UpperCAmelCase		)
        model_arr.append(UpperCAmelCase		)

        lowercase					:   Union[str, Any]				  =Rectangle(height=0.4_6     ,      width=0.4_6		).set_stroke(width=0.0		).set_fill(UpperCAmelCase     ,      opacity=0.8		)
        cpu_target.move_to(cpu_left_col_base[i]		)
        model_cpu_arr.append(UpperCAmelCase		)

       self.add(*UpperCAmelCase     ,      *UpperCAmelCase		)

       lowercase					:   List[Any]				  =[meta_mem.copy() for i in range(6		)]
       lowercase					:   Any				  =[meta_mem.copy() for i in range(6		)]
       lowercase					:   Optional[int]				  =VGroup(*UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   str				  =VGroup(*UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   Optional[Any]				  =VGroup(UpperCAmelCase     ,      UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0		)
       lowercase					:   List[str]				  =Text('''Disk'''     ,      font_size=24		)
       lowercase					:   Optional[int]				  =Group(UpperCAmelCase     ,      UpperCAmelCase		).arrange(UpperCAmelCase     ,      buff=0.5     ,      aligned_edge=UpperCAmelCase		)
       disk.move_to([-4, -1.2_5, 0]		)
       self.add(UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   List[str]				  =Square(side_length=2.2		)
       key.move_to([-5, 2, 0]		)

       lowercase					:   List[str]				  =MarkupText(
           f'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'     ,      font_size=18     ,      )

       key_text.move_to([-5, 2.4, 0]		)

       self.add(UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   str				  =MarkupText(
           f'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'     ,      font_size=18     ,      )

       blue_text.next_to(UpperCAmelCase     ,      DOWN * 2.4     ,      aligned_edge=key_text.get_left()		)
       self.add(UpperCAmelCase		)

       lowercase					:   List[Any]				  =MarkupText(
           f'Now watch as an input is passed through the model\nand how the memory is utilized and handled.'     ,      font_size=24     ,      )
       step_a.move_to([2, 2, 0]		)

       self.play(Write(UpperCAmelCase		)		)

       lowercase					:   Optional[int]				  =Square(0.3		)
       input.set_fill(UpperCAmelCase     ,      opacity=1.0		)
       input.set_stroke(width=0.0		)
       input.next_to(model_base[0]     ,      UpperCAmelCase     ,      buff=0.5		)

       self.play(Write(UpperCAmelCase		)		)

       input.generate_target()
       input.target.next_to(model_arr[0]     ,      direction=UpperCAmelCase     ,      buff=0.0_2		)
       self.play(MoveToTarget(UpperCAmelCase		)		)

       self.play(FadeOut(UpperCAmelCase		)		)

       lowercase					:   str				  =Arrow(start=UpperCAmelCase     ,      end=UpperCAmelCase     ,      color=UpperCAmelCase     ,      buff=0.5		)
       a.next_to(model_arr[0].get_left()     ,      UpperCAmelCase     ,      buff=0.2		)

       model_cpu_arr[0].generate_target()
       model_cpu_arr[0].target.move_to(gpu_rect[0]		)

       lowercase					:   Optional[int]				  =MarkupText(
           f'As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'     ,      font_size=24     ,      )
       step_a.move_to([2, 2, 0]		)

       self.play(Write(UpperCAmelCase     ,      run_time=3		)		)

       lowercase					:   int				  ={'''run_time''': 1, '''fade_in''': True, '''fade_out''': True, '''buff''': 0.0_2}

       self.play(
           Write(UpperCAmelCase		)     ,      Circumscribe(model_arr[0]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      Circumscribe(model_cpu_arr[0]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      Circumscribe(gpu_rect[0]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      )
       self.play(MoveToTarget(model_cpu_arr[0]		)		)

       lowercase					:   Union[str, Any]				  =a.copy()
       for i in range(6		):
        a_c.next_to(model_arr[i].get_right() + 0.0_2     ,      UpperCAmelCase     ,      buff=0.2		)

        input.generate_target()
        input.target.move_to(model_arr[i].get_right() + 0.0_2		)

        lowercase					:   Union[str, Any]				  =AnimationGroup(
            FadeOut(UpperCAmelCase     ,      run_time=0.5		)     ,      MoveToTarget(UpperCAmelCase     ,      run_time=0.5		)     ,      FadeIn(UpperCAmelCase     ,      run_time=0.5		)     ,      lag_ratio=0.2		)

        self.play(UpperCAmelCase		)

        model_cpu_arr[i].generate_target()
        model_cpu_arr[i].target.move_to(cpu_left_col_base[i]		)

        if i < 5:
         model_cpu_arr[i + 1].generate_target()
         model_cpu_arr[i + 1].target.move_to(gpu_rect[0]		)
         if i >= 1:
          lowercase					:   str				  =0.7

         self.play(
             Circumscribe(model_arr[i]     ,      **UpperCAmelCase		)     ,      Circumscribe(cpu_left_col_base[i]     ,      **UpperCAmelCase		)     ,      Circumscribe(cpu_left_col_base[i + 1]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      Circumscribe(gpu_rect[0]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      Circumscribe(model_arr[i + 1]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      )
         if i < 1:
          self.play(
              MoveToTarget(model_cpu_arr[i]		)     ,      MoveToTarget(model_cpu_arr[i + 1]		)     ,      )
         else:
          self.play(
              MoveToTarget(model_cpu_arr[i]     ,      run_time=0.7		)     ,      MoveToTarget(model_cpu_arr[i + 1]     ,      run_time=0.7		)     ,      )
        else:
         model_cpu_arr[i].generate_target()
         model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]		)
         input.generate_target()
         input.target.next_to(model_arr[-1].get_right()     ,      RIGHT + 0.0_2     ,      buff=0.2		)

         self.play(
             Circumscribe(model_arr[-1]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      Circumscribe(cpu_left_col_base[-1]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      Circumscribe(gpu_rect[0]     ,      color=UpperCAmelCase     ,      **UpperCAmelCase		)     ,      )

         self.play(MoveToTarget(model_cpu_arr[i]		)		)

        lowercase					:   Tuple				  =a_c
        lowercase					:   Union[str, Any]				  =a_c.copy()

       input.generate_target()
       input.target.next_to(model_base[-1]     ,      RIGHT + 0.0_2     ,      buff=0.5		)
       self.play(
           FadeOut(UpperCAmelCase		)     ,      FadeOut(UpperCAmelCase     ,      run_time=0.5		)     ,      )

       lowercase					:   Tuple				  =MarkupText(f'Inference on a model too large for GPU memory\nis successfully completed.'     ,      font_size=24		)
       step_a.move_to([2, 2, 0]		)

       self.play(Write(UpperCAmelCase     ,      run_time=3		)     ,      MoveToTarget(UpperCAmelCase		)		)

       self.wait()











'''simple docstring'''



from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          TFRoFormerForCausalLM,
          TFRoFormerForMaskedLM,
          TFRoFormerForMultipleChoice,
          TFRoFormerForQuestionAnswering,
          TFRoFormerForSequenceClassification,
          TFRoFormerForTokenClassification,
          TFRoFormerModel,
      )
      from transformers.models.roformer.modeling_tf_roformer import (
          TFRoFormerSelfAttention,
          TFRoFormerSinusoidalPositionalEmbedding,
      )
class UpperCAmelCase_					:




      """simple docstring"""




      def __init__( self			:		Optional[Any]     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		Union[str, Any]=13     ,      UpperCAmelCase			:		int=7     ,      UpperCAmelCase			:		Union[str, Any]=True     ,      UpperCAmelCase			:		int=True     ,      UpperCAmelCase			:		Any=True     ,      UpperCAmelCase			:		int=True     ,      UpperCAmelCase			:		Any=99     ,      UpperCAmelCase			:		List[Any]=32     ,      UpperCAmelCase			:		str=2     ,      UpperCAmelCase			:		str=4     ,      UpperCAmelCase			:		List[Any]=37     ,      UpperCAmelCase			:		str="gelu"     ,      UpperCAmelCase			:		List[Any]=0.1     ,      UpperCAmelCase			:		int=0.1     ,      UpperCAmelCase			:		Dict=512     ,      UpperCAmelCase			:		List[Any]=16     ,      UpperCAmelCase			:		List[str]=2     ,      UpperCAmelCase			:		Any=0.0_2     ,      UpperCAmelCase			:		List[str]=3     ,      UpperCAmelCase			:		Dict=4     ,      UpperCAmelCase			:		Optional[int]=None     ,      )  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =parent
       lowercase					:   Tuple				  =13
       lowercase					:   Any				  =7
       lowercase					:   Union[str, Any]				  =True
       lowercase					:   Any				  =True
       lowercase					:   Optional[int]				  =True
       lowercase					:   List[str]				  =True
       lowercase					:   Tuple				  =99
       lowercase					:   str				  =32
       lowercase					:   Union[str, Any]				  =2
       lowercase					:   Dict				  =4
       lowercase					:   Union[str, Any]				  =37
       lowercase					:   Union[str, Any]				  ='''gelu'''
       lowercase					:   Any				  =0.1
       lowercase					:   Dict				  =0.1
       lowercase					:   Dict				  =512
       lowercase					:   List[str]				  =16
       lowercase					:   Dict				  =2
       lowercase					:   int				  =0.0_2
       lowercase					:   List[Any]				  =3
       lowercase					:   List[str]				  =4
       lowercase					:   Optional[Any]				  =None






      def 				A__      ( self			:		Union[str, Any]		)  ->  int:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size		)

       lowercase					:   str				  =None
       if self.use_input_mask:
        lowercase					:   int				  =random_attention_mask([self.batch_size, self.seq_length]		)

       lowercase					:   Any				  =None
       if self.use_token_type_ids:
        lowercase					:   List[str]				  =ids_tensor([self.batch_size, self.seq_length]     ,      self.type_vocab_size		)

       lowercase					:   List[Any]				  =None
       lowercase					:   List[str]				  =None
       lowercase					:   List[str]				  =None
       if self.use_labels:
        lowercase					:   Optional[Any]				  =ids_tensor([self.batch_size]     ,      self.type_sequence_label_size		)
        lowercase					:   List[Any]				  =ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels		)
        lowercase					:   Any				  =ids_tensor([self.batch_size]     ,      self.num_choices		)

       lowercase					:   List[Any]				  =RoFormerConfig(
           vocab_size=self.vocab_size     ,      hidden_size=self.hidden_size     ,      num_hidden_layers=self.num_hidden_layers     ,      num_attention_heads=self.num_attention_heads     ,      intermediate_size=self.intermediate_size     ,      hidden_act=self.hidden_act     ,      hidden_dropout_prob=self.hidden_dropout_prob     ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,      max_position_embeddings=self.max_position_embeddings     ,      type_vocab_size=self.type_vocab_size     ,      initializer_range=self.initializer_range     ,      return_dict=UpperCAmelCase     ,      )

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






      def 				A__      ( self			:		List[str]     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Optional[int]     ,      UpperCAmelCase			:		Tuple     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		List[Any]		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =TFRoFormerModel(config=UpperCAmelCase		)
       lowercase					:   Optional[int]				  ={'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

       lowercase					:   Tuple				  =[input_ids, input_mask]
       lowercase					:   str				  =model(UpperCAmelCase		)

       lowercase					:   Dict				  =model(UpperCAmelCase		)

       self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)		)






      def 				A__      ( self			:		Tuple     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		List[str]		)  ->  Any:


       '''simple docstring'''

       lowercase					:   Dict				  =True
       lowercase					:   List[Any]				  =TFRoFormerForCausalLM(config=UpperCAmelCase		)
       lowercase					:   Union[str, Any]				  ={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       lowercase					:   Optional[Any]				  =model(UpperCAmelCase		)['''logits''']
       self.parent.assertListEqual(
           list(prediction_scores.numpy().shape		)     ,      [self.batch_size, self.seq_length, self.vocab_size]		)






      def 				A__      ( self			:		List[Any]     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		Tuple		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   List[Any]				  =TFRoFormerForMaskedLM(config=UpperCAmelCase		)
       lowercase					:   List[str]				  ={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       lowercase					:   Dict				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)		)






      def 				A__      ( self			:		Tuple     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Optional[int]     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		Tuple     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		int		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =self.num_labels
       lowercase					:   Optional[int]				  =TFRoFormerForSequenceClassification(config=UpperCAmelCase		)
       lowercase					:   Optional[int]				  ={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }

       lowercase					:   Optional[Any]				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_labels)		)






      def 				A__      ( self			:		Optional[Any]     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Optional[Any]     ,      UpperCAmelCase			:		Tuple		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   int				  =self.num_choices
       lowercase					:   Tuple				  =TFRoFormerForMultipleChoice(config=UpperCAmelCase		)
       lowercase					:   Union[str, Any]				  =tf.tile(tf.expand_dims(UpperCAmelCase     ,      1		)     ,      (1, self.num_choices, 1)		)
       lowercase					:   List[Any]				  =tf.tile(tf.expand_dims(UpperCAmelCase     ,      1		)     ,      (1, self.num_choices, 1)		)
       lowercase					:   Tuple				  =tf.tile(tf.expand_dims(UpperCAmelCase     ,      1		)     ,      (1, self.num_choices, 1)		)
       lowercase					:   List[Any]				  ={
           '''input_ids''': multiple_choice_inputs_ids,
           '''attention_mask''': multiple_choice_input_mask,
           '''token_type_ids''': multiple_choice_token_type_ids,
       }
       lowercase					:   Dict				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_choices)		)






      def 				A__      ( self			:		Optional[int]     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		Any     ,      UpperCAmelCase			:		List[str]     ,      UpperCAmelCase			:		int		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self.num_labels
       lowercase					:   Union[str, Any]				  =TFRoFormerForTokenClassification(config=UpperCAmelCase		)
       lowercase					:   Tuple				  ={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       lowercase					:   List[str]				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)		)






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Dict     ,      UpperCAmelCase			:		Optional[int]     ,      UpperCAmelCase			:		Union[str, Any]     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		str		)  ->  Any:


       '''simple docstring'''

       lowercase					:   Tuple				  =TFRoFormerForQuestionAnswering(config=UpperCAmelCase		)
       lowercase					:   List[str]				  ={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }

       lowercase					:   List[str]				  =model(UpperCAmelCase		)
       self.parent.assertEqual(result.start_logits.shape     ,      (self.batch_size, self.seq_length)		)
       self.parent.assertEqual(result.end_logits.shape     ,      (self.batch_size, self.seq_length)		)






      def 				A__      ( self			:		List[Any]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =self.prepare_config_and_inputs()
       (
           (
           lowercase
       )  ,		(
           lowercase
       )  ,		(
           lowercase
       )  ,		(
           lowercase
       )  ,		(
           lowercase
       )  ,		(
           lowercase
       )  ,		(
           lowercase
       )  ,		
       )					:   Optional[int]				  =config_and_inputs
       lowercase					:   str				  ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict





@require_tf
class UpperCAmelCase_					(    __A				,       __A				,       unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			(
          (
              TFRoFormerModel,
              TFRoFormerForCausalLM,
              TFRoFormerForMaskedLM,
              TFRoFormerForQuestionAnswering,
              TFRoFormerForSequenceClassification,
              TFRoFormerForTokenClassification,
              TFRoFormerForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      UpperCamelCase_								=			(
          {
              '''feature-extraction''': TFRoFormerModel,
              '''fill-mask''': TFRoFormerForMaskedLM,
              '''question-answering''': TFRoFormerForQuestionAnswering,
              '''text-classification''': TFRoFormerForSequenceClassification,
              '''text-generation''': TFRoFormerForCausalLM,
              '''token-classification''': TFRoFormerForTokenClassification,
              '''zero-shot''': TFRoFormerForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )

      UpperCamelCase_								=			False
      UpperCamelCase_								=			False






      def 				A__      ( self			:		Dict     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		List[Any]     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Optional[int]     ,      UpperCAmelCase			:		str		)  ->  Tuple:


       '''simple docstring'''

       if pipeline_test_casse_name == "TextGenerationPipelineTests":
        return True

       return False






      def 				A__      ( self			:		str		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =TFRoFormerModelTester(self		)
       lowercase					:   Union[str, Any]				  =ConfigTester(self     ,      config_class=UpperCAmelCase     ,      hidden_size=37		)






      def 				A__      ( self			:		Optional[Any]		)  ->  Union[str, Any]:


       '''simple docstring'''

       self.config_tester.run_common_tests()






      def 				A__      ( self			:		List[str]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   str				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*UpperCAmelCase		)






      def 				A__      ( self			:		List[Any]		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   Tuple				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase		)






      def 				A__      ( self			:		str		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   Any				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_lm_head(*UpperCAmelCase		)






      def 				A__      ( self			:		int		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase		)






      def 				A__      ( self			:		Dict		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   Dict				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase		)






      def 				A__      ( self			:		Dict		)  ->  Any:


       '''simple docstring'''

       lowercase					:   int				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase		)






      def 				A__      ( self			:		List[Any]		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   str				  =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase		)






      @slow
      def 				A__      ( self			:		str		)  ->  str:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''		)
       self.assertIsNotNone(UpperCAmelCase		)





@require_tf
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      @slow
      def 				A__      ( self			:		Union[str, Any]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Any				  =TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''		)
       lowercase					:   Optional[Any]				  =tf.constant([[0, 1, 2, 3, 4, 5]]		)
       lowercase					:   List[str]				  =model(UpperCAmelCase		)[0]

       # TODO Replace vocab size
       lowercase					:   Tuple				  =5_0000

       lowercase					:   List[str]				  =[1, 6, vocab_size]
       self.assertEqual(output.shape     ,      UpperCAmelCase		)

       print(output[:, :3, :3]		)

       # TODO Replace values below with what was printed above.
       lowercase					:   Dict				  =tf.constant(
           [
               [
                   [-0.1_2_0_5_3_3_4_1, -1.0_2_6_4_9_0_1, 0.2_9_2_2_1_9_4_6],
                   [-1.5_1_3_3_7_8_3, 0.1_9_7_4_3_3, 0.1_5_1_9_0_6_0_7],
                   [-5.0_1_3_5_4_0_3, -3.9_0_0_2_5_6, -0.8_4_0_3_8_7_6_4],
               ]
           ]		)
       tf.debugging.assert_near(output[:, :3, :3]     ,      UpperCAmelCase     ,      atol=1e-4		)





@require_tf
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			1e-4






      def 				A__      ( self			:		int		)  ->  List[Any]:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =tf.constant([[4, 10]]		)
       lowercase					:   List[Any]				  =TFRoFormerSinusoidalPositionalEmbedding(num_positions=6     ,      embedding_dim=6		)

       lowercase					:   Any				  =emba(input_ids.shape		)
       lowercase					:   List[str]				  =tf.constant(
           [[0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0, 1.0_0_0_0, 1.0_0_0_0, 1.0_0_0_0], [0.8_4_1_5, 0.0_4_6_4, 0.0_0_2_2, 0.5_4_0_3, 0.9_9_8_9, 1.0_0_0_0]]		)

       tf.debugging.assert_near(UpperCAmelCase     ,      UpperCAmelCase     ,      atol=self.tolerance		)






      def 				A__      ( self			:		Optional[Any]		)  ->  int:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =tf.constant(
           [
               [0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0],
               [0.8_4_1_5, 0.8_2_1_9, 0.8_0_2_0, 0.7_8_1_9, 0.7_6_1_7],
               [0.9_0_9_3, 0.9_3_6_4, 0.9_5_8_1, 0.9_7_4_9, 0.9_8_7_0],
           ]		)
       lowercase					:   Tuple				  =TFRoFormerSinusoidalPositionalEmbedding(num_positions=512     ,      embedding_dim=512		)
       emba([2, 16, 512]		)
       lowercase					:   str				  =emba.weight[:3, :5]

       tf.debugging.assert_near(UpperCAmelCase     ,      UpperCAmelCase     ,      atol=self.tolerance		)





@require_tf
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			1e-4






      def 				A__      ( self			:		Dict		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   str				  =tf.reshape(tf.range(2 * 12 * 16 * 64     ,      dtype=tf.floataa		)     ,      shape=(2, 12, 16, 64)		) / 100

       lowercase					:   Any				  =-tf.reshape(tf.range(2 * 12 * 16 * 64     ,      dtype=tf.floataa		)     ,      shape=(2, 12, 16, 64)		) / 100

       lowercase					:   Any				  =TFRoFormerSinusoidalPositionalEmbedding(num_positions=32     ,      embedding_dim=64		)
       lowercase					:   Optional[Any]				  =embed_positions([2, 16, 768]		)[None, None, :, :]

       lowercase  ,		lowercase					:   Optional[int]				  =TFRoFormerSelfAttention.apply_rotary_position_embeddings(
           UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase		)

       lowercase					:   Any				  =tf.constant(
           [
               [0.0_0_0_0, 0.0_1_0_0, 0.0_2_0_0, 0.0_3_0_0, 0.0_4_0_0, 0.0_5_0_0, 0.0_6_0_0, 0.0_7_0_0],
               [-0.2_0_1_2, 0.8_8_9_7, 0.0_2_6_3, 0.9_4_0_1, 0.2_0_7_4, 0.9_4_6_3, 0.3_4_8_1, 0.9_3_4_3],
               [-1.7_0_5_7, 0.6_2_7_1, -1.2_1_4_5, 1.3_8_9_7, -0.6_3_0_3, 1.7_6_4_7, -0.1_1_7_3, 1.8_9_8_5],
               [-2.1_7_3_1, -1.6_3_9_7, -2.7_3_5_8, 0.2_8_5_4, -2.1_8_4_0, 1.7_1_8_3, -1.3_0_1_8, 2.4_8_7_1],
               [0.2_7_1_7, -3.6_1_7_3, -2.9_2_0_6, -2.1_9_8_8, -3.6_6_3_8, 0.3_8_5_8, -2.9_1_5_5, 2.2_9_8_0],
               [3.9_8_5_9, -2.1_5_8_0, -0.7_9_8_4, -4.4_9_0_4, -4.1_1_8_1, -2.0_2_5_2, -4.4_7_8_2, 1.1_2_5_3],
           ]		)
       lowercase					:   int				  =tf.constant(
           [
               [0.0_0_0_0, -0.0_1_0_0, -0.0_2_0_0, -0.0_3_0_0, -0.0_4_0_0, -0.0_5_0_0, -0.0_6_0_0, -0.0_7_0_0],
               [0.2_0_1_2, -0.8_8_9_7, -0.0_2_6_3, -0.9_4_0_1, -0.2_0_7_4, -0.9_4_6_3, -0.3_4_8_1, -0.9_3_4_3],
               [1.7_0_5_7, -0.6_2_7_1, 1.2_1_4_5, -1.3_8_9_7, 0.6_3_0_3, -1.7_6_4_7, 0.1_1_7_3, -1.8_9_8_5],
               [2.1_7_3_1, 1.6_3_9_7, 2.7_3_5_8, -0.2_8_5_4, 2.1_8_4_0, -1.7_1_8_3, 1.3_0_1_8, -2.4_8_7_1],
               [-0.2_7_1_7, 3.6_1_7_3, 2.9_2_0_6, 2.1_9_8_8, 3.6_6_3_8, -0.3_8_5_8, 2.9_1_5_5, -2.2_9_8_0],
               [-3.9_8_5_9, 2.1_5_8_0, 0.7_9_8_4, 4.4_9_0_4, 4.1_1_8_1, 2.0_2_5_2, 4.4_7_8_2, -1.1_2_5_3],
           ]		)

       tf.debugging.assert_near(query_layer[0, 0, :6, :8]     ,      UpperCAmelCase     ,      atol=self.tolerance		)
       tf.debugging.assert_near(key_layer[0, 0, :6, :8]     ,      UpperCAmelCase     ,      atol=self.tolerance		)



