'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import (
       TFRoFormerForCausalLM,
       TFRoFormerForMaskedLM,
       TFRoFormerForMultipleChoice,
       TFRoFormerForQuestionAnswering,
       TFRoFormerForSequenceClassification,
       TFRoFormerForTokenClassification,
       TFRoFormerModel,
   )
   from transformers.models.roformer.modeling_tf_roformer import (
       TFRoFormerSelfAttention,
       TFRoFormerSinusoidalPositionalEmbedding,
   )







class 	SCREAMING_SNAKE_CASE	:




       def __init__(					self      : Union[str, Any]	,    A__      : int	,    A__      : Any=13	,    A__      : int=7	,    A__      : int=True	,    A__      : int=True	,    A__      : Tuple=True	,    A__      : Optional[int]=True	,    A__      : Optional[Any]=99	,    A__      : Optional[int]=32	,    A__      : Any=2	,    A__      : Dict=4	,    A__      : List[Any]=37	,    A__      : Any="gelu"	,    A__      : Tuple=0.1	,    A__      : Optional[Any]=0.1	,    A__      : Union[str, Any]=512	,    A__      : Optional[Any]=16	,    A__      : List[Any]=2	,    A__      : str=0.02	,    A__      : int=3	,    A__      : Any=4	,    A__      : Optional[int]=None	,    ):


            """simple docstring"""







            __lowerCamelCase						:       Union[str, Any]				   =				parent
            __lowerCamelCase						:       Any				   =				13
            __lowerCamelCase						:       List[str]				   =				7
            __lowerCamelCase						:       List[Any]				   =				True
            __lowerCamelCase						:       Optional[Any]				   =				True
            __lowerCamelCase						:       Any				   =				True
            __lowerCamelCase						:       List[Any]				   =				True
            __lowerCamelCase						:       Optional[int]				   =				99
            __lowerCamelCase						:       List[str]				   =				32
            __lowerCamelCase						:       int				   =				2
            __lowerCamelCase						:       Optional[int]				   =				4
            __lowerCamelCase						:       List[str]				   =				37
            __lowerCamelCase						:       List[str]				   =				"""gelu"""
            __lowerCamelCase						:       List[Any]				   =				0.1
            __lowerCamelCase						:       int				   =				0.1
            __lowerCamelCase						:       Tuple				   =				512
            __lowerCamelCase						:       str				   =				16
            __lowerCamelCase						:       List[str]				   =				2
            __lowerCamelCase						:       Any				   =				0.02
            __lowerCamelCase						:       Union[str, Any]				   =				3
            __lowerCamelCase						:       Optional[Any]				   =				4
            __lowerCamelCase						:       Union[str, Any]				   =				None




       def 					a_					(					self      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Tuple				   =				ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size					)

            __lowerCamelCase						:       Dict				   =				None
            if self.use_input_mask:
                 __lowerCamelCase						:       List[str]				   =				random_attention_mask([self.batch_size, self.seq_length]					)

            __lowerCamelCase						:       Optional[Any]				   =				None
            if self.use_token_type_ids:
                 __lowerCamelCase						:       Dict				   =				ids_tensor([self.batch_size, self.seq_length]	,    self.type_vocab_size					)

            __lowerCamelCase						:       int				   =				None
            __lowerCamelCase						:       List[str]				   =				None
            __lowerCamelCase						:       Tuple				   =				None
            if self.use_labels:
                 __lowerCamelCase						:       List[str]				   =				ids_tensor([self.batch_size]	,    self.type_sequence_label_size					)
                 __lowerCamelCase						:       Optional[Any]				   =				ids_tensor([self.batch_size, self.seq_length]	,    self.num_labels					)
                 __lowerCamelCase						:       Union[str, Any]				   =				ids_tensor([self.batch_size]	,    self.num_choices					)

            __lowerCamelCase						:       List[str]				   =				RoFormerConfig(
                vocab_size=self.vocab_size	,    hidden_size=self.hidden_size	,    num_hidden_layers=self.num_hidden_layers	,    num_attention_heads=self.num_attention_heads	,    intermediate_size=self.intermediate_size	,    hidden_act=self.hidden_act	,    hidden_dropout_prob=self.hidden_dropout_prob	,    attention_probs_dropout_prob=self.attention_probs_dropout_prob	,    max_position_embeddings=self.max_position_embeddings	,    type_vocab_size=self.type_vocab_size	,    initializer_range=self.initializer_range	,    return_dict=A__	,    )

            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




       def 					a_					(					self      : Optional[int]	,    A__      : List[Any]	,    A__      : List[str]	,    A__      : int	,    A__      : Tuple	,    A__      : int	,    A__      : Optional[int]	,    A__      : Union[str, Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				TFRoFormerModel(config=A__					)
            __lowerCamelCase						:       int				   =				{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

            __lowerCamelCase						:       Any				   =				[input_ids, input_mask]
            __lowerCamelCase						:       Optional[int]				   =				model(A__					)

            __lowerCamelCase						:       Any				   =				model(A__					)

            self.parent.assertEqual(result.last_hidden_state.shape	,    (self.batch_size, self.seq_length, self.hidden_size)					)




       def 					a_					(					self      : Any	,    A__      : List[Any]	,    A__      : List[str]	,    A__      : Optional[Any]	,    A__      : List[Any]	,    A__      : str	,    A__      : Optional[Any]	,    A__      : Tuple					):


            """simple docstring"""







            __lowerCamelCase						:       str				   =				True
            __lowerCamelCase						:       Dict				   =				TFRoFormerForCausalLM(config=A__					)
            __lowerCamelCase						:       Dict				   =				{
                """input_ids""": input_ids,
                """attention_mask""": input_mask,
                """token_type_ids""": token_type_ids,
            }
            __lowerCamelCase						:       List[Any]				   =				model(A__					)["""logits"""]
            self.parent.assertListEqual(
                list(prediction_scores.numpy().shape					)	,    [self.batch_size, self.seq_length, self.vocab_size]					)




       def 					a_					(					self      : str	,    A__      : Optional[Any]	,    A__      : str	,    A__      : Any	,    A__      : Dict	,    A__      : Optional[Any]	,    A__      : Optional[Any]	,    A__      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				TFRoFormerForMaskedLM(config=A__					)
            __lowerCamelCase						:       Union[str, Any]				   =				{
                """input_ids""": input_ids,
                """attention_mask""": input_mask,
                """token_type_ids""": token_type_ids,
            }
            __lowerCamelCase						:       List[Any]				   =				model(A__					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.vocab_size)					)




       def 					a_					(					self      : List[str]	,    A__      : int	,    A__      : str	,    A__      : Optional[int]	,    A__      : List[Any]	,    A__      : Optional[Any]	,    A__      : Any	,    A__      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				self.num_labels
            __lowerCamelCase						:       Union[str, Any]				   =				TFRoFormerForSequenceClassification(config=A__					)
            __lowerCamelCase						:       List[Any]				   =				{
                """input_ids""": input_ids,
                """attention_mask""": input_mask,
                """token_type_ids""": token_type_ids,
            }

            __lowerCamelCase						:       Union[str, Any]				   =				model(A__					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_labels)					)




       def 					a_					(					self      : Tuple	,    A__      : int	,    A__      : int	,    A__      : int	,    A__      : Optional[Any]	,    A__      : int	,    A__      : int	,    A__      : Tuple					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				self.num_choices
            __lowerCamelCase						:       int				   =				TFRoFormerForMultipleChoice(config=A__					)
            __lowerCamelCase						:       List[str]				   =				tf.tile(tf.expand_dims(A__	,    1					)	,    (1, self.num_choices, 1)					)
            __lowerCamelCase						:       List[str]				   =				tf.tile(tf.expand_dims(A__	,    1					)	,    (1, self.num_choices, 1)					)
            __lowerCamelCase						:       int				   =				tf.tile(tf.expand_dims(A__	,    1					)	,    (1, self.num_choices, 1)					)
            __lowerCamelCase						:       Any				   =				{
                """input_ids""": multiple_choice_inputs_ids,
                """attention_mask""": multiple_choice_input_mask,
                """token_type_ids""": multiple_choice_token_type_ids,
            }
            __lowerCamelCase						:       Tuple				   =				model(A__					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_choices)					)




       def 					a_					(					self      : Any	,    A__      : Dict	,    A__      : Optional[Any]	,    A__      : str	,    A__      : Tuple	,    A__      : List[Any]	,    A__      : Union[str, Any]	,    A__      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				self.num_labels
            __lowerCamelCase						:       Optional[int]				   =				TFRoFormerForTokenClassification(config=A__					)
            __lowerCamelCase						:       Union[str, Any]				   =				{
                """input_ids""": input_ids,
                """attention_mask""": input_mask,
                """token_type_ids""": token_type_ids,
            }
            __lowerCamelCase						:       str				   =				model(A__					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.num_labels)					)




       def 					a_					(					self      : Tuple	,    A__      : Optional[int]	,    A__      : Tuple	,    A__      : Tuple	,    A__      : Dict	,    A__      : List[str]	,    A__      : Any	,    A__      : Optional[Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Union[str, Any]				   =				TFRoFormerForQuestionAnswering(config=A__					)
            __lowerCamelCase						:       Tuple				   =				{
                """input_ids""": input_ids,
                """attention_mask""": input_mask,
                """token_type_ids""": token_type_ids,
            }

            __lowerCamelCase						:       Optional[Any]				   =				model(A__					)
            self.parent.assertEqual(result.start_logits.shape	,    (self.batch_size, self.seq_length)					)
            self.parent.assertEqual(result.end_logits.shape	,    (self.batch_size, self.seq_length)					)




       def 					a_					(					self      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				self.prepare_config_and_inputs()
            (
                (
                __lowerCamelCase		
            )   ,      (
                __lowerCamelCase		
            )   ,      (
                __lowerCamelCase		
            )   ,      (
                __lowerCamelCase		
            )   ,      (
                __lowerCamelCase		
            )   ,      (
                __lowerCamelCase		
            )   ,      (
                __lowerCamelCase		
            )   ,      		
            )						:       int				   =				config_and_inputs
            __lowerCamelCase						:       int				   =				{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
            return config, inputs_dict







@require_tf
class 	SCREAMING_SNAKE_CASE	(      __a    ,   __a    ,   unittest.TestCase  ):
       snake_case__    :     Optional[int]   		=			(
           (
               TFRoFormerModel,
               TFRoFormerForCausalLM,
               TFRoFormerForMaskedLM,
               TFRoFormerForQuestionAnswering,
               TFRoFormerForSequenceClassification,
               TFRoFormerForTokenClassification,
               TFRoFormerForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       snake_case__    :     Optional[int]   		=			(
           {
               'feature-extraction': TFRoFormerModel,
               'fill-mask': TFRoFormerForMaskedLM,
               'question-answering': TFRoFormerForQuestionAnswering,
               'text-classification': TFRoFormerForSequenceClassification,
               'text-generation': TFRoFormerForCausalLM,
               'token-classification': TFRoFormerForTokenClassification,
               'zero-shot': TFRoFormerForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )

       snake_case__    :     Optional[int]   		=			False
       snake_case__    :     List[str]   		=			False




       def 					a_					(					self      : Dict	,    A__      : Tuple	,    A__      : Optional[int]	,    A__      : Any	,    A__      : List[str]	,    A__      : Optional[int]					):


            """simple docstring"""







            if pipeline_test_casse_name == "TextGenerationPipelineTests":
                 return True

            return False




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Tuple				   =				TFRoFormerModelTester(self					)
            __lowerCamelCase						:       Optional[int]				   =				ConfigTester(self	,    config_class=A__	,    hidden_size=37					)




       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            self.config_tester.run_common_tests()




       def 					a_					(					self      : Any					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*A__					)




       def 					a_					(					self      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Union[str, Any]				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*A__					)




       def 					a_					(					self      : Tuple					):


            """simple docstring"""







            __lowerCamelCase						:       Union[str, Any]				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_lm_head(*A__					)




       def 					a_					(					self      : Any					):


            """simple docstring"""







            __lowerCamelCase						:       str				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_multiple_choice(*A__					)




       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_question_answering(*A__					)




       def 					a_					(					self      : int					):


            """simple docstring"""







            __lowerCamelCase						:       List[Any]				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_sequence_classification(*A__					)




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*A__					)




       @slow
       def 					a_					(					self      : Tuple					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[int]				   =				TFRoFormerModel.from_pretrained("""junnyu/roformer_chinese_base"""					)
            self.assertIsNotNone(A__					)







@require_tf
class 	SCREAMING_SNAKE_CASE	(      unittest.TestCase  ):




       @slow
       def 					a_					(					self      : int					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				TFRoFormerForMaskedLM.from_pretrained("""junnyu/roformer_chinese_base"""					)
            __lowerCamelCase						:       List[Any]				   =				tf.constant([[0, 1, 2, 3, 4, 5]]					)
            __lowerCamelCase						:       int				   =				model(A__					)[0]

            # TODO Replace vocab size
            __lowerCamelCase						:       Dict				   =				50000

            __lowerCamelCase						:       Any				   =				[1, 6, vocab_size]
            self.assertEqual(output.shape	,    A__					)

            print(output[:, :3, :3]					)

            # TODO Replace values below with what was printed above.
            __lowerCamelCase						:       str				   =				tf.constant(
                [
                    [
                        [-0.1205_3341, -1.026_4901, 0.2922_1946],
                        [-1.513_3783, 0.19_7433, 0.1519_0607],
                        [-5.013_5403, -3.90_0256, -0.8403_8764],
                    ]
                ]					)
            tf.debugging.assert_near(output[:, :3, :3]	,    A__	,    atol=1e-4					)







@require_tf
class 	SCREAMING_SNAKE_CASE	(      unittest.TestCase  ):
       snake_case__    :     int   		=			1E-4




       def 					a_					(					self      : int					):


            """simple docstring"""







            __lowerCamelCase						:       Dict				   =				tf.constant([[4, 10]]					)
            __lowerCamelCase						:       int				   =				TFRoFormerSinusoidalPositionalEmbedding(num_positions=6	,    embedding_dim=6					)

            __lowerCamelCase						:       Optional[Any]				   =				emba(input_ids.shape					)
            __lowerCamelCase						:       Dict				   =				tf.constant(
                [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]					)

            tf.debugging.assert_near(A__	,    A__	,    atol=self.tolerance					)




       def 					a_					(					self      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				tf.constant(
                [
                    [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
                    [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
                    [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
                ]					)
            __lowerCamelCase						:       List[Any]				   =				TFRoFormerSinusoidalPositionalEmbedding(num_positions=512	,    embedding_dim=512					)
            emba([2, 16, 512]					)
            __lowerCamelCase						:       List[Any]				   =				emba.weight[:3, :5]

            tf.debugging.assert_near(A__	,    A__	,    atol=self.tolerance					)







@require_tf
class 	SCREAMING_SNAKE_CASE	(      unittest.TestCase  ):
       snake_case__    :     Tuple   		=			1E-4




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				tf.reshape(tf.range(2 * 12 * 16 * 64	,    dtype=tf.floataa					)	,    shape=(2, 12, 16, 64)					) / 100

            __lowerCamelCase						:       Dict				   =				-tf.reshape(tf.range(2 * 12 * 16 * 64	,    dtype=tf.floataa					)	,    shape=(2, 12, 16, 64)					) / 100

            __lowerCamelCase						:       List[Any]				   =				TFRoFormerSinusoidalPositionalEmbedding(num_positions=32	,    embedding_dim=64					)
            __lowerCamelCase						:       str				   =				embed_positions([2, 16, 768]					)[None, None, :, :]

            __lowerCamelCase   ,      __lowerCamelCase						:       str				   =				TFRoFormerSelfAttention.apply_rotary_position_embeddings(
                A__	,    A__	,    A__					)

            __lowerCamelCase						:       Optional[Any]				   =				tf.constant(
                [
                    [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
                    [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
                    [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
                    [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
                    [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
                    [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
                ]					)
            __lowerCamelCase						:       Tuple				   =				tf.constant(
                [
                    [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
                    [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
                    [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
                    [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
                    [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
                    [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
                ]					)

            tf.debugging.assert_near(query_layer[0, 0, :6, :8]	,    A__	,    atol=self.tolerance					)
            tf.debugging.assert_near(key_layer[0, 0, :6, :8]	,    A__	,    atol=self.tolerance					)











'''simple docstring'''

import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__						:List[str]          =			get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class 	SCREAMING_SNAKE_CASE	(      lowerCAmelCase_    ,   unittest.TestCase  ):
       snake_case__    :     Optional[Any]   		=			XLMRobertaTokenizer
       snake_case__    :     str   		=			XLMRobertaTokenizerFast
       snake_case__    :     str   		=			True
       snake_case__    :     int   		=			True




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            super().setUp()

            # We have a SentencePiece fixture for testing
            __lowerCamelCase						:       str				   =				XLMRobertaTokenizer(A__	,    keep_accents=A__					)
            tokenizer.save_pretrained(self.tmpdirname					)




       def 					a_					(					self      : int					):


            """simple docstring"""







            __lowerCamelCase						:       List[Any]				   =				"""<pad>"""
            __lowerCamelCase						:       List[Any]				   =				1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(A__					)	,    A__					)
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(A__					)	,    A__					)




       def 					a_					(					self      : str					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				list(self.get_tokenizer().get_vocab().keys()					)

            self.assertEqual(vocab_keys[0]	,    """<s>"""					)
            self.assertEqual(vocab_keys[1]	,    """<pad>"""					)
            self.assertEqual(vocab_keys[-1]	,    """<mask>"""					)
            self.assertEqual(len(A__					)	,    1002					)




       def 					a_					(					self      : List[Any]					):


            """simple docstring"""







            self.assertEqual(self.get_tokenizer().vocab_size	,    1002					)




       def 					a_					(					self      : Tuple					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				XLMRobertaTokenizer(A__	,    keep_accents=A__					)

            __lowerCamelCase						:       List[str]				   =				tokenizer.tokenize("""This is a test"""					)
            self.assertListEqual(A__	,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(A__					)	,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,    )

            __lowerCamelCase						:       Any				   =				tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
            self.assertListEqual(
                A__	,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]	,    )
            __lowerCamelCase						:       List[str]				   =				tokenizer.convert_tokens_to_ids(A__					)
            self.assertListEqual(
                A__	,    [
                    value + tokenizer.fairseq_offset
                    for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                    #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
                ]	,    )

            __lowerCamelCase						:       Dict				   =				tokenizer.convert_ids_to_tokens(A__					)
            self.assertListEqual(
                A__	,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]	,    )




       def 					a_					(					self      : Any					):


            """simple docstring"""







            if not self.test_slow_tokenizer:
                 # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                 return

            __lowerCamelCase						:       Tuple				   =				(self.rust_tokenizer_class, """hf-internal-testing/tiny-xlm-roberta""", {})
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                      __lowerCamelCase						:       List[Any]				   =				self.rust_tokenizer_class.from_pretrained(A__	,    **A__					)
                      __lowerCamelCase						:       Dict				   =				self.tokenizer_class.from_pretrained(A__	,    **A__					)

                      __lowerCamelCase						:       str				   =				tempfile.mkdtemp()

                      __lowerCamelCase						:       Union[str, Any]				   =				tokenizer_r.save_pretrained(A__					)
                      __lowerCamelCase						:       Union[str, Any]				   =				tokenizer_p.save_pretrained(A__					)

                      # Checks it save with the same files + the tokenizer.json file for the fast one
                      self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files					)					)
                      __lowerCamelCase						:       Any				   =				tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f					)
                      self.assertSequenceEqual(A__	,    A__					)

                      # Checks everything loads correctly in the same way
                      __lowerCamelCase						:       str				   =				tokenizer_r.from_pretrained(A__					)
                      __lowerCamelCase						:       str				   =				tokenizer_p.from_pretrained(A__					)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(A__	,    A__					)					)
                           # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                           # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                      shutil.rmtree(A__					)

                      # Save tokenizer rust, legacy_format=True
                      __lowerCamelCase						:       Optional[Any]				   =				tempfile.mkdtemp()

                      __lowerCamelCase						:       Dict				   =				tokenizer_r.save_pretrained(A__	,    legacy_format=A__					)
                      __lowerCamelCase						:       List[Any]				   =				tokenizer_p.save_pretrained(A__					)

                      # Checks it save with the same files
                      self.assertSequenceEqual(A__	,    A__					)

                      # Checks everything loads correctly in the same way
                      __lowerCamelCase						:       Any				   =				tokenizer_r.from_pretrained(A__					)
                      __lowerCamelCase						:       List[str]				   =				tokenizer_p.from_pretrained(A__					)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(A__	,    A__					)					)

                      shutil.rmtree(A__					)

                      # Save tokenizer rust, legacy_format=False
                      __lowerCamelCase						:       int				   =				tempfile.mkdtemp()

                      __lowerCamelCase						:       Any				   =				tokenizer_r.save_pretrained(A__	,    legacy_format=A__					)
                      __lowerCamelCase						:       List[str]				   =				tokenizer_p.save_pretrained(A__					)

                      # Checks it saved the tokenizer.json file
                      self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files					)					)

                      # Checks everything loads correctly in the same way
                      __lowerCamelCase						:       List[Any]				   =				tokenizer_r.from_pretrained(A__					)
                      __lowerCamelCase						:       Optional[int]				   =				tokenizer_p.from_pretrained(A__					)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(A__	,    A__					)					)

                      shutil.rmtree(A__					)




       @cached_property
       def 					a_					(					self      : str					):


            """simple docstring"""







            return XLMRobertaTokenizer.from_pretrained("""xlm-roberta-base"""					)




       def 					a_					(					self      : int					):


            """simple docstring"""







            with tempfile.NamedTemporaryFile() as f:
                 shutil.copyfile(A__	,    f.name					)
                 __lowerCamelCase						:       int				   =				XLMRobertaTokenizer(f.name	,    keep_accents=A__					)
                 __lowerCamelCase						:       str				   =				pickle.dumps(A__					)
            pickle.loads(A__					)




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            if not self.test_rust_tokenizer:
                 return

            __lowerCamelCase						:       Union[str, Any]				   =				self.get_tokenizer()
            __lowerCamelCase						:       Tuple				   =				self.get_rust_tokenizer()

            __lowerCamelCase						:       Optional[Any]				   =				"""I was born in 92000, and this is falsé."""

            __lowerCamelCase						:       int				   =				tokenizer.tokenize(A__					)
            __lowerCamelCase						:       Optional[Any]				   =				rust_tokenizer.tokenize(A__					)
            self.assertListEqual(A__	,    A__					)

            __lowerCamelCase						:       Any				   =				tokenizer.encode(A__	,    add_special_tokens=A__					)
            __lowerCamelCase						:       Dict				   =				rust_tokenizer.encode(A__	,    add_special_tokens=A__					)
            self.assertListEqual(A__	,    A__					)

            __lowerCamelCase						:       Optional[Any]				   =				self.get_rust_tokenizer()
            __lowerCamelCase						:       Optional[int]				   =				tokenizer.encode(A__					)
            __lowerCamelCase						:       str				   =				rust_tokenizer.encode(A__					)
            self.assertListEqual(A__	,    A__					)




       @slow
       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            __lowerCamelCase						:       List[Any]				   =				"""Hello World!"""
            __lowerCamelCase						:       str				   =				[0, 35378, 6661, 38, 2]
            # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
            # xlmr.eval()
            # xlmr.encode(symbols)

            self.assertListEqual(A__	,    self.big_tokenizer.encode(A__					)					)




       @slow
       def 					a_					(					self      : Optional[Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				(
                """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
                """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
            )
            __lowerCamelCase						:       Optional[Any]				   =				[
                0,
                3293,
                83,
                10,
                4552,
                4989,
                7986,
                678,
                10,
                5915,
                111,
                179459,
                124850,
                4,
                6044,
                237,
                12,
                6,
                5,
                6,
                4,
                6780,
                705,
                15,
                1388,
                44,
                378,
                10114,
                711,
                152,
                20,
                6,
                5,
                22376,
                642,
                1221,
                15190,
                34153,
                450,
                5608,
                959,
                1119,
                57702,
                136,
                186,
                47,
                1098,
                29367,
                47,
                # 4426, # What fairseq tokenizes from "<unk>": "_<"
                # 3678, # What fairseq tokenizes from "<unk>": "unk"
                # 2740, # What fairseq tokenizes from "<unk>": ">"
                3,  # What we tokenize from "<unk>": "<unk>"
                6,  # Residue from the tokenization: an extra sentencepiece underline
                4,
                6044,
                237,
                6284,
                50901,
                528,
                31,
                90,
                34,
                927,
                2,
            ]
            # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
            # xlmr.eval()
            # xlmr.encode(symbols)

            self.assertListEqual(A__	,    self.big_tokenizer.encode(A__					)					)




       @slow
       def 					a_					(					self      : Any					):


            """simple docstring"""







            __lowerCamelCase						:       Tuple				   =				{"""input_ids""": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=A__	,    model_name="""xlm-roberta-base"""	,    revision="""d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"""	,    )



