



'''simple docstring'''

import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       FlaubertForMultipleChoice,
       FlaubertForQuestionAnswering,
       FlaubertForQuestionAnsweringSimple,
       FlaubertForSequenceClassification,
       FlaubertForTokenClassification,
       FlaubertModel,
       FlaubertWithLMHeadModel,
   )
   from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 	SCREAMING_SNAKE_CASE	(      __UpperCAmelCase  ):




       def __init__(					self      : Optional[int]	,    A__      : List[str]	,    A__      : Optional[int]=13	,    A__      : Any=7	,    A__      : Any=True	,    A__      : Any=True	,    A__      : List[Any]=True	,    A__      : int=True	,    A__      : int=True	,    A__      : List[str]=False	,    A__      : Union[str, Any]=False	,    A__      : int=False	,    A__      : Optional[int]=2	,    A__      : Tuple=99	,    A__      : Dict=0	,    A__      : str=32	,    A__      : Optional[int]=5	,    A__      : List[Any]=4	,    A__      : Optional[Any]=0.1	,    A__      : Any=0.1	,    A__      : Dict=512	,    A__      : Optional[Any]=12	,    A__      : List[Any]=2	,    A__      : Optional[Any]=0.02	,    A__      : Union[str, Any]=3	,    A__      : Tuple=4	,    A__      : List[Any]="last"	,    A__      : Tuple=None	,    A__      : Any=None	,    ):


            """simple docstring"""







            __lowerCamelCase						:       Optional[int]				   =				parent
            __lowerCamelCase						:       Optional[Any]				   =				batch_size
            __lowerCamelCase						:       Dict				   =				seq_length
            __lowerCamelCase						:       List[Any]				   =				is_training
            __lowerCamelCase						:       Tuple				   =				use_input_lengths
            __lowerCamelCase						:       List[str]				   =				use_token_type_ids
            __lowerCamelCase						:       Optional[Any]				   =				use_labels
            __lowerCamelCase						:       Any				   =				gelu_activation
            __lowerCamelCase						:       Any				   =				sinusoidal_embeddings
            __lowerCamelCase						:       Union[str, Any]				   =				causal
            __lowerCamelCase						:       List[str]				   =				asm
            __lowerCamelCase						:       int				   =				n_langs
            __lowerCamelCase						:       int				   =				vocab_size
            __lowerCamelCase						:       Union[str, Any]				   =				n_special
            __lowerCamelCase						:       Dict				   =				hidden_size
            __lowerCamelCase						:       Dict				   =				num_hidden_layers
            __lowerCamelCase						:       List[str]				   =				num_attention_heads
            __lowerCamelCase						:       str				   =				hidden_dropout_prob
            __lowerCamelCase						:       Optional[Any]				   =				attention_probs_dropout_prob
            __lowerCamelCase						:       List[str]				   =				max_position_embeddings
            __lowerCamelCase						:       List[str]				   =				type_vocab_size
            __lowerCamelCase						:       Any				   =				type_sequence_label_size
            __lowerCamelCase						:       Optional[Any]				   =				initializer_range
            __lowerCamelCase						:       int				   =				num_labels
            __lowerCamelCase						:       Optional[Any]				   =				num_choices
            __lowerCamelCase						:       List[Any]				   =				summary_type
            __lowerCamelCase						:       Dict				   =				use_proj
            __lowerCamelCase						:       Optional[int]				   =				scope




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size					)
            __lowerCamelCase						:       Union[str, Any]				   =				random_attention_mask([self.batch_size, self.seq_length]					)

            __lowerCamelCase						:       List[str]				   =				None
            if self.use_input_lengths:
                 __lowerCamelCase						:       Optional[Any]				   =				(
                     ids_tensor([self.batch_size]	,    vocab_size=2					) + self.seq_length - 2
                 )  # small variation of seq_length

            __lowerCamelCase						:       Dict				   =				None
            if self.use_token_type_ids:
                 __lowerCamelCase						:       Tuple				   =				ids_tensor([self.batch_size, self.seq_length]	,    self.n_langs					)

            __lowerCamelCase						:       List[Any]				   =				None
            __lowerCamelCase						:       Tuple				   =				None
            __lowerCamelCase						:       Tuple				   =				None
            if self.use_labels:
                 __lowerCamelCase						:       Optional[Any]				   =				ids_tensor([self.batch_size]	,    self.type_sequence_label_size					)
                 __lowerCamelCase						:       Union[str, Any]				   =				ids_tensor([self.batch_size, self.seq_length]	,    self.num_labels					)
                 __lowerCamelCase						:       Optional[Any]				   =				ids_tensor([self.batch_size]	,    2					).float()
                 __lowerCamelCase						:       Dict				   =				ids_tensor([self.batch_size]	,    self.num_choices					)

            __lowerCamelCase						:       str				   =				self.get_config()

            return (
                config,
                input_ids,
                token_type_ids,
                input_lengths,
                sequence_labels,
                token_labels,
                is_impossible_labels,
                choice_labels,
                input_mask,
            )




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            return FlaubertConfig(
                vocab_size=self.vocab_size	,    n_special=self.n_special	,    emb_dim=self.hidden_size	,    n_layers=self.num_hidden_layers	,    n_heads=self.num_attention_heads	,    dropout=self.hidden_dropout_prob	,    attention_dropout=self.attention_probs_dropout_prob	,    gelu_activation=self.gelu_activation	,    sinusoidal_embeddings=self.sinusoidal_embeddings	,    asm=self.asm	,    causal=self.causal	,    n_langs=self.n_langs	,    max_position_embeddings=self.max_position_embeddings	,    initializer_range=self.initializer_range	,    summary_type=self.summary_type	,    use_proj=self.use_proj	,    )




       def 					a_					(					self      : int	,    A__      : Dict	,    A__      : str	,    A__      : List[str]	,    A__      : Any	,    A__      : List[str]	,    A__      : Any	,    A__      : Optional[int]	,    A__      : Tuple	,    A__      : Tuple	,    ):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				FlaubertModel(config=__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()
            __lowerCamelCase						:       int				   =				model(__SCREAMING_SNAKE_CASE	,    lengths=__SCREAMING_SNAKE_CASE	,    langs=__SCREAMING_SNAKE_CASE					)
            __lowerCamelCase						:       Optional[int]				   =				model(__SCREAMING_SNAKE_CASE	,    langs=__SCREAMING_SNAKE_CASE					)
            __lowerCamelCase						:       Optional[Any]				   =				model(__SCREAMING_SNAKE_CASE					)
            self.parent.assertEqual(result.last_hidden_state.shape	,    (self.batch_size, self.seq_length, self.hidden_size)					)




       def 					a_					(					self      : Any	,    A__      : Tuple	,    A__      : List[str]	,    A__      : Optional[int]	,    A__      : Any	,    A__      : str	,    A__      : Optional[Any]	,    A__      : str	,    A__      : Any	,    A__      : List[str]	,    ):


            """simple docstring"""







            __lowerCamelCase						:       str				   =				FlaubertWithLMHeadModel(__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()

            __lowerCamelCase						:       Optional[Any]				   =				model(__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE					)
            self.parent.assertEqual(result.loss.shape	,    ()					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.vocab_size)					)




       def 					a_					(					self      : int	,    A__      : List[str]	,    A__      : str	,    A__      : Union[str, Any]	,    A__      : str	,    A__      : Union[str, Any]	,    A__      : Optional[Any]	,    A__      : Optional[Any]	,    A__      : List[Any]	,    A__      : List[Any]	,    ):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				FlaubertForQuestionAnsweringSimple(__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()

            __lowerCamelCase						:       Dict				   =				model(__SCREAMING_SNAKE_CASE					)

            __lowerCamelCase						:       Tuple				   =				model(__SCREAMING_SNAKE_CASE	,    start_positions=__SCREAMING_SNAKE_CASE	,    end_positions=__SCREAMING_SNAKE_CASE					)
            self.parent.assertEqual(result.start_logits.shape	,    (self.batch_size, self.seq_length)					)
            self.parent.assertEqual(result.end_logits.shape	,    (self.batch_size, self.seq_length)					)




       def 					a_					(					self      : Union[str, Any]	,    A__      : str	,    A__      : Optional[Any]	,    A__      : Optional[int]	,    A__      : Optional[int]	,    A__      : Union[str, Any]	,    A__      : Union[str, Any]	,    A__      : Optional[int]	,    A__      : str	,    A__      : Tuple	,    ):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				FlaubertForQuestionAnswering(__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()

            __lowerCamelCase						:       Tuple				   =				model(__SCREAMING_SNAKE_CASE					)

            __lowerCamelCase						:       Tuple				   =				model(
                __SCREAMING_SNAKE_CASE	,    start_positions=__SCREAMING_SNAKE_CASE	,    end_positions=__SCREAMING_SNAKE_CASE	,    cls_index=__SCREAMING_SNAKE_CASE	,    is_impossible=__SCREAMING_SNAKE_CASE	,    p_mask=__SCREAMING_SNAKE_CASE	,    )

            __lowerCamelCase						:       Any				   =				model(
                __SCREAMING_SNAKE_CASE	,    start_positions=__SCREAMING_SNAKE_CASE	,    end_positions=__SCREAMING_SNAKE_CASE	,    cls_index=__SCREAMING_SNAKE_CASE	,    is_impossible=__SCREAMING_SNAKE_CASE	,    )

            ((__lowerCamelCase     )   ,           )						:       Any				   =				result_with_labels.to_tuple()

            __lowerCamelCase						:       int				   =				model(__SCREAMING_SNAKE_CASE	,    start_positions=__SCREAMING_SNAKE_CASE	,    end_positions=__SCREAMING_SNAKE_CASE					)

            ((__lowerCamelCase     )   ,           )						:       Tuple				   =				result_with_labels.to_tuple()

            self.parent.assertEqual(result_with_labels.loss.shape	,    ()					)
            self.parent.assertEqual(result.start_top_log_probs.shape	,    (self.batch_size, model.config.start_n_top)					)
            self.parent.assertEqual(result.start_top_index.shape	,    (self.batch_size, model.config.start_n_top)					)
            self.parent.assertEqual(
                result.end_top_log_probs.shape	,    (self.batch_size, model.config.start_n_top * model.config.end_n_top)					)
            self.parent.assertEqual(
                result.end_top_index.shape	,    (self.batch_size, model.config.start_n_top * model.config.end_n_top)					)
            self.parent.assertEqual(result.cls_logits.shape	,    (self.batch_size,)					)




       def 					a_					(					self      : int	,    A__      : Any	,    A__      : str	,    A__      : List[str]	,    A__      : Optional[int]	,    A__      : str	,    A__      : Optional[int]	,    A__      : str	,    A__      : Dict	,    A__      : Optional[Any]	,    ):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				FlaubertForSequenceClassification(__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()

            __lowerCamelCase						:       Tuple				   =				model(__SCREAMING_SNAKE_CASE					)
            __lowerCamelCase						:       Any				   =				model(__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE					)

            self.parent.assertEqual(result.loss.shape	,    ()					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.type_sequence_label_size)					)




       def 					a_					(					self      : Optional[int]	,    A__      : Dict	,    A__      : Dict	,    A__      : Dict	,    A__      : int	,    A__      : Dict	,    A__      : str	,    A__      : Union[str, Any]	,    A__      : Optional[Any]	,    A__      : List[Any]	,    ):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				self.num_labels
            __lowerCamelCase						:       List[Any]				   =				FlaubertForTokenClassification(__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()

            __lowerCamelCase						:       Union[str, Any]				   =				model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE					)
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.num_labels)					)




       def 					a_					(					self      : int	,    A__      : int	,    A__      : str	,    A__      : Dict	,    A__      : Union[str, Any]	,    A__      : Dict	,    A__      : List[Any]	,    A__      : Optional[int]	,    A__      : Union[str, Any]	,    A__      : Optional[int]	,    ):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				self.num_choices
            __lowerCamelCase						:       List[str]				   =				FlaubertForMultipleChoice(config=__SCREAMING_SNAKE_CASE					)
            model.to(__SCREAMING_SNAKE_CASE					)
            model.eval()
            __lowerCamelCase						:       List[str]				   =				input_ids.unsqueeze(1					).expand(-1	,    self.num_choices	,    -1					).contiguous()
            __lowerCamelCase						:       Tuple				   =				token_type_ids.unsqueeze(1					).expand(-1	,    self.num_choices	,    -1					).contiguous()
            __lowerCamelCase						:       Union[str, Any]				   =				input_mask.unsqueeze(1					).expand(-1	,    self.num_choices	,    -1					).contiguous()
            __lowerCamelCase						:       Dict				   =				model(
                __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE	,    )
            self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_choices)					)




       def 					a_					(					self      : Tuple					):


            """simple docstring"""







            __lowerCamelCase						:       str				   =				self.prepare_config_and_inputs()
            (
                (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,      (
                __lowerCamelCase     
            )   ,           
            )						:       Optional[Any]				   =				config_and_inputs
            __lowerCamelCase						:       Tuple				   =				{
                """input_ids""": input_ids,
                """token_type_ids""": token_type_ids,
                """lengths""": input_lengths,
                """attention_mask""": input_mask,
            }
            return config, inputs_dict







@require_torch
class 	SCREAMING_SNAKE_CASE	(      __UpperCAmelCase    ,   __UpperCAmelCase    ,   unittest.TestCase  ):
       snake_case__    :     Any   		=			(
           (
               FlaubertModel,
               FlaubertWithLMHeadModel,
               FlaubertForQuestionAnswering,
               FlaubertForQuestionAnsweringSimple,
               FlaubertForSequenceClassification,
               FlaubertForTokenClassification,
               FlaubertForMultipleChoice,
           )
           if is_torch_available()
           else ()
       )
       snake_case__    :     List[Any]   		=			(
           {
               '''feature-extraction''': FlaubertModel,
               '''fill-mask''': FlaubertWithLMHeadModel,
               '''question-answering''': FlaubertForQuestionAnsweringSimple,
               '''text-classification''': FlaubertForSequenceClassification,
               '''token-classification''': FlaubertForTokenClassification,
               '''zero-shot''': FlaubertForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )




       def 					a_					(					self      : int	,    A__      : Tuple	,    A__      : Any	,    A__      : Union[str, Any]	,    A__      : Optional[int]	,    A__      : Union[str, Any]					):


            """simple docstring"""







            if (
                pipeline_test_casse_name == "QAPipelineTests"
                and tokenizer_name is not None
                and not tokenizer_name.endswith("""Fast"""					)
            ):
                 # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                 # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                 # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                 return True

            return False




       def 					a_					(					self      : Any	,    A__      : Any	,    A__      : Optional[Any]	,    A__      : int=False					):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				super()._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    return_labels=__SCREAMING_SNAKE_CASE					)

            if return_labels:
                 if model_class.__name__ == "FlaubertForQuestionAnswering":
                      __lowerCamelCase						:       List[Any]				   =				torch.zeros(
                          self.model_tester.batch_size	,    dtype=torch.long	,    device=__SCREAMING_SNAKE_CASE					)
                      __lowerCamelCase						:       str				   =				torch.zeros(
                          self.model_tester.batch_size	,    dtype=torch.long	,    device=__SCREAMING_SNAKE_CASE					)

            return inputs_dict




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       Dict				   =				FlaubertModelTester(self					)
            __lowerCamelCase						:       Optional[int]				   =				ConfigTester(self	,    config_class=__SCREAMING_SNAKE_CASE	,    emb_dim=37					)




       def 					a_					(					self      : Optional[Any]					):


            """simple docstring"""







            self.config_tester.run_common_tests()




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_model(*__SCREAMING_SNAKE_CASE					)




       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_lm_head(*__SCREAMING_SNAKE_CASE					)




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       List[Any]				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_simple_qa(*__SCREAMING_SNAKE_CASE					)




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_qa(*__SCREAMING_SNAKE_CASE					)




       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            __lowerCamelCase						:       str				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_sequence_classif(*__SCREAMING_SNAKE_CASE					)




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_token_classif(*__SCREAMING_SNAKE_CASE					)




       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            __lowerCamelCase						:       str				   =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_flaubert_multiple_choice(*__SCREAMING_SNAKE_CASE					)




       @slow
       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __lowerCamelCase						:       int				   =				FlaubertModel.from_pretrained(__SCREAMING_SNAKE_CASE					)
                 self.assertIsNotNone(__SCREAMING_SNAKE_CASE					)




       @slow
       @require_torch_gpu
       def 					a_					(					self      : Any					):


            """simple docstring"""







            __lowerCamelCase   ,      __lowerCamelCase						:       Optional[int]				   =				self.model_tester.prepare_config_and_inputs_for_common()
            for model_class in self.all_model_classes:
                 # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
                 if model_class == FlaubertForMultipleChoice:
                      return

                 __lowerCamelCase						:       Tuple				   =				True
                 __lowerCamelCase						:       Union[str, Any]				   =				model_class(config=__SCREAMING_SNAKE_CASE					)

                 __lowerCamelCase						:       Tuple				   =				self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE					)
                 __lowerCamelCase						:       int				   =				torch.jit.trace(
                     __SCREAMING_SNAKE_CASE	,    (inputs_dict["""input_ids"""].to("""cpu"""					), inputs_dict["""attention_mask"""].to("""cpu"""					))					)

                 with tempfile.TemporaryDirectory() as tmp:
                      torch.jit.save(__SCREAMING_SNAKE_CASE	,    os.path.join(__SCREAMING_SNAKE_CASE	,    """traced_model.pt"""					)					)
                      __lowerCamelCase						:       str				   =				torch.jit.load(os.path.join(__SCREAMING_SNAKE_CASE	,    """traced_model.pt"""					)	,    map_location=__SCREAMING_SNAKE_CASE					)
                      loaded(inputs_dict["""input_ids"""].to(__SCREAMING_SNAKE_CASE					)	,    inputs_dict["""attention_mask"""].to(__SCREAMING_SNAKE_CASE					)					)







@require_torch
class 	SCREAMING_SNAKE_CASE	(      unittest.TestCase  ):




       @slow
       def 					a_					(					self      : Optional[Any]					):


            """simple docstring"""







            __lowerCamelCase						:       Any				   =				FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""					)
            __lowerCamelCase						:       str				   =				torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]					)
            with torch.no_grad():
                 __lowerCamelCase						:       str				   =				model(__SCREAMING_SNAKE_CASE					)[0]
            __lowerCamelCase						:       Any				   =				torch.Size((1, 11, 768)					)
            self.assertEqual(output.shape	,    __SCREAMING_SNAKE_CASE					)
            __lowerCamelCase						:       Optional[Any]				   =				torch.tensor(
                [[[-2.6251, -1.4298, -0.0227], [-2.8510, -1.6387, 0.2258], [-2.8114, -1.1832, -0.3066]]]					)

            self.assertTrue(torch.allclose(output[:, :3, :3]	,    __SCREAMING_SNAKE_CASE	,    atol=1e-4					)					)











'''simple docstring'''

import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	SCREAMING_SNAKE_CASE	(      lowerCAmelCase_    ,   unittest.TestCase  ):
       snake_case__    :     str   		=			RobertaTokenizer
       snake_case__    :     Optional[int]   		=			RobertaTokenizerFast
       snake_case__    :     Union[str, Any]   		=			True
       snake_case__    :     Tuple   		=			{'cls_token': '<s>'}




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            __lowerCamelCase						:       Tuple				   =				[
                """l""",
                """o""",
                """w""",
                """e""",
                """r""",
                """s""",
                """t""",
                """i""",
                """d""",
                """n""",
                """\u0120""",
                """\u0120l""",
                """\u0120n""",
                """\u0120lo""",
                """\u0120low""",
                """er""",
                """\u0120lowest""",
                """\u0120newer""",
                """\u0120wider""",
                """<unk>""",
            ]
            __lowerCamelCase						:       Tuple				   =				dict(zip(A__	,    range(len(A__					)					)					)					)
            __lowerCamelCase						:       List[str]				   =				["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
            __lowerCamelCase						:       Tuple				   =				{"""unk_token""": """<unk>"""}

            __lowerCamelCase						:       Optional[int]				   =				os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["""vocab_file"""]					)
            __lowerCamelCase						:       Union[str, Any]				   =				os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["""merges_file"""]					)
            with open(self.vocab_file	,    """w"""	,    encoding="""utf-8"""					) as fp:
                 fp.write(json.dumps(A__					) + """\n"""					)
            with open(self.merges_file	,    """w"""	,    encoding="""utf-8"""					) as fp:
                 fp.write("""\n""".join(A__					)					)




       def 					a_					(					self      : Tuple	,    **A__      : Dict					):


            """simple docstring"""







            kwargs.update(self.special_tokens_map					)
            return self.tokenizer_class.from_pretrained(self.tmpdirname	,    **A__					)




       def 					a_					(					self      : str	,    **A__      : List[Any]					):


            """simple docstring"""







            kwargs.update(self.special_tokens_map					)
            return RobertaTokenizerFast.from_pretrained(self.tmpdirname	,    **A__					)




       def 					a_					(					self      : Tuple	,    A__      : Any					):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				"""lower newer"""
            __lowerCamelCase						:       int				   =				"""lower newer"""
            return input_text, output_text




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            __lowerCamelCase						:       List[str]				   =				self.tokenizer_class(self.vocab_file	,    self.merges_file	,    **self.special_tokens_map					)
            __lowerCamelCase						:       Any				   =				"""lower newer"""
            __lowerCamelCase						:       Union[str, Any]				   =				["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
            __lowerCamelCase						:       Optional[Any]				   =				tokenizer.tokenize(A__					)  # , add_prefix_space=True)
            self.assertListEqual(A__	,    A__					)

            __lowerCamelCase						:       List[str]				   =				tokens + [tokenizer.unk_token]
            __lowerCamelCase						:       str				   =				[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(A__					)	,    A__					)




       def 					a_					(					self      : Dict					):


            """simple docstring"""







            __lowerCamelCase						:       List[Any]				   =				self.get_tokenizer()

            self.assertListEqual(tokenizer.encode("""Hello world!"""	,    add_special_tokens=A__					)	,    [0, 31414, 232, 328, 2]					)
            self.assertListEqual(
                tokenizer.encode("""Hello world! cécé herlolip 418"""	,    add_special_tokens=A__					)	,    [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]	,    )




       @slow
       def 					a_					(					self      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Dict				   =				self.tokenizer_class.from_pretrained("""roberta-base"""					)

            __lowerCamelCase						:       Any				   =				tokenizer.encode("""sequence builders"""	,    add_special_tokens=A__					)
            __lowerCamelCase						:       Dict				   =				tokenizer.encode("""multi-sequence build"""	,    add_special_tokens=A__					)

            __lowerCamelCase						:       Dict				   =				tokenizer.encode(
                """sequence builders"""	,    add_special_tokens=A__	,    add_prefix_space=A__					)
            __lowerCamelCase						:       List[Any]				   =				tokenizer.encode(
                """sequence builders"""	,    """multi-sequence build"""	,    add_special_tokens=A__	,    add_prefix_space=A__					)

            __lowerCamelCase						:       List[str]				   =				tokenizer.build_inputs_with_special_tokens(A__					)
            __lowerCamelCase						:       Tuple				   =				tokenizer.build_inputs_with_special_tokens(A__	,    A__					)

            assert encoded_sentence == encoded_text_from_decode
            assert encoded_pair == encoded_pair_from_decode




       def 					a_					(					self      : List[str]					):


            """simple docstring"""







            __lowerCamelCase						:       Optional[Any]				   =				self.get_tokenizer()

            __lowerCamelCase						:       str				   =				"""Encode this sequence."""
            __lowerCamelCase						:       str				   =				tokenizer.byte_encoder[""" """.encode("""utf-8"""					)[0]]

            # Testing encoder arguments
            __lowerCamelCase						:       Union[str, Any]				   =				tokenizer.encode(A__	,    add_special_tokens=A__	,    add_prefix_space=A__					)
            __lowerCamelCase						:       List[str]				   =				tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
            self.assertNotEqual(A__	,    A__					)

            __lowerCamelCase						:       Any				   =				tokenizer.encode(A__	,    add_special_tokens=A__	,    add_prefix_space=A__					)
            __lowerCamelCase						:       int				   =				tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
            self.assertEqual(A__	,    A__					)

            tokenizer.add_special_tokens({"""bos_token""": """<s>"""}					)
            __lowerCamelCase						:       Tuple				   =				tokenizer.encode(A__	,    add_special_tokens=A__					)
            __lowerCamelCase						:       List[str]				   =				tokenizer.convert_ids_to_tokens(encoded[1]					)[0]
            self.assertNotEqual(A__	,    A__					)

            # Testing spaces after special tokens
            __lowerCamelCase						:       str				   =				"""<mask>"""
            tokenizer.add_special_tokens(
                {"""mask_token""": AddedToken(A__	,    lstrip=A__	,    rstrip=A__					)}					)  # mask token has a left space
            __lowerCamelCase						:       str				   =				tokenizer.convert_tokens_to_ids(A__					)

            __lowerCamelCase						:       str				   =				"""Encode <mask> sequence"""
            __lowerCamelCase						:       Optional[int]				   =				"""Encode <mask>sequence"""

            __lowerCamelCase						:       Optional[Any]				   =				tokenizer.encode(A__					)
            __lowerCamelCase						:       str				   =				encoded.index(A__					)
            __lowerCamelCase						:       Any				   =				tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
            self.assertEqual(A__	,    A__					)

            __lowerCamelCase						:       Any				   =				tokenizer.encode(A__					)
            __lowerCamelCase						:       Union[str, Any]				   =				encoded.index(A__					)
            __lowerCamelCase						:       Optional[int]				   =				tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
            self.assertNotEqual(A__	,    A__					)




       def 					a_					(					self      : Union[str, Any]					):


            """simple docstring"""







            pass




       def 					a_					(					self      : str					):


            """simple docstring"""







            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                      __lowerCamelCase						:       Tuple				   =				self.rust_tokenizer_class.from_pretrained(A__	,    **A__					)
                      __lowerCamelCase						:       List[Any]				   =				self.tokenizer_class.from_pretrained(A__	,    **A__					)
                      __lowerCamelCase						:       Tuple				   =				"""A, <mask> AllenNLP sentence."""
                      __lowerCamelCase						:       int				   =				tokenizer_r.encode_plus(A__	,    add_special_tokens=A__	,    return_token_type_ids=A__					)
                      __lowerCamelCase						:       List[str]				   =				tokenizer_p.encode_plus(A__	,    add_special_tokens=A__	,    return_token_type_ids=A__					)

                      # token_type_ids should put 0 everywhere
                      self.assertEqual(sum(tokens_r["""token_type_ids"""]					)	,    sum(tokens_p["""token_type_ids"""]					)					)

                      # attention_mask should put 1 everywhere, so sum over length should be 1
                      self.assertEqual(
                          sum(tokens_r["""attention_mask"""]					) / len(tokens_r["""attention_mask"""]					)	,    sum(tokens_p["""attention_mask"""]					) / len(tokens_p["""attention_mask"""]					)	,    )

                      __lowerCamelCase						:       Optional[int]				   =				tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]					)
                      __lowerCamelCase						:       Any				   =				tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]					)

                      # Rust correctly handles the space before the mask while python doesnt
                      self.assertSequenceEqual(tokens_p["""input_ids"""]	,    [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]					)
                      self.assertSequenceEqual(tokens_r["""input_ids"""]	,    [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]					)

                      self.assertSequenceEqual(
                          A__	,    ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]					)
                      self.assertSequenceEqual(
                          A__	,    ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]					)




       def 					a_					(					self      : str					):


            """simple docstring"""







            for trim_offsets, add_prefix_space in itertools.product([True, False]	,    repeat=2					):
                 __lowerCamelCase						:       Tuple				   =				self.rust_tokenizer_class.from_pretrained(
                     self.tmpdirname	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)

                 __lowerCamelCase						:       Union[str, Any]				   =				json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()					)
                 __lowerCamelCase						:       Dict				   =				json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()					)

                 self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]	,    A__					)

                 self.assertEqual(post_processor_state["""add_prefix_space"""]	,    A__					)
                 self.assertEqual(post_processor_state["""trim_offsets"""]	,    A__					)




       def 					a_					(					self      : List[Any]					):


            """simple docstring"""







            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                      __lowerCamelCase						:       Tuple				   =				"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                      __lowerCamelCase						:       Optional[Any]				   =				f"{text_of_1_token} {text_of_1_token}"

                      __lowerCamelCase						:       List[Any]				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       str				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (0, len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (len(A__					) + 1, len(A__					) + 1 + len(A__					))	,    )

                      __lowerCamelCase						:       Tuple				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       List[str]				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (0, len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (len(A__					) + 1, len(A__					) + 1 + len(A__					))	,    )

                      __lowerCamelCase						:       int				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       Union[str, Any]				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (0, len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (len(A__					), len(A__					) + 1 + len(A__					))	,    )

                      __lowerCamelCase						:       Optional[int]				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       Tuple				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (0, len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (len(A__					), len(A__					) + 1 + len(A__					))	,    )

                      __lowerCamelCase						:       str				   =				f" {text}"

                      # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                      #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                      # )
                      # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                      # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                      # self.assertEqual(
                      #     encoding.offset_mapping[1],
                      #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                      # )

                      __lowerCamelCase						:       Tuple				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       Optional[int]				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (1, 1 + len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (1 + len(A__					) + 1, 1 + len(A__					) + 1 + len(A__					))	,    )

                      __lowerCamelCase						:       Dict				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       Dict				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (0, 1 + len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (1 + len(A__					), 1 + len(A__					) + 1 + len(A__					))	,    )

                      __lowerCamelCase						:       Optional[int]				   =				self.rust_tokenizer_class.from_pretrained(
                          A__	,    use_fast=A__	,    add_prefix_space=A__	,    trim_offsets=A__					)
                      __lowerCamelCase						:       Optional[int]				   =				tokenizer_r(A__	,    return_offsets_mapping=A__	,    add_special_tokens=A__					)
                      self.assertEqual(encoding.offset_mapping[0]	,    (0, 1 + len(A__					))					)
                      self.assertEqual(
                          encoding.offset_mapping[1]	,    (1 + len(A__					), 1 + len(A__					) + 1 + len(A__					))	,    )



