


'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


a_						:       Union[str, Any]      			=				logging.get_logger(__name__)

a_						:       Optional[int]      			=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_						:       Any      			=				{
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

a_						:       Any      			=				{
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}





class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						VOCAB_FILES_NAMES
  _lowerCamelCase						=						PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase						=						["input_ids", "attention_mask"]
  _lowerCamelCase						=						RobertaTokenizer






  def __init__(					self    ,						UpperCamelCase=None    ,						UpperCamelCase=None    ,						UpperCamelCase=None    ,						UpperCamelCase="replace"    ,						UpperCamelCase="<s>"    ,						UpperCamelCase="</s>"    ,						UpperCamelCase="</s>"    ,						UpperCamelCase="<s>"    ,						UpperCamelCase="<unk>"    ,						UpperCamelCase="<pad>"    ,						UpperCamelCase="<mask>"    ,						UpperCamelCase=False    ,						UpperCamelCase=True    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   super().__init__(
       UpperCamelCase    ,						UpperCamelCase    ,						tokenizer_file=UpperCamelCase    ,						errors=UpperCamelCase    ,						bos_token=UpperCamelCase    ,						eos_token=UpperCamelCase    ,						sep_token=UpperCamelCase    ,						cls_token=UpperCamelCase    ,						unk_token=UpperCamelCase    ,						pad_token=UpperCamelCase    ,						mask_token=UpperCamelCase    ,						add_prefix_space=UpperCamelCase    ,						trim_offsets=UpperCamelCase    ,						**UpperCamelCase    ,						)

   lowerCamelCase_  				=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
   if pre_tok_state.get("add_prefix_space"    ,						UpperCamelCase  ) != add_prefix_space:
    lowerCamelCase_  				=						getattr(UpperCamelCase    ,						pre_tok_state.pop("type"  )  )
    lowerCamelCase_  				=						add_prefix_space
    lowerCamelCase_  				=						pre_tok_class(**UpperCamelCase  )

   lowerCamelCase_  				=						add_prefix_space

   lowerCamelCase_  				=						"post_processor"
   lowerCamelCase_  				=						getattr(self.backend_tokenizer    ,						UpperCamelCase    ,						UpperCamelCase  )
   if tokenizer_component_instance:
    lowerCamelCase_  				=						json.loads(tokenizer_component_instance.__getstate__()  )

    # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
    if "sep" in state:
     lowerCamelCase_  				=						tuple(state["sep"]  )
    if "cls" in state:
     lowerCamelCase_  				=						tuple(state["cls"]  )

    lowerCamelCase_  				=						False

    if state.get("add_prefix_space"    ,						UpperCamelCase  ) != add_prefix_space:
     lowerCamelCase_  				=						add_prefix_space
     lowerCamelCase_  				=						True

    if state.get("trim_offsets"    ,						UpperCamelCase  ) != trim_offsets:
     lowerCamelCase_  				=						trim_offsets
     lowerCamelCase_  				=						True

    if changes_to_apply:
     lowerCamelCase_  				=						getattr(UpperCamelCase    ,						state.pop("type"  )  )
     lowerCamelCase_  				=						component_class(**UpperCamelCase  )
     setattr(self.backend_tokenizer    ,						UpperCamelCase    ,						UpperCamelCase  )






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   if self._mask_token is None:
    if self.verbose:
     logger.error("Using mask_token, but it is not set yet."  )
    return None
   return str(self._mask_token  )






  @mask_token.setter
  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						AddedToken(UpperCamelCase    ,						lstrip=UpperCamelCase    ,						rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase    ,						UpperCamelCase  ) else value
   lowerCamelCase_  				=						value






  def 						snake_case	(					self    ,						*UpperCamelCase    ,						**UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						kwargs.get("is_split_into_words"    ,						UpperCamelCase  )
   assert self.add_prefix_space or not is_split_into_words, (
       f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*UpperCamelCase    ,						**UpperCamelCase  )






  def 						snake_case	(					self    ,						*UpperCamelCase    ,						**UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						kwargs.get("is_split_into_words"    ,						UpperCamelCase  )

   assert self.add_prefix_space or not is_split_into_words, (
       f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*UpperCamelCase    ,						**UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   lowerCamelCase_  				=						self._tokenizer.model.save(UpperCamelCase    ,						name=UpperCamelCase  )
   return tuple(UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase=None  ):







   """simple docstring"""
   lowerCamelCase_  				=						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
   if token_ids_a is None:
    return output

   return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   lowerCamelCase_  				=						[self.sep_token_id]
   lowerCamelCase_  				=						[self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep  ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







'''simple docstring'''

import unittest

import numpy as np

from transformers import DistilBertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
     import jax.numpy as jnp

     from transformers.models.distilbert.modeling_flax_distilbert import (
         FlaxDistilBertForMaskedLM,
         FlaxDistilBertForMultipleChoice,
         FlaxDistilBertForQuestionAnswering,
         FlaxDistilBertForSequenceClassification,
         FlaxDistilBertForTokenClassification,
         FlaxDistilBertModel,
     )





class   snake_case					(					unittest.TestCase    ):





  """simple docstring"""




  def __init__(					self    ,						UpperCamelCase    ,						UpperCamelCase=13    ,						UpperCamelCase=7    ,						UpperCamelCase=True    ,						UpperCamelCase=True    ,						UpperCamelCase=True    ,						UpperCamelCase=True    ,						UpperCamelCase=99    ,						UpperCamelCase=32    ,						UpperCamelCase=5    ,						UpperCamelCase=4    ,						UpperCamelCase=37    ,						UpperCamelCase="gelu"    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=512    ,						UpperCamelCase=16    ,						UpperCamelCase=2    ,						UpperCamelCase=0.02    ,						UpperCamelCase=4    ,						):







   """simple docstring"""
   lowerCamelCase_  				=						parent
   lowerCamelCase_  				=						batch_size
   lowerCamelCase_  				=						seq_length
   lowerCamelCase_  				=						is_training
   lowerCamelCase_  				=						use_attention_mask
   lowerCamelCase_  				=						use_token_type_ids
   lowerCamelCase_  				=						use_labels
   lowerCamelCase_  				=						vocab_size
   lowerCamelCase_  				=						hidden_size
   lowerCamelCase_  				=						num_hidden_layers
   lowerCamelCase_  				=						num_attention_heads
   lowerCamelCase_  				=						intermediate_size
   lowerCamelCase_  				=						hidden_act
   lowerCamelCase_  				=						hidden_dropout_prob
   lowerCamelCase_  				=						attention_probs_dropout_prob
   lowerCamelCase_  				=						max_position_embeddings
   lowerCamelCase_  				=						type_vocab_size
   lowerCamelCase_  				=						type_sequence_label_size
   lowerCamelCase_  				=						initializer_range
   lowerCamelCase_  				=						num_choices






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						ids_tensor([self.batch_size, self.seq_length]    ,						self.vocab_size  )

   lowerCamelCase_  				=						None
   if self.use_attention_mask:
    lowerCamelCase_  				=						random_attention_mask([self.batch_size, self.seq_length]  )

   lowerCamelCase_  				=						DistilBertConfig(
       vocab_size=self.vocab_size    ,						dim=self.hidden_size    ,						n_layers=self.num_hidden_layers    ,						n_heads=self.num_attention_heads    ,						hidden_dim=self.intermediate_size    ,						hidden_act=self.hidden_act    ,						dropout=self.hidden_dropout_prob    ,						attention_dropout=self.attention_probs_dropout_prob    ,						max_position_embeddings=self.max_position_embeddings    ,						initializer_range=self.initializer_range    ,						tie_weights_=UpperCamelCase    ,						)

   return config, input_ids, attention_mask







  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.prepare_config_and_inputs()
   lowerCamelCase_				,lowerCamelCase_				,lowerCamelCase_  				=						config_and_inputs
   lowerCamelCase_  				=						{"input_ids": input_ids, "attention_mask": attention_mask}
   return config, inputs_dict





@require_flax
class   snake_case					(					lowercase    ,		unittest.TestCase    ):





  """simple docstring"""




  _lowerCamelCase						=						(
      (
          FlaxDistilBertModel,
          FlaxDistilBertForMaskedLM,
          FlaxDistilBertForMultipleChoice,
          FlaxDistilBertForQuestionAnswering,
          FlaxDistilBertForSequenceClassification,
          FlaxDistilBertForTokenClassification,
          FlaxDistilBertForQuestionAnswering,
      )
      if is_flax_available()
      else ()
  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						FlaxDistilBertModelTester(self  )







  @slow
  def 						snake_case	(					self  ):







   """simple docstring"""
   for model_class_name in self.all_model_classes:
    lowerCamelCase_  				=						model_class_name.from_pretrained("distilbert-base-uncased"  )
    lowerCamelCase_  				=						model(np.ones((1, 1)  )  )
    self.assertIsNotNone(UpperCamelCase  )





@require_flax
class   snake_case					(					unittest.TestCase    ):





  """simple docstring"""




  @slow
  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						FlaxDistilBertModel.from_pretrained("distilbert-base-uncased"  )
   lowerCamelCase_  				=						np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]  )
   lowerCamelCase_  				=						np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]  )
   lowerCamelCase_  				=						model(UpperCamelCase    ,						attention_mask=UpperCamelCase  )[0]
   lowerCamelCase_  				=						(1, 11, 768)
   self.assertEqual(output.shape    ,						UpperCamelCase  )
   lowerCamelCase_  				=						np.array([[[-0.1_639, 0.3_299, 0.1_648], [-0.1_746, 0.3_289, 0.1_710], [-0.1_884, 0.3_357, 0.1_810]]]  )

   self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]    ,						UpperCamelCase    ,						atol=1e-4  )  )



