

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


UpperCAmelCase__ :     str				    =    logging.get_logger(__name__)

UpperCAmelCase__ :     int				    =    {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__ :     str				    =    {
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/tokenizer.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/tokenizer.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json",
        "roberta-base-openai-detector": (
            "https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"
        ),
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__ :     Optional[Any]				    =    {
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}







class        a__       (   lowercase__       ):







  """simple docstring"""




  UpperCAmelCase__      : Optional[Any]			       =VOCAB_FILES_NAMES
  UpperCAmelCase__      : Union[str, Any]			       =PRETRAINED_VOCAB_FILES_MAP
  UpperCAmelCase__      : int			       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCAmelCase__      : Dict			       =["""input_ids""", """attention_mask"""]
  UpperCAmelCase__      : str			       =RobertaTokenizer


  def __init__(			self    : Tuple				,				UpperCAmelCase__    : Any=None				,				UpperCAmelCase__    : str=None				,				UpperCAmelCase__    : Dict=None				,				UpperCAmelCase__    : int="replace"				,				UpperCAmelCase__    : Optional[Any]="<s>"				,				UpperCAmelCase__    : Any="</s>"				,				UpperCAmelCase__    : Union[str, Any]="</s>"				,				UpperCAmelCase__    : Dict="<s>"				,				UpperCAmelCase__    : Any="<unk>"				,				UpperCAmelCase__    : str="<pad>"				,				UpperCAmelCase__    : int="<mask>"				,				UpperCAmelCase__    : List[str]=False				,				UpperCAmelCase__    : Tuple=True				,				**UpperCAmelCase__    : int				,				)					->int:
        """simple docstring"""







        super().__init__(
            UpperCAmelCase__				,				UpperCAmelCase__				,				tokenizer_file=UpperCAmelCase__				,				errors=UpperCAmelCase__				,				bos_token=UpperCAmelCase__				,				eos_token=UpperCAmelCase__				,				sep_token=UpperCAmelCase__				,				cls_token=UpperCAmelCase__				,				unk_token=UpperCAmelCase__				,				pad_token=UpperCAmelCase__				,				mask_token=UpperCAmelCase__				,				add_prefix_space=UpperCAmelCase__				,				trim_offsets=UpperCAmelCase__				,				**UpperCAmelCase__				,				)

        SCREAMING_SNAKE_CASE :      List[Any]       =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
        if pre_tok_state.get("""add_prefix_space"""				,				UpperCAmelCase__		) != add_prefix_space:
              SCREAMING_SNAKE_CASE :      int       =   getattr(UpperCAmelCase__				,				pre_tok_state.pop("""type"""		)		)
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   add_prefix_space
              SCREAMING_SNAKE_CASE :      List[str]       =   pre_tok_class(**UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      Optional[int]       =   add_prefix_space

        SCREAMING_SNAKE_CASE :      int       =   "post_processor"
        SCREAMING_SNAKE_CASE :      Any       =   getattr(self.backend_tokenizer				,				UpperCAmelCase__				,				UpperCAmelCase__		)
        if tokenizer_component_instance:
              SCREAMING_SNAKE_CASE :      Tuple       =   json.loads(tokenizer_component_instance.__getstate__()		)

              # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
              if "sep" in state:
                    SCREAMING_SNAKE_CASE :      Optional[int]       =   tuple(state["""sep"""]		)
              if "cls" in state:
                    SCREAMING_SNAKE_CASE :      Union[str, Any]       =   tuple(state["""cls"""]		)

              SCREAMING_SNAKE_CASE :      Optional[int]       =   False

              if state.get("""add_prefix_space"""				,				UpperCAmelCase__		) != add_prefix_space:
                    SCREAMING_SNAKE_CASE :      Optional[Any]       =   add_prefix_space
                    SCREAMING_SNAKE_CASE :      Tuple       =   True

              if state.get("""trim_offsets"""				,				UpperCAmelCase__		) != trim_offsets:
                    SCREAMING_SNAKE_CASE :      List[str]       =   trim_offsets
                    SCREAMING_SNAKE_CASE :      str       =   True

              if changes_to_apply:
                    SCREAMING_SNAKE_CASE :      Optional[int]       =   getattr(UpperCAmelCase__				,				state.pop("""type"""		)		)
                    SCREAMING_SNAKE_CASE :      List[Any]       =   component_class(**UpperCAmelCase__		)
                    setattr(self.backend_tokenizer				,				UpperCAmelCase__				,				UpperCAmelCase__		)


  @property
  def  _lowercase      (			self    : List[str]		)					->str:
        """simple docstring"""







        if self._mask_token is None:
              if self.verbose:
                    logger.error("""Using mask_token, but it is not set yet."""		)
              return None
        return str(self._mask_token		)


  @mask_token.setter
  def  _lowercase      (			self    : Tuple				,				UpperCAmelCase__    : Optional[Any]		)					->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   AddedToken(UpperCAmelCase__				,				lstrip=UpperCAmelCase__				,				rstrip=UpperCAmelCase__		) if isinstance(UpperCAmelCase__				,				UpperCAmelCase__		) else value
        SCREAMING_SNAKE_CASE :      List[Any]       =   value


  def  _lowercase      (			self    : Union[str, Any]				,				*UpperCAmelCase__    : Union[str, Any]				,				**UpperCAmelCase__    : List[Any]		)					->BatchEncoding:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   kwargs.get("""is_split_into_words"""				,				UpperCAmelCase__		)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*UpperCAmelCase__				,				**UpperCAmelCase__		)


  def  _lowercase      (			self    : List[str]				,				*UpperCAmelCase__    : Dict				,				**UpperCAmelCase__    : List[str]		)					->BatchEncoding:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   kwargs.get("""is_split_into_words"""				,				UpperCAmelCase__		)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*UpperCAmelCase__				,				**UpperCAmelCase__		)


  def  _lowercase      (			self    : Dict				,				UpperCAmelCase__    : str				,				UpperCAmelCase__    : Optional[str] = None		)					->Tuple[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   self._tokenizer.model.save(UpperCAmelCase__				,				name=UpperCAmelCase__		)
        return tuple(UpperCAmelCase__		)


  def  _lowercase      (			self    : Any				,				UpperCAmelCase__    : Union[str, Any]				,				UpperCAmelCase__    : str=None		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
              return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




  def  _lowercase      (			self    : Union[str, Any]				,				UpperCAmelCase__    : List[int]				,				UpperCAmelCase__    : Optional[List[int]] = None		)					->List[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      str       =   [self.sep_token_id]
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   [self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep		) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]







import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


UpperCAmelCase__ :     int				    =    get_tests_dir("""fixtures""")







class        a__       (   unittest.TestCase       ):







  """simple docstring"""




  def  _lowercase      (			self    : Dict		)					->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   mock.Mock()
        SCREAMING_SNAKE_CASE :      Optional[int]       =   5_0_0
        SCREAMING_SNAKE_CASE :      List[str]       =   {}
        SCREAMING_SNAKE_CASE :      List[Any]       =   HTTPError
        SCREAMING_SNAKE_CASE :      int       =   {}

        # Download this model to make sure it's in the cache.
        SCREAMING_SNAKE_CASE :      List[Any]       =   WavaVecaFeatureExtractor.from_pretrained("""hf-internal-testing/tiny-random-wav2vec2"""		)
        # Under the mock environment we get a 500 error when trying to reach the model.
        with mock.patch("""requests.Session.request"""				,				return_value=UpperCAmelCase__		) as mock_head:
              SCREAMING_SNAKE_CASE :      Optional[int]       =   WavaVecaFeatureExtractor.from_pretrained("""hf-internal-testing/tiny-random-wav2vec2"""		)
              # This check we did call the fake head request
              mock_head.assert_called()




  def  _lowercase      (			self    : Optional[Any]		)					->int:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      str       =   WavaVecaFeatureExtractor.from_pretrained(
            """https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json"""		)







@is_staging_test
class        a__       (   unittest.TestCase       ):







  """simple docstring"""




  @classmethod
  def  _lowercase      (			cls    : List[str]		)					->Dict:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   TOKEN
        HfFolder.save_token(UpperCAmelCase__		)


  @classmethod
  def  _lowercase      (			cls    : List[str]		)					->Optional[int]:
        """simple docstring"""







        try:
              delete_repo(token=cls._token				,				repo_id="""test-feature-extractor"""		)
        except HTTPError:
              pass

        try:
              delete_repo(token=cls._token				,				repo_id="""valid_org/test-feature-extractor-org"""		)
        except HTTPError:
              pass

        try:
              delete_repo(token=cls._token				,				repo_id="""test-dynamic-feature-extractor"""		)
        except HTTPError:
              pass


  def  _lowercase      (			self    : int		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   WavaVecaFeatureExtractor.from_pretrained(UpperCAmelCase__		)
        feature_extractor.push_to_hub("""test-feature-extractor"""				,				use_auth_token=self._token		)

        SCREAMING_SNAKE_CASE :      Any       =   WavaVecaFeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor"		)
        for k, v in feature_extractor.__dict__.items():
              self.assertEqual(UpperCAmelCase__				,				getattr(UpperCAmelCase__				,				UpperCAmelCase__		)		)

        # Reset repo
        delete_repo(token=self._token				,				repo_id="""test-feature-extractor"""		)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
              feature_extractor.save_pretrained(
                  UpperCAmelCase__				,				repo_id="""test-feature-extractor"""				,				push_to_hub=UpperCAmelCase__				,				use_auth_token=self._token		)

        SCREAMING_SNAKE_CASE :      List[str]       =   WavaVecaFeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor"		)
        for k, v in feature_extractor.__dict__.items():
              self.assertEqual(UpperCAmelCase__				,				getattr(UpperCAmelCase__				,				UpperCAmelCase__		)		)


  def  _lowercase      (			self    : List[Any]		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   WavaVecaFeatureExtractor.from_pretrained(UpperCAmelCase__		)
        feature_extractor.push_to_hub("""valid_org/test-feature-extractor"""				,				use_auth_token=self._token		)

        SCREAMING_SNAKE_CASE :      Optional[Any]       =   WavaVecaFeatureExtractor.from_pretrained("""valid_org/test-feature-extractor"""		)
        for k, v in feature_extractor.__dict__.items():
              self.assertEqual(UpperCAmelCase__				,				getattr(UpperCAmelCase__				,				UpperCAmelCase__		)		)

        # Reset repo
        delete_repo(token=self._token				,				repo_id="""valid_org/test-feature-extractor"""		)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
              feature_extractor.save_pretrained(
                  UpperCAmelCase__				,				repo_id="""valid_org/test-feature-extractor-org"""				,				push_to_hub=UpperCAmelCase__				,				use_auth_token=self._token		)

        SCREAMING_SNAKE_CASE :      List[Any]       =   WavaVecaFeatureExtractor.from_pretrained("""valid_org/test-feature-extractor-org"""		)
        for k, v in feature_extractor.__dict__.items():
              self.assertEqual(UpperCAmelCase__				,				getattr(UpperCAmelCase__				,				UpperCAmelCase__		)		)




  def  _lowercase      (			self    : Optional[Any]		)					->Optional[int]:
        """simple docstring"""







        CustomFeatureExtractor.register_for_auto_class()
        SCREAMING_SNAKE_CASE :      Any       =   CustomFeatureExtractor.from_pretrained(UpperCAmelCase__		)

        feature_extractor.push_to_hub("""test-dynamic-feature-extractor"""				,				use_auth_token=self._token		)

        # This has added the proper auto_map field to the config
        self.assertDictEqual(
            feature_extractor.auto_map				,				{"""AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor"""}				,				)

        SCREAMING_SNAKE_CASE :      Any       =   AutoFeatureExtractor.from_pretrained(
            f"{USER}/test-dynamic-feature-extractor"				,				trust_remote_code=UpperCAmelCase__		)
        # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
        self.assertEqual(new_feature_extractor.__class__.__name__				,				"""CustomFeatureExtractor"""		)




