



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


a_						:     Optional[Any]	     =		logging.get_logger(__name__)

a_						:     List[Any]	     =		{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a_						:     List[Any]	     =		{
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

a_						:     str	     =		{
    'roberta-base': 5_1_2,
    'roberta-large': 5_1_2,
    'roberta-large-mnli': 5_1_2,
    'distilroberta-base': 5_1_2,
    'roberta-base-openai-detector': 5_1_2,
    'roberta-large-openai-detector': 5_1_2,
}





class    lowerCamelCase__  (						__SCREAMING_SNAKE_CASE):
     """simple docstring"""



     _A       =			VOCAB_FILES_NAMES
     _A       =			PRETRAINED_VOCAB_FILES_MAP
     _A       =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A       =			['input_ids', 'attention_mask']
     _A       =			RobertaTokenizer







     def __init__(self				,			__a=None				,			__a=None				,			__a=None				,			__a="replace"				,			__a="<s>"				,			__a="</s>"				,			__a="</s>"				,			__a="<s>"				,			__a="<unk>"				,			__a="<pad>"				,			__a="<mask>"				,			__a=False				,			__a=True				,			**__a				,			):






           '''simple docstring'''



           super().__init__(
               _a				,			_a				,			tokenizer_file=_a				,			errors=_a				,			bos_token=_a				,			eos_token=_a				,			sep_token=_a				,			cls_token=_a				,			unk_token=_a				,			pad_token=_a				,			mask_token=_a				,			add_prefix_space=_a				,			trim_offsets=_a				,			**_a				,			)

           lowerCamelCase								=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
           if pre_tok_state.get("add_prefix_space"				,			_a    ) != add_prefix_space:
                 lowerCamelCase								=		getattr(_a				,			pre_tok_state.pop("type"    )    )
                 lowerCamelCase								=		add_prefix_space
                 lowerCamelCase								=		pre_tok_class(**_a    )

           lowerCamelCase								=		add_prefix_space

           lowerCamelCase								=		"post_processor"
           lowerCamelCase								=		getattr(self.backend_tokenizer				,			_a				,			_a    )
           if tokenizer_component_instance:
                 lowerCamelCase								=		json.loads(tokenizer_component_instance.__getstate__()    )

                 # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                 if "sep" in state:
                       lowerCamelCase								=		tuple(state["sep"]    )
                 if "cls" in state:
                       lowerCamelCase								=		tuple(state["cls"]    )

                 lowerCamelCase								=		False

                 if state.get("add_prefix_space"				,			_a    ) != add_prefix_space:
                       lowerCamelCase								=		add_prefix_space
                       lowerCamelCase								=		True

                 if state.get("trim_offsets"				,			_a    ) != trim_offsets:
                       lowerCamelCase								=		trim_offsets
                       lowerCamelCase								=		True

                 if changes_to_apply:
                       lowerCamelCase								=		getattr(_a				,			state.pop("type"    )    )
                       lowerCamelCase								=		component_class(**_a    )
                       setattr(self.backend_tokenizer				,			_a				,			_a    )







     @property
     def 						_a      (self    ):






           '''simple docstring'''



           if self._mask_token is None:
                 if self.verbose:
                       logger.error("Using mask_token, but it is not set yet."    )
                 return None
           return str(self._mask_token    )







     @mask_token.setter
     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		AddedToken(_a				,			lstrip=_a				,			rstrip=_a    ) if isinstance(_a				,			_a    ) else value
           lowerCamelCase								=		value







     def 						_a      (self				,			*__a				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase								=		kwargs.get("is_split_into_words"				,			_a    )
           assert self.add_prefix_space or not is_split_into_words, (
               F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
               "to use it with pretokenized inputs."
           )

           return super()._batch_encode_plus(*_a				,			**_a    )







     def 						_a      (self				,			*__a				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase								=		kwargs.get("is_split_into_words"				,			_a    )

           assert self.add_prefix_space or not is_split_into_words, (
               F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
               "to use it with pretokenized inputs."
           )

           return super()._encode_plus(*_a				,			**_a    )







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		self._tokenizer.model.save(_a				,			name=_a    )
           return tuple(_a    )







     def 						_a      (self				,			__a				,			__a=None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
           if token_ids_a is None:
                 return output

           return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep    ) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


a_						:     int	     =		{'configuration_van': ['VAN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VanConfig']}


try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_						:     Dict	     =		[
         'VAN_PRETRAINED_MODEL_ARCHIVE_LIST',
         'VanForImageClassification',
         'VanModel',
         'VanPreTrainedModel',
     ]

if TYPE_CHECKING:
     from .configuration_van import VAN_PRETRAINED_CONFIG_ARCHIVE_MAP, VanConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_van import (
              VAN_PRETRAINED_MODEL_ARCHIVE_LIST,
              VanForImageClassification,
              VanModel,
              VanPreTrainedModel,
          )

else:
     import sys

     a_						:     Union[str, Any]	     =		_LazyModule(__name__, globals()['__file__'], _import_structure)