


'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


lowerCamelCase__    	=     logging.get_logger(__name__)

lowerCamelCase__    	=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__    	=     {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase__    	=     {
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}






class 	lowerCAmelCase__			(			UpperCAmelCase__	):
			lowerCAmelCase :	int          =     VOCAB_FILES_NAMES
			lowerCAmelCase :	Optional[Any]          =     PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase :	Any          =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase :	int          =     ["input_ids", "attention_mask"]
			lowerCAmelCase :	List[Any]          =     RobertaTokenizer




			def __init__(		self  :     int	,  lowerCamelCase__  :     int=None	,  lowerCamelCase__  :     List[Any]=None	,  lowerCamelCase__  :     Tuple=None	,  lowerCamelCase__  :     Dict="replace"	,  lowerCamelCase__  :     Any="<s>"	,  lowerCamelCase__  :     Union[str, Any]="</s>"	,  lowerCamelCase__  :     Any="</s>"	,  lowerCamelCase__  :     Tuple="<s>"	,  lowerCamelCase__  :     Tuple="<unk>"	,  lowerCamelCase__  :     Dict="<pad>"	,  lowerCamelCase__  :     Dict="<mask>"	,  lowerCamelCase__  :     int=False	,  lowerCamelCase__  :     int=True	,  **lowerCamelCase__  :     Dict	,  )   ->Any:



					'''simple docstring'''







					super().__init__(
					    lowerCamelCase__	,  lowerCamelCase__	,  tokenizer_file=lowerCamelCase__	,  errors=lowerCamelCase__	,  bos_token=lowerCamelCase__	,  eos_token=lowerCamelCase__	,  sep_token=lowerCamelCase__	,  cls_token=lowerCamelCase__	,  unk_token=lowerCamelCase__	,  pad_token=lowerCamelCase__	,  mask_token=lowerCamelCase__	,  add_prefix_space=lowerCamelCase__	,  trim_offsets=lowerCamelCase__	,  **lowerCamelCase__	,  )

					_UpperCAmelCase		:       Tuple								=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
					if pre_tok_state.get("add_prefix_space"	,  lowerCamelCase__ ) != add_prefix_space:
							_UpperCAmelCase		:       Optional[Any]								=      getattr(lowerCamelCase__	,  pre_tok_state.pop("type" ) )
							_UpperCAmelCase		:       Dict								=      add_prefix_space
							_UpperCAmelCase		:       Union[str, Any]								=      pre_tok_class(**lowerCamelCase__ )

					_UpperCAmelCase		:       List[Any]								=      add_prefix_space

					_UpperCAmelCase		:       Optional[int]								=      "post_processor"
					_UpperCAmelCase		:       Optional[Any]								=      getattr(self.backend_tokenizer	,  lowerCamelCase__	,  lowerCamelCase__ )
					if tokenizer_component_instance:
							_UpperCAmelCase		:       Tuple								=      json.loads(tokenizer_component_instance.__getstate__() )

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
									_UpperCAmelCase		:       List[str]								=      tuple(state["sep"] )
							if "cls" in state:
									_UpperCAmelCase		:       str								=      tuple(state["cls"] )

							_UpperCAmelCase		:       List[str]								=      False

							if state.get("add_prefix_space"	,  lowerCamelCase__ ) != add_prefix_space:
									_UpperCAmelCase		:       Any								=      add_prefix_space
									_UpperCAmelCase		:       Any								=      True

							if state.get("trim_offsets"	,  lowerCamelCase__ ) != trim_offsets:
									_UpperCAmelCase		:       int								=      trim_offsets
									_UpperCAmelCase		:       Union[str, Any]								=      True

							if changes_to_apply:
									_UpperCAmelCase		:       Any								=      getattr(lowerCamelCase__	,  state.pop("type" ) )
									_UpperCAmelCase		:       Optional[int]								=      component_class(**lowerCamelCase__ )
									setattr(self.backend_tokenizer	,  lowerCamelCase__	,  lowerCamelCase__ )




			@property
			def 							lowerCAmelCase__    (		self  :     List[str] )   ->str:



					'''simple docstring'''







					if self._mask_token is None:
							if self.verbose:
									logger.error("Using mask_token, but it is not set yet." )
							return None
					return str(self._mask_token )




			@mask_token.setter
			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     List[str] )   ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:       List[str]								=      AddedToken(lowerCamelCase__	,  lstrip=lowerCamelCase__	,  rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__	,  lowerCamelCase__ ) else value
					_UpperCAmelCase		:       Any								=      value




			def 							lowerCAmelCase__    (		self  :     Tuple	,  *lowerCamelCase__  :     int	,  **lowerCamelCase__  :     List[Any] )   ->BatchEncoding:



					'''simple docstring'''







					_UpperCAmelCase		:       int								=      kwargs.get("is_split_into_words"	,  lowerCamelCase__ )
					assert self.add_prefix_space or not is_split_into_words, (
					    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    "to use it with pretokenized inputs."
					)

					return super()._batch_encode_plus(*lowerCamelCase__	,  **lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     Tuple	,  *lowerCamelCase__  :     str	,  **lowerCamelCase__  :     Optional[Any] )   ->BatchEncoding:



					'''simple docstring'''







					_UpperCAmelCase		:       Dict								=      kwargs.get("is_split_into_words"	,  lowerCamelCase__ )

					assert self.add_prefix_space or not is_split_into_words, (
					    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    "to use it with pretokenized inputs."
					)

					return super()._encode_plus(*lowerCamelCase__	,  **lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     List[str]	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Optional[str] = None )   ->Tuple[str]:



					'''simple docstring'''







					_UpperCAmelCase		:       List[Any]								=      self._tokenizer.model.save(lowerCamelCase__	,  name=lowerCamelCase__ )
					return tuple(lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Any=None )   ->int:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
							return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       Tuple								=      [self.sep_token_id]
					_UpperCAmelCase		:       List[Any]								=      [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]







'''simple docstring'''


from typing import Optional
from urllib.parse import quote

import huggingface_hub as hfh
from packaging import version






def  __lowerCAmelCase (__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase = None   ):
		if version.parse(hfh.__version__   ).release < version.parse("0.11.0"   ).release:
				# old versions of hfh don't url-encode the file path
				_UpperCAmelCase		:       str								=      quote(__lowerCAmelCase   )
		return hfh.hf_hub_url(__lowerCAmelCase      , __lowerCAmelCase      , repo_type="dataset"      , revision=__lowerCAmelCase   )



