



"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


a_			   =					logging.get_logger(__name__)

a_			   =					{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a_			   =					{
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

a_			   =					{
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							Optional[Any]							       =		VOCAB_FILES_NAMES
						_A		:							Optional[int]							       =		PRETRAINED_VOCAB_FILES_MAP
						_A		:							Any							       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_A		:							Optional[Any]							       =		["""input_ids""", """attention_mask"""]
						_A		:							Optional[Any]							       =		RobertaTokenizer







						def __init__(self   ,		lowercase__=None   ,		lowercase__=None   ,		lowercase__=None   ,		lowercase__="replace"   ,		lowercase__="<s>"   ,		lowercase__="</s>"   ,		lowercase__="</s>"   ,		lowercase__="<s>"   ,		lowercase__="<unk>"   ,		lowercase__="<pad>"   ,		lowercase__="<mask>"   ,		lowercase__=False   ,		lowercase__=True   ,		**lowercase__   ,		):
											super().__init__(
											    lowercase__   ,		lowercase__   ,		tokenizer_file=lowercase__   ,		errors=lowercase__   ,		bos_token=lowercase__   ,		eos_token=lowercase__   ,		sep_token=lowercase__   ,		cls_token=lowercase__   ,		unk_token=lowercase__   ,		pad_token=lowercase__   ,		mask_token=lowercase__   ,		add_prefix_space=lowercase__   ,		trim_offsets=lowercase__   ,		**lowercase__   ,		)

											snake_case_					:		Dict 					=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
											if pre_tok_state.get("""add_prefix_space"""   ,		lowercase__       ) != add_prefix_space:
																snake_case_					:		Optional[int] 					=					getattr(lowercase__   ,		pre_tok_state.pop("""type"""       )       )
																snake_case_					:		Union[str, Any] 					=					add_prefix_space
																snake_case_					:		str 					=					pre_tok_class(**lowercase__       )

											snake_case_					:		int 					=					add_prefix_space

											snake_case_					:		Dict 					=					"""post_processor"""
											snake_case_					:		Dict 					=					getattr(self.backend_tokenizer   ,		lowercase__   ,		lowercase__       )
											if tokenizer_component_instance:
																snake_case_					:		Dict 					=					json.loads(tokenizer_component_instance.__getstate__()       )

																# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																if "sep" in state:
																					snake_case_					:		Dict 					=					tuple(state["""sep"""]       )
																if "cls" in state:
																					snake_case_					:		Dict 					=					tuple(state["""cls"""]       )

																snake_case_					:		List[Any] 					=					False

																if state.get("""add_prefix_space"""   ,		lowercase__       ) != add_prefix_space:
																					snake_case_					:		str 					=					add_prefix_space
																					snake_case_					:		Optional[Any] 					=					True

																if state.get("""trim_offsets"""   ,		lowercase__       ) != trim_offsets:
																					snake_case_					:		Optional[int] 					=					trim_offsets
																					snake_case_					:		Optional[Any] 					=					True

																if changes_to_apply:
																					snake_case_					:		Dict 					=					getattr(lowercase__   ,		state.pop("""type"""       )       )
																					snake_case_					:		Any 					=					component_class(**lowercase__       )
																					setattr(self.backend_tokenizer   ,		lowercase__   ,		lowercase__       )







						@property
						def 						__UpperCamelCase							(self       ):
											if self._mask_token is None:
																if self.verbose:
																					logger.error("""Using mask_token, but it is not set yet."""       )
																return None
											return str(self._mask_token       )







						@mask_token.setter
						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					:		List[Any] 					=					AddedToken(lowercase__   ,		lstrip=lowercase__   ,		rstrip=lowercase__       ) if isinstance(lowercase__   ,		lowercase__       ) else value
											snake_case_					:		Any 					=					value







						def 						__UpperCamelCase							(self   ,		*lowercase__   ,		**lowercase__       ):
											snake_case_					:		List[Any] 					=					kwargs.get("""is_split_into_words"""   ,		lowercase__       )
											assert self.add_prefix_space or not is_split_into_words, (
											    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*lowercase__   ,		**lowercase__       )







						def 						__UpperCamelCase							(self   ,		*lowercase__   ,		**lowercase__       ):
											snake_case_					:		List[Any] 					=					kwargs.get("""is_split_into_words"""   ,		lowercase__       )

											assert self.add_prefix_space or not is_split_into_words, (
											    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*lowercase__   ,		**lowercase__       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = None       ):
											snake_case_					:		Any 					=					self._tokenizer.model.save(lowercase__   ,		name=lowercase__       )
											return tuple(lowercase__       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__=None       ):
											snake_case_					:		Union[str, Any] 					=					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = None       ):
											snake_case_					:		Dict 					=					[self.sep_token_id]
											snake_case_					:		int 					=					[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]









"""simple docstring"""

def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  int			):



					"""simple docstring"""





					return numa ^ numa < 0


if __name__ == "__main__":
			import doctest

			doctest.testmod()




