

from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


A											=     logging.get_logger(__name__)

A											=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

A											=     {
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

A											=     {'allegro/herbert-base-cased': 514}
A											=     {}


class SCREAMING_SNAKE_CASE  (		__snake_case							):






			"""simple docstring"""




			__A						  =  VOCAB_FILES_NAMES
			__A						  =  PRETRAINED_VOCAB_FILES_MAP
			__A						  =  PRETRAINED_INIT_CONFIGURATION
			__A						  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__A						  =  HerbertTokenizer


			def __init__(    self ,				__UpperCamelCase=None ,				__UpperCamelCase=None ,				__UpperCamelCase=None ,				__UpperCamelCase="<s>" ,				__UpperCamelCase="<unk>" ,				__UpperCamelCase="<pad>" ,				__UpperCamelCase="<mask>" ,				__UpperCamelCase="</s>" ,				**__UpperCamelCase ,				):

				"""simple docstring"""


				super().__init__(
				    __UpperCamelCase ,				__UpperCamelCase ,				tokenizer_file=__UpperCamelCase ,				cls_token=__UpperCamelCase ,				unk_token=__UpperCamelCase ,				pad_token=__UpperCamelCase ,				mask_token=__UpperCamelCase ,				sep_token=__UpperCamelCase ,				**__UpperCamelCase ,				)


			def 			__lowerCAmelCase (    self ,				__UpperCamelCase ,				__UpperCamelCase = None							):

				"""simple docstring"""


				snake_case_      		= [self.cls_token_id]
				snake_case_      		= [self.sep_token_id]
				if token_ids_a is None:
					return cls + token_ids_a + sep

				return cls + token_ids_a + sep + token_ids_a + sep


			def 			__lowerCAmelCase (    self ,				__UpperCamelCase ,				__UpperCamelCase = None ,				__UpperCamelCase = False							):

				"""simple docstring"""


				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=__UpperCamelCase ,				token_ids_a=__UpperCamelCase ,				already_has_special_tokens=__UpperCamelCase							)

				if token_ids_a is None:
					return [1] + ([0] * len(__UpperCamelCase							)) + [1]
				return [1] + ([0] * len(__UpperCamelCase							)) + [1] + ([0] * len(__UpperCamelCase							)) + [1]


			def 			__lowerCAmelCase (    self ,				__UpperCamelCase ,				__UpperCamelCase = None							):

				"""simple docstring"""


				snake_case_      		= [self.sep_token_id]
				snake_case_      		= [self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep							) * [0]
				return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







			def 			__lowerCAmelCase (    self ,				__UpperCamelCase ,				__UpperCamelCase = None							):

				"""simple docstring"""


				snake_case_      		= self._tokenizer.model.save(__UpperCamelCase ,				name=__UpperCamelCase							)
				return tuple(__UpperCamelCase							)



from ..utils import DummyObject, requires_backends


class SCREAMING_SNAKE_CASE  (		metaclass=__snake_case							):






			"""simple docstring"""




			__A						  =  ["""torch""", """transformers""", """onnx"""]


			def __init__(    self ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(self ,				['torch', 'transformers', 'onnx']							)


			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







class SCREAMING_SNAKE_CASE  (		metaclass=__snake_case							):






			"""simple docstring"""




			__A						  =  ["""torch""", """transformers""", """onnx"""]


			def __init__(    self ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(self ,				['torch', 'transformers', 'onnx']							)


			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







class SCREAMING_SNAKE_CASE  (		metaclass=__snake_case							):






			"""simple docstring"""




			__A						  =  ["""torch""", """transformers""", """onnx"""]


			def __init__(    self ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(self ,				['torch', 'transformers', 'onnx']							)


			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







class SCREAMING_SNAKE_CASE  (		metaclass=__snake_case							):






			"""simple docstring"""




			__A						  =  ["""torch""", """transformers""", """onnx"""]


			def __init__(    self ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(self ,				['torch', 'transformers', 'onnx']							)


			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







class SCREAMING_SNAKE_CASE  (		metaclass=__snake_case							):






			"""simple docstring"""




			__A						  =  ["""torch""", """transformers""", """onnx"""]


			def __init__(    self ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(self ,				['torch', 'transformers', 'onnx']							)


			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







class SCREAMING_SNAKE_CASE  (		metaclass=__snake_case							):






			"""simple docstring"""




			__A						  =  ["""torch""", """transformers""", """onnx"""]


			def __init__(    self ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(self ,				['torch', 'transformers', 'onnx']							)


			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)







			@classmethod
			def 			__lowerCAmelCase (    cls ,				*__UpperCamelCase ,				**__UpperCamelCase							):

				"""simple docstring"""


				requires_backends(cls ,				['torch', 'transformers', 'onnx']							)
