



'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)

lowerCamelCase	 =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase	 =       {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase	 =       {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCamelCase	 =       {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_INIT_CONFIGURATION
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    RealmTokenizer

				def __init__(							self  :  List[str]  ,       _lowerCAmelCase  :  Optional[int]=None  ,       _lowerCAmelCase  :  Optional[Any]=None  ,       _lowerCAmelCase  :  Union[str, Any]=True  ,       _lowerCAmelCase  :  List[Any]="[UNK]"  ,       _lowerCAmelCase  :  List[str]="[SEP]"  ,       _lowerCAmelCase  :  Union[str, Any]="[PAD]"  ,       _lowerCAmelCase  :  str="[CLS]"  ,       _lowerCAmelCase  :  Dict="[MASK]"  ,       _lowerCAmelCase  :  Any=True  ,       _lowerCAmelCase  :  Any=None  ,       **_lowerCAmelCase  :  Dict  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       do_lower_case=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       sep_token=_lowerCAmelCase  ,       pad_token=_lowerCAmelCase  ,       cls_token=_lowerCAmelCase  ,       mask_token=_lowerCAmelCase  ,       tokenize_chinese_chars=_lowerCAmelCase  ,       strip_accents=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											__lowercase       	=json.loads(self.backend_tokenizer.normalizer.__getstate__())
											if (
											    normalizer_state.get('lowercase'  ,       _lowerCAmelCase) != do_lower_case
											    or normalizer_state.get('strip_accents'  ,       _lowerCAmelCase) != strip_accents
											    or normalizer_state.get('handle_chinese_chars'  ,       _lowerCAmelCase) != tokenize_chinese_chars
											):
																		__lowercase       	=getattr(_lowerCAmelCase  ,       normalizer_state.pop('type'))
																		__lowercase       	=do_lower_case
																		__lowercase       	=strip_accents
																		__lowercase       	=tokenize_chinese_chars
																		__lowercase       	=normalizer_class(**_lowerCAmelCase)

											__lowercase       	=do_lower_case

				def 				__lowerCamelCase     (							self  :  List[str]  ,       _lowerCAmelCase  :  Dict  ,       **_lowerCAmelCase  :  Tuple):







											'''simple docstring'''

											__lowercase       	=PaddingStrategy.MAX_LENGTH

											__lowercase       	=text
											__lowercase       	=kwargs.pop('text_pair'  ,       _lowerCAmelCase)
											__lowercase       	=kwargs.pop('return_tensors'  ,       _lowerCAmelCase)

											__lowercase       	={
											    'input_ids': [],
											    'attention_mask': [],
											    'token_type_ids': [],
											}

											for idx, candidate_text in enumerate(_lowerCAmelCase):
																		if batch_text_pair is not None:
																									__lowercase       	=batch_text_pair[idx]
																		else:
																									__lowercase       	=None

																		__lowercase       	=super().__call__(_lowerCAmelCase  ,       _lowerCAmelCase  ,       return_tensors=_lowerCAmelCase  ,       **_lowerCAmelCase)

																		__lowercase       	=encoded_candidates.get('input_ids')
																		__lowercase       	=encoded_candidates.get('attention_mask')
																		__lowercase       	=encoded_candidates.get('token_type_ids')

																		if encoded_input_ids is not None:
																									output_data["input_ids"].append(_lowerCAmelCase)
																		if encoded_attention_mask is not None:
																									output_data["attention_mask"].append(_lowerCAmelCase)
																		if encoded_token_type_ids is not None:
																									output_data["token_type_ids"].append(_lowerCAmelCase)

											__lowercase       	={key: item for key, item in output_data.items() if len(_lowerCAmelCase) != 0}

											return BatchEncoding(_lowerCAmelCase  ,       tensor_type=_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  Tuple  ,       _lowerCAmelCase  :  Any  ,       _lowerCAmelCase  :  Optional[int]=None):







											'''simple docstring'''

											__lowercase       	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       _lowerCAmelCase  :  List[int]  ,       _lowerCAmelCase  :  Optional[List[int]] = None):







											'''simple docstring'''

											__lowercase       	=[self.sep_token_id]
											__lowercase       	=[self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep) * [0]
											return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)








'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


lowerCamelCase	 =       logging.get_logger(__name__)


lowerCamelCase	 =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
lowerCamelCase	 =       {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase	 =       {
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    ["""input_ids""", """attention_mask"""]
				lowerCAmelCase__					    =    BartTokenizer

				def __init__(							self  :  Optional[int]  ,       _lowerCAmelCase  :  Any=None  ,       _lowerCAmelCase  :  Tuple=None  ,       _lowerCAmelCase  :  Optional[int]=None  ,       _lowerCAmelCase  :  str="replace"  ,       _lowerCAmelCase  :  List[Any]="<s>"  ,       _lowerCAmelCase  :  int="</s>"  ,       _lowerCAmelCase  :  Dict="</s>"  ,       _lowerCAmelCase  :  Optional[int]="<s>"  ,       _lowerCAmelCase  :  str="<unk>"  ,       _lowerCAmelCase  :  List[str]="<pad>"  ,       _lowerCAmelCase  :  Dict="<mask>"  ,       _lowerCAmelCase  :  Tuple=False  ,       _lowerCAmelCase  :  str=True  ,       **_lowerCAmelCase  :  Tuple  ,       ):







											'''simple docstring'''

											super().__init__(
											    _lowerCAmelCase  ,       _lowerCAmelCase  ,       tokenizer_file=_lowerCAmelCase  ,       errors=_lowerCAmelCase  ,       bos_token=_lowerCAmelCase  ,       eos_token=_lowerCAmelCase  ,       sep_token=_lowerCAmelCase  ,       cls_token=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       pad_token=_lowerCAmelCase  ,       mask_token=_lowerCAmelCase  ,       add_prefix_space=_lowerCAmelCase  ,       trim_offsets=_lowerCAmelCase  ,       **_lowerCAmelCase  ,       )

											__lowercase       	=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																		__lowercase       	=getattr(_lowerCAmelCase  ,       pre_tok_state.pop('type'))
																		__lowercase       	=add_prefix_space
																		__lowercase       	=pre_tok_class(**_lowerCAmelCase)

											__lowercase       	=add_prefix_space

											# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
											__lowercase       	='post_processor'
											__lowercase       	=getattr(self.backend_tokenizer  ,       _lowerCAmelCase  ,       _lowerCAmelCase)
											if tokenizer_component_instance:
																		__lowercase       	=json.loads(tokenizer_component_instance.__getstate__())

																		# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																		if "sep" in state:
																									__lowercase       	=tuple(state['sep'])
																		if "cls" in state:
																									__lowercase       	=tuple(state['cls'])

																		__lowercase       	=False

																		if state.get('add_prefix_space'  ,       _lowerCAmelCase) != add_prefix_space:
																									__lowercase       	=add_prefix_space
																									__lowercase       	=True

																		if state.get('trim_offsets'  ,       _lowerCAmelCase) != trim_offsets:
																									__lowercase       	=trim_offsets
																									__lowercase       	=True

																		if changes_to_apply:
																									__lowercase       	=getattr(_lowerCAmelCase  ,       state.pop('type'))
																									__lowercase       	=component_class(**_lowerCAmelCase)
																									setattr(self.backend_tokenizer  ,       _lowerCAmelCase  ,       _lowerCAmelCase)

				@property
				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											if self._mask_token is None:
																		if self.verbose:
																									logger.error('Using mask_token, but it is not set yet.')
																		return None
											return str(self._mask_token)

				@mask_token.setter
				def 				__lowerCamelCase     (							self  :  Union[str, Any]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											__lowercase       	=AddedToken(_lowerCAmelCase  ,       lstrip=_lowerCAmelCase  ,       rstrip=_lowerCAmelCase) if isinstance(_lowerCAmelCase  ,       _lowerCAmelCase) else value
											__lowercase       	=value

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       *_lowerCAmelCase  :  int  ,       **_lowerCAmelCase  :  str):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    'to use it with pretokenized inputs.')

											return super()._batch_encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       *_lowerCAmelCase  :  List[str]  ,       **_lowerCAmelCase  :  Tuple):







											'''simple docstring'''

											__lowercase       	=kwargs.get('is_split_into_words'  ,       _lowerCAmelCase)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    'to use it with pretokenized inputs.')

											return super()._encode_plus(*_lowerCAmelCase  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  int  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=self._tokenizer.model.save(_lowerCAmelCase  ,       name=_lowerCAmelCase)
											return tuple(_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  List[str]=None):







											'''simple docstring'''

											__lowercase       	=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																		return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

				def 				__lowerCamelCase     (							self  :  List[str]  ,       _lowerCAmelCase  :  List[int]  ,       _lowerCAmelCase  :  Optional[List[int]] = None):







											'''simple docstring'''

											__lowercase       	=[self.sep_token_id]
											__lowercase       	=[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



