




import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


lowercase__  		=		logging.get_logger(__name__)


lowercase__  		=		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all LED models at https://huggingface.co/models?filter=LED
lowercase__  		=		{
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

lowercase__  		=		{
    """allenai/led-base-16384""": 1_6384,
}







@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def   _snake_case    (       ):
				_lowerCamelCase  :    List[str]					=					(
				    list(range(ord('!'	)							,					ord('~'	) + 1	)	) + list(range(ord('¡'	)							,					ord('¬'	) + 1	)	) + list(range(ord('®'	)							,					ord('ÿ'	) + 1	)	)
				)
				_lowerCamelCase  :    Any					=					bs[:]
				_lowerCamelCase  :    Optional[Any]					=					0
				for b in range(2**8	):
								if b not in bs:
												bs.append(lowercase__	)
												cs.append(2**8 + n	)
												n += 1
				_lowerCamelCase  :    Dict					=					[chr(lowercase__	) for n in cs]
				return dict(zip(lowercase__							,					lowercase__	)	)







def   _snake_case    (       lowercase__	):
				_lowerCamelCase  :    Union[str, Any]					=					set()
				_lowerCamelCase  :    Any					=					word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)	)
								_lowerCamelCase  :    Any					=					char
				return pairs






class lowerCAmelCase__			(  lowercase		):






			'''simple docstring'''



			lowerCamelCase__   =    VOCAB_FILES_NAMES
			lowerCamelCase__   =    PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__   =    ["""input_ids""", """attention_mask"""]
			def __init__(						self						,   lowercase						,   lowercase						,   lowercase="replace"						,   lowercase="<s>"						,   lowercase="</s>"						,   lowercase="</s>"						,   lowercase="<s>"						,   lowercase="<unk>"						,   lowercase="<pad>"						,   lowercase="<mask>"						,   lowercase=False						,   **lowercase						,   ):
							_lowerCamelCase  :    Dict					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else bos_token
							_lowerCamelCase  :    List[str]					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else eos_token
							_lowerCamelCase  :    Optional[int]					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else sep_token
							_lowerCamelCase  :    Union[str, Any]					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else cls_token
							_lowerCamelCase  :    Union[str, Any]					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else unk_token
							_lowerCamelCase  :    List[str]					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							_lowerCamelCase  :    Union[str, Any]					=					AddedToken(lowercase						,   lstrip=lowercase						,   rstrip=lowercase						) if isinstance(lowercase						,   lowercase						) else mask_token

							super().__init__(
							    errors=lowercase						,   bos_token=lowercase						,   eos_token=lowercase						,   unk_token=lowercase						,   sep_token=lowercase						,   cls_token=lowercase						,   pad_token=lowercase						,   mask_token=lowercase						,   add_prefix_space=lowercase						,   **lowercase						,   )

							with open(lowercase						,   encoding='utf-8'						) as vocab_handle:
											_lowerCamelCase  :    List[str]					=					json.load(lowercase						)
							_lowerCamelCase  :    Union[str, Any]					=					{v: k for k, v in self.encoder.items()}
							_lowerCamelCase  :    Dict					=					errors  # how to handle errors in decoding
							_lowerCamelCase  :    Tuple					=					bytes_to_unicode()
							_lowerCamelCase  :    List[Any]					=					{v: k for k, v in self.byte_encoder.items()}
							with open(lowercase						,   encoding='utf-8'						) as merges_handle:
											_lowerCamelCase  :    int					=					merges_handle.read().split('\n'						)[1:-1]
							_lowerCamelCase  :    Tuple					=					[tuple(merge.split()						) for merge in bpe_merges]
							_lowerCamelCase  :    Tuple					=					dict(zip(lowercase						,   range(len(lowercase						)						)						)						)
							_lowerCamelCase  :    Tuple					=					{}
							_lowerCamelCase  :    Dict					=					add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							_lowerCamelCase  :    str					=					re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'						)
			@property
			# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
			def       A_					(						self						):
							return len(self.encoder						)
			def       A_					(						self						):
							return dict(self.encoder						,   **self.added_tokens_encoder						)
			def       A_					(						self						,   lowercase						):
							if token in self.cache:
											return self.cache[token]
							_lowerCamelCase  :    List[Any]					=					tuple(lowercase						)
							_lowerCamelCase  :    List[Any]					=					get_pairs(lowercase						)

							if not pairs:
											return token

							while True:
											_lowerCamelCase  :    Union[str, Any]					=					min(lowercase						,   key=lambda lowercase						: self.bpe_ranks.get(lowercase						,   float('inf'						)						)						)
											if bigram not in self.bpe_ranks:
															break
											_lowerCamelCase  :    Optional[int]					=					bigram
											_lowerCamelCase  :    Any					=					[]
											_lowerCamelCase  :    List[Any]					=					0
											while i < len(lowercase						):
															try:
																			_lowerCamelCase  :    str					=					word.index(lowercase						,   lowercase						)
															except ValueError:
																			new_word.extend(word[i:]						)
																			break
															else:
																			new_word.extend(word[i:j]						)
																			_lowerCamelCase  :    str					=					j

															if word[i] == first and i < len(lowercase						) - 1 and word[i + 1] == second:
																			new_word.append(first + second						)
																			i += 2
															else:
																			new_word.append(word[i]						)
																			i += 1
											_lowerCamelCase  :    Tuple					=					tuple(lowercase						)
											_lowerCamelCase  :    Tuple					=					new_word
											if len(lowercase						) == 1:
															break
											else:
															_lowerCamelCase  :    List[str]					=					get_pairs(lowercase						)
							_lowerCamelCase  :    List[Any]					=					' '.join(lowercase						)
							_lowerCamelCase  :    str					=					word
							return word
			def       A_					(						self						,   lowercase						):
							_lowerCamelCase  :    Optional[int]					=					[]
							for token in re.findall(self.pat						,   lowercase						):
											_lowerCamelCase  :    int					=					''.join(
											    self.byte_encoder[b] for b in token.encode('utf-8'						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase						).split(' '						)						)
							return bpe_tokens
			def       A_					(						self						,   lowercase						):
							return self.encoder.get(lowercase						,   self.encoder.get(self.unk_token						)						)
			def       A_					(						self						,   lowercase						):
							return self.decoder.get(lowercase						)
			def       A_					(						self						,   lowercase						):
							_lowerCamelCase  :    List[str]					=					''.join(lowercase						)
							_lowerCamelCase  :    Dict					=					bytearray([self.byte_decoder[c] for c in text]						).decode('utf-8'						,   errors=self.errors						)
							return text
			def       A_					(						self						,   lowercase						,   lowercase = None						):
							if not os.path.isdir(lowercase						):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
							_lowerCamelCase  :    Dict					=					os.path.join(
							    lowercase						,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
							_lowerCamelCase  :    Any					=					os.path.join(
							    lowercase						,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

							with open(lowercase						,   'w'						,   encoding='utf-8'						) as f:
											f.write(json.dumps(self.encoder						,   indent=2						,   sort_keys=lowercase						,   ensure_ascii=lowercase						) + '\n'						)

							_lowerCamelCase  :    List[Any]					=					0
							with open(lowercase						,   'w'						,   encoding='utf-8'						) as writer:
											writer.write('#version: 0.2\n'						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,   key=lambda lowercase						: kv[1]						):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    ' Please check that the tokenizer is not corrupted!'						)
																			_lowerCamelCase  :    Optional[Any]					=					token_index
															writer.write(' '.join(lowercase						) + '\n'						)
															index += 1

							return vocab_file, merge_file
			def       A_					(						self						,   lowercase						,   lowercase = None						):
							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_lowerCamelCase  :    Any					=					[self.cls_token_id]
							_lowerCamelCase  :    int					=					[self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep
			def       A_					(						self						,   lowercase						,   lowercase = None						,   lowercase = False						):
							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowercase						,   token_ids_a=lowercase						,   already_has_special_tokens=lowercase						)

							if token_ids_a is None:
											return [1] + ([0] * len(lowercase						)) + [1]
							return [1] + ([0] * len(lowercase						)) + [1, 1] + ([0] * len(lowercase						)) + [1]
			def       A_					(						self						,   lowercase						,   lowercase = None						):
							_lowerCamelCase  :    Dict					=					[self.sep_token_id]
							_lowerCamelCase  :    int					=					[self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
			def       A_					(						self						,   lowercase						,   lowercase=False						,   **lowercase						):
							_lowerCamelCase  :    Union[str, Any]					=					kwargs.pop('add_prefix_space'						,   self.add_prefix_space						)
							if (is_split_into_words or add_prefix_space) and (len(lowercase						) > 0 and not text[0].isspace()):
											_lowerCamelCase  :    int					=					' ' + text
							return (text, kwargs)






			def       A_					(						self						,   lowercase						,   lowercase = None						,   lowercase = PaddingStrategy.DO_NOT_PAD						,   lowercase = None						,   lowercase = None						,   ):
							_lowerCamelCase  :    Dict					=					super()._pad(
							    encoded_inputs=lowercase						,   max_length=lowercase						,   padding_strategy=lowercase						,   pad_to_multiple_of=lowercase						,   return_attention_mask=lowercase						,   )

							# Load from model defaults
							if return_attention_mask is None:
											_lowerCamelCase  :    List[str]					=					'attention_mask' in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
											_lowerCamelCase  :    List[str]					=					encoded_inputs[self.model_input_names[0]]
											# `global_attention_mask` need to have the same length as other (sequential) inputs.
											_lowerCamelCase  :    Any					=					len(encoded_inputs['global_attention_mask']						) != len(lowercase						)

											if needs_to_be_padded:
															_lowerCamelCase  :    int					=					len(lowercase						) - len(encoded_inputs['global_attention_mask']						)

															if self.padding_side == "right":
																			# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																			_lowerCamelCase  :    str					=					(
																			    encoded_inputs['global_attention_mask'] + [-1] * difference
																			)
															elif self.padding_side == "left":
																			_lowerCamelCase  :    Union[str, Any]					=					[-1] * difference + encoded_inputs[
																			    'global_attention_mask'
																			]
															else:
																			raise ValueError('Invalid padding strategy:' + str(self.padding_side						)						)

							return encoded_inputs





"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__  		=		logging.get_logger(__name__)

lowercase__  		=		{
    """google/realm-cc-news-pretrained-embedder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-encoder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-scorer""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-openqa""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"""
    ),
    """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json""",
    """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json""",
    """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json""",
    """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json""",
    # See all REALM models at https://huggingface.co/models?filter=realm
}






class lowerCAmelCase__			(  lowercase		):






			'''simple docstring'''



			lowerCamelCase__   =    """realm"""
			def __init__(						self						,   lowercase=30522						,   lowercase=768						,   lowercase=128						,   lowercase=12						,   lowercase=12						,   lowercase=8						,   lowercase=3072						,   lowercase="gelu_new"						,   lowercase=0.1						,   lowercase=0.1						,   lowercase=512						,   lowercase=2						,   lowercase=0.02						,   lowercase=1E-12						,   lowercase=256						,   lowercase=10						,   lowercase=1E-3						,   lowercase=5						,   lowercase=320						,   lowercase=13353718						,   lowercase=5000						,   lowercase=1						,   lowercase=0						,   lowercase=2						,   **lowercase						,   ):
							super().__init__(pad_token_id=lowercase						,   bos_token_id=lowercase						,   eos_token_id=lowercase						,   **lowercase						)

							# Common config
							_lowerCamelCase  :    str					=					vocab_size
							_lowerCamelCase  :    Dict					=					max_position_embeddings
							_lowerCamelCase  :    int					=					hidden_size
							_lowerCamelCase  :    Optional[Any]					=					retriever_proj_size
							_lowerCamelCase  :    Dict					=					num_hidden_layers
							_lowerCamelCase  :    Any					=					num_attention_heads
							_lowerCamelCase  :    int					=					num_candidates
							_lowerCamelCase  :    List[Any]					=					intermediate_size
							_lowerCamelCase  :    int					=					hidden_act
							_lowerCamelCase  :    Union[str, Any]					=					hidden_dropout_prob
							_lowerCamelCase  :    Dict					=					attention_probs_dropout_prob
							_lowerCamelCase  :    Union[str, Any]					=					initializer_range
							_lowerCamelCase  :    List[Any]					=					type_vocab_size
							_lowerCamelCase  :    int					=					layer_norm_eps

							# Reader config
							_lowerCamelCase  :    Tuple					=					span_hidden_size
							_lowerCamelCase  :    int					=					max_span_width
							_lowerCamelCase  :    Tuple					=					reader_layer_norm_eps
							_lowerCamelCase  :    Union[str, Any]					=					reader_beam_size
							_lowerCamelCase  :    Union[str, Any]					=					reader_seq_len

							# Retrieval config
							_lowerCamelCase  :    Optional[Any]					=					num_block_records
							_lowerCamelCase  :    str					=					searcher_beam_size