




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase_						=logging.get_logger(__name__)

UpperCAmelCase_						={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_						={
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase_						={
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

UpperCAmelCase_						={
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Union[str, Any]										=VOCAB_FILES_NAMES
				__a							: int										=PRETRAINED_VOCAB_FILES_MAP
				__a							: Any										=PRETRAINED_INIT_CONFIGURATION
				__a							: Optional[int]										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a							: Optional[int]										=RealmTokenizer



				def __init__( self       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=True       , UpperCAmelCase_="[UNK]"       , UpperCAmelCase_="[SEP]"       , UpperCAmelCase_="[PAD]"       , UpperCAmelCase_="[CLS]"       , UpperCAmelCase_="[MASK]"       , UpperCAmelCase_=True       , UpperCAmelCase_=None       , **UpperCAmelCase_       , ):
											super().__init__(
											    UpperCAmelCase_       , tokenizer_file=UpperCAmelCase_       , do_lower_case=UpperCAmelCase_       , unk_token=UpperCAmelCase_       , sep_token=UpperCAmelCase_       , pad_token=UpperCAmelCase_       , cls_token=UpperCAmelCase_       , mask_token=UpperCAmelCase_       , tokenize_chinese_chars=UpperCAmelCase_       , strip_accents=UpperCAmelCase_       , **UpperCAmelCase_       , )

											lowerCAmelCase   =		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
											if (
											    normalizer_state.get('''lowercase'''       , UpperCAmelCase_						) != do_lower_case
											    or normalizer_state.get('''strip_accents'''       , UpperCAmelCase_						) != strip_accents
											    or normalizer_state.get('''handle_chinese_chars'''       , UpperCAmelCase_						) != tokenize_chinese_chars
											):
																		lowerCAmelCase   =		getattr(UpperCAmelCase_       , normalizer_state.pop('''type'''						)						)
																		lowerCAmelCase   =		do_lower_case
																		lowerCAmelCase   =		strip_accents
																		lowerCAmelCase   =		tokenize_chinese_chars
																		lowerCAmelCase   =		normalizer_class(**UpperCAmelCase_						)

											lowerCAmelCase   =		do_lower_case



				def       __snake_case		( self       , UpperCAmelCase_       , **UpperCAmelCase_						):
											lowerCAmelCase   =		PaddingStrategy.MAX_LENGTH

											lowerCAmelCase   =		text
											lowerCAmelCase   =		kwargs.pop('''text_pair'''       , UpperCAmelCase_						)
											lowerCAmelCase   =		kwargs.pop('''return_tensors'''       , UpperCAmelCase_						)

											lowerCAmelCase   =		{
											    '''input_ids''': [],
											    '''attention_mask''': [],
											    '''token_type_ids''': [],
											}

											for idx, candidate_text in enumerate(UpperCAmelCase_						):
																		if batch_text_pair is not None:
																									lowerCAmelCase   =		batch_text_pair[idx]
																		else:
																									lowerCAmelCase   =		None

																		lowerCAmelCase   =		super().__call__(UpperCAmelCase_       , UpperCAmelCase_       , return_tensors=UpperCAmelCase_       , **UpperCAmelCase_						)

																		lowerCAmelCase   =		encoded_candidates.get('''input_ids'''						)
																		lowerCAmelCase   =		encoded_candidates.get('''attention_mask'''						)
																		lowerCAmelCase   =		encoded_candidates.get('''token_type_ids'''						)

																		if encoded_input_ids is not None:
																									output_data["input_ids"].append(UpperCAmelCase_						)
																		if encoded_attention_mask is not None:
																									output_data["attention_mask"].append(UpperCAmelCase_						)
																		if encoded_token_type_ids is not None:
																									output_data["token_type_ids"].append(UpperCAmelCase_						)

											lowerCAmelCase   =		{key: item for key, item in output_data.items() if len(UpperCAmelCase_						) != 0}

											return BatchEncoding(UpperCAmelCase_       , tensor_type=UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_=None						):
											lowerCAmelCase   =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		[self.sep_token_id]
											lowerCAmelCase   =		[self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		self._tokenizer.model.save(UpperCAmelCase_       , name=UpperCAmelCase_						)
											return tuple(UpperCAmelCase_						)









from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


UpperCAmelCase_						=logging.get_logger(__name__)


class 							__UpperCamelCase			( __UpperCAmelCase     ,					__UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Optional[Any]										="""maskformer-swin"""

				__a							: Optional[int]										={
				    """num_attention_heads""": """num_heads""",
				    """num_hidden_layers""": """num_layers""",
				}



				def __init__( self       , UpperCAmelCase_=2_24       , UpperCAmelCase_=4       , UpperCAmelCase_=3       , UpperCAmelCase_=96       , UpperCAmelCase_=[2, 2, 6, 2]       , UpperCAmelCase_=[3, 6, 12, 24]       , UpperCAmelCase_=7       , UpperCAmelCase_=4.0       , UpperCAmelCase_=True       , UpperCAmelCase_=0.0       , UpperCAmelCase_=0.0       , UpperCAmelCase_=0.1       , UpperCAmelCase_="gelu"       , UpperCAmelCase_=False       , UpperCAmelCase_=0.02       , UpperCAmelCase_=1E-5       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , **UpperCAmelCase_       , ):
											super().__init__(**UpperCAmelCase_						)

											lowerCAmelCase   =		image_size
											lowerCAmelCase   =		patch_size
											lowerCAmelCase   =		num_channels
											lowerCAmelCase   =		embed_dim
											lowerCAmelCase   =		depths
											lowerCAmelCase   =		len(UpperCAmelCase_						)
											lowerCAmelCase   =		num_heads
											lowerCAmelCase   =		window_size
											lowerCAmelCase   =		mlp_ratio
											lowerCAmelCase   =		qkv_bias
											lowerCAmelCase   =		hidden_dropout_prob
											lowerCAmelCase   =		attention_probs_dropout_prob
											lowerCAmelCase   =		drop_path_rate
											lowerCAmelCase   =		hidden_act
											lowerCAmelCase   =		use_absolute_embeddings
											lowerCAmelCase   =		layer_norm_eps
											lowerCAmelCase   =		initializer_range
											# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
											# this indicates the channel dimension after the last stage of the model
											lowerCAmelCase   =		int(embed_dim * 2 ** (len(UpperCAmelCase_						) - 1)						)
											lowerCAmelCase   =		['''stem'''] + [F"""stage{idx}""" for idx in range(1       , len(UpperCAmelCase_						) + 1						)]
											lowerCAmelCase   ,     lowerCAmelCase   =		get_aligned_output_features_output_indices(
											    out_features=UpperCAmelCase_       , out_indices=UpperCAmelCase_       , stage_names=self.stage_names						)



