



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowercase_								=     logging.get_logger(__name__)

lowercase_								=     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase_								=     {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase_								=     {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowercase_								=     {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}
class 				_snake_case			(					lowercase__):
				UpperCamelCase__				:					Optional[Any]     		=VOCAB_FILES_NAMES
				UpperCamelCase__				:					List[Any]     		=PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase__				:					List[str]     		=PRETRAINED_INIT_CONFIGURATION
				UpperCamelCase__				:					List[Any]     		=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase__				:					Any     		=RealmTokenizer




				def __init__(						self					:						List[Any],					__lowercase					:						Union[str, Any]=None,					__lowercase					:						Dict=None,					__lowercase					:						List[Any]=True,					__lowercase					:						List[str]="[UNK]",					__lowercase					:						int="[SEP]",					__lowercase					:						int="[PAD]",					__lowercase					:						List[str]="[CLS]",					__lowercase					:						Dict="[MASK]",					__lowercase					:						List[Any]=True,					__lowercase					:						Union[str, Any]=None,					**__lowercase					:						Tuple,					):
							super().__init__(
							    __lowercase,					tokenizer_file=__lowercase,					do_lower_case=__lowercase,					unk_token=__lowercase,					sep_token=__lowercase,					pad_token=__lowercase,					cls_token=__lowercase,					mask_token=__lowercase,					tokenize_chinese_chars=__lowercase,					strip_accents=__lowercase,					**__lowercase,					)

							lowercase__					  =				json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
							if (
							    normalizer_state.get("lowercase",					__lowercase						) != do_lower_case
							    or normalizer_state.get("strip_accents",					__lowercase						) != strip_accents
							    or normalizer_state.get("handle_chinese_chars",					__lowercase						) != tokenize_chinese_chars
							):
										lowercase__					  =				getattr(__lowercase,					normalizer_state.pop("type"						)						)
										lowercase__					  =				do_lower_case
										lowercase__					  =				strip_accents
										lowercase__					  =				tokenize_chinese_chars
										lowercase__					  =				normalizer_class(**__lowercase						)

							lowercase__					  =				do_lower_case




				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						Dict,					**__lowercase					:						Union[str, Any]						):
							lowercase__					  =				PaddingStrategy.MAX_LENGTH

							lowercase__					  =				text
							lowercase__					  =				kwargs.pop("text_pair",					__lowercase						)
							lowercase__					  =				kwargs.pop("return_tensors",					__lowercase						)

							lowercase__					  =				{
							    "input_ids": [],
							    "attention_mask": [],
							    "token_type_ids": [],
							}

							for idx, candidate_text in enumerate(__lowercase						):
										if batch_text_pair is not None:
													lowercase__					  =				batch_text_pair[idx]
										else:
													lowercase__					  =				None

										lowercase__					  =				super().__call__(__lowercase,					__lowercase,					return_tensors=__lowercase,					**__lowercase						)

										lowercase__					  =				encoded_candidates.get("input_ids"						)
										lowercase__					  =				encoded_candidates.get("attention_mask"						)
										lowercase__					  =				encoded_candidates.get("token_type_ids"						)

										if encoded_input_ids is not None:
													output_data["input_ids"].append(__lowercase						)
										if encoded_attention_mask is not None:
													output_data["attention_mask"].append(__lowercase						)
										if encoded_token_type_ids is not None:
													output_data["token_type_ids"].append(__lowercase						)

							lowercase__					  =				{key: item for key, item in output_data.items() if len(__lowercase						) != 0}

							return BatchEncoding(__lowercase,					tensor_type=__lowercase						)




				def 							A__      (						self					:						List[Any],					__lowercase					:						Any,					__lowercase					:						List[str]=None						):
							lowercase__					  =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

							return output




				def 							A__      (						self					:						List[str],					__lowercase					:						List[int],					__lowercase					:						Optional[List[int]] = None						):
							lowercase__					  =				[self.sep_token_id]
							lowercase__					  =				[self.cls_token_id]
							if token_ids_a is None:
										return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




				def 							A__      (						self					:						Any,					__lowercase					:						str,					__lowercase					:						Optional[str] = None						):
							lowercase__					  =				self._tokenizer.model.save(__lowercase,					name=__lowercase						)
							return tuple(__lowercase						)









import re
from pathlib import Path
from unittest import TestCase

import pytest
@pytest.mark.integration
class 				_snake_case			(					lowercase__):




				def 							A__      (						self					:						Optional[Any],					__lowercase					:						str						):

							with open(__lowercase,					encoding="utf-8"						) as input_file:
										lowercase__					  =				re.compile(R"(?!.*\b(?:encoding|rb|w|wb|w+|wb+|ab|ab+)\b)(?<=\s)(open)\((.*)\)"						)
										lowercase__					  =				input_file.read()
										lowercase__					  =				regexp.search(__lowercase						)

							return match




				def 							A__      (						self					:						str,					__lowercase					:						str						):

							with open(__lowercase,					encoding="utf-8"						) as input_file:
										lowercase__					  =				re.compile(R"#[^\r\n]*print\(|\"[^\r\n]*print\(|\"\"\".*?print\(.*?\"\"\"|(print\()",					re.DOTALL						)
										lowercase__					  =				input_file.read()
										# use `re.finditer` to handle the case where the ignored groups would be matched first by `re.search`
										lowercase__					  =				regexp.finditer(__lowercase						)

							lowercase__					  =				[match for match in matches if match is not None and match.group(1						) is not None]
							return matches[0] if matches else None




				def 							A__      (						self					:						Union[str, Any]						):
							lowercase__					  =				Path("./datasets"						)
							lowercase__					  =				list(dataset_paths.absolute().glob("**/*.py"						)						)

							for dataset in dataset_files:
										if self._no_encoding_on_file_open(str(__lowercase						)						):
													raise AssertionError(F'''open(...) must use utf-8 encoding in {dataset}'''						)




				def 							A__      (						self					:						Union[str, Any]						):
							lowercase__					  =				Path("./datasets"						)
							lowercase__					  =				list(dataset_paths.absolute().glob("**/*.py"						)						)

							for dataset in dataset_files:
										if self._no_print_statements(str(__lowercase						)						):
													raise AssertionError(F'''print statement found in {dataset}. Use datasets.logger/logging instead.'''						)




