






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__UpperCAmelCase							:   List[str]							=  logging.get_logger(__name__)

__UpperCAmelCase							:   Dict							=  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase							:   Optional[int]							=  {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCAmelCase							:   int							=  {
    "google/realm-cc-news-pretrained-embedder": 5_1_2,
    "google/realm-cc-news-pretrained-encoder": 5_1_2,
    "google/realm-cc-news-pretrained-scorer": 5_1_2,
    "google/realm-cc-news-pretrained-openqa": 5_1_2,
    "google/realm-orqa-nq-openqa": 5_1_2,
    "google/realm-orqa-nq-reader": 5_1_2,
    "google/realm-orqa-wq-openqa": 5_1_2,
    "google/realm-orqa-wq-reader": 5_1_2,
}

__UpperCAmelCase							:   List[Any]							=  {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_INIT_CONFIGURATION
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   RealmTokenizer






	def __init__(		self					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase=True					,UpperCamelCase="[UNK]"					,UpperCamelCase="[SEP]"					,UpperCamelCase="[PAD]"					,UpperCamelCase="[CLS]"					,UpperCamelCase="[MASK]"					,UpperCamelCase=True					,UpperCamelCase=None					,**UpperCamelCase					,)						->    Optional[Any]:
					super().__init__(
					    UpperCamelCase					,tokenizer_file=UpperCamelCase					,do_lower_case=UpperCamelCase					,unk_token=UpperCamelCase					,sep_token=UpperCamelCase					,pad_token=UpperCamelCase					,cls_token=UpperCamelCase					,mask_token=UpperCamelCase					,tokenize_chinese_chars=UpperCamelCase					,strip_accents=UpperCamelCase					,**UpperCamelCase					,)

					snake_case__      :List[str]         =   json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
					if (
					    normalizer_state.get("lowercase"					,UpperCamelCase       ) != do_lower_case
					    or normalizer_state.get("strip_accents"					,UpperCamelCase       ) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"					,UpperCamelCase       ) != tokenize_chinese_chars
					):
									snake_case__      :str         =   getattr(UpperCamelCase					,normalizer_state.pop("type"       )       )
									snake_case__      :Tuple         =   do_lower_case
									snake_case__      :Union[str, Any]         =   strip_accents
									snake_case__      :List[Any]         =   tokenize_chinese_chars
									snake_case__      :Any         =   normalizer_class(**UpperCamelCase       )

					snake_case__      :Any         =   do_lower_case






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,**UpperCamelCase       )						->    List[Any]:
					snake_case__      :Optional[int]         =   PaddingStrategy.MAX_LENGTH

					snake_case__      :Any         =   text
					snake_case__      :List[Any]         =   kwargs.pop("text_pair"					,UpperCamelCase       )
					snake_case__      :Optional[int]         =   kwargs.pop("return_tensors"					,UpperCamelCase       )

					snake_case__      :List[Any]         =   {
					    "input_ids": [],
					    "attention_mask": [],
					    "token_type_ids": [],
					}

					for idx, candidate_text in enumerate(UpperCamelCase       ):
									if batch_text_pair is not None:
													snake_case__      :Tuple         =   batch_text_pair[idx]
									else:
													snake_case__      :str         =   None

									snake_case__      :Optional[Any]         =   super().__call__(UpperCamelCase					,UpperCamelCase					,return_tensors=UpperCamelCase					,**UpperCamelCase       )

									snake_case__      :Union[str, Any]         =   encoded_candidates.get("input_ids"       )
									snake_case__      :Optional[Any]         =   encoded_candidates.get("attention_mask"       )
									snake_case__      :int         =   encoded_candidates.get("token_type_ids"       )

									if encoded_input_ids is not None:
													output_data["input_ids"].append(UpperCamelCase       )
									if encoded_attention_mask is not None:
													output_data["attention_mask"].append(UpperCamelCase       )
									if encoded_token_type_ids is not None:
													output_data["token_type_ids"].append(UpperCamelCase       )

					snake_case__      :str         =   {key: item for key, item in output_data.items() if len(UpperCamelCase       ) != 0}

					return BatchEncoding(UpperCamelCase					,tensor_type=UpperCamelCase       )






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase=None       )						->    str:
					snake_case__      :List[str]         =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :List[str]         =   [self.sep_token_id]
					snake_case__      :str         =   [self.cls_token_id]
					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					snake_case__      :Tuple         =   self._tokenizer.model.save(UpperCamelCase					,name=UpperCamelCase       )
					return tuple(UpperCamelCase       )







import pytest


__UpperCAmelCase							:   int							=  "__dummy_dataset1__"

__UpperCAmelCase							:   int							=  "\nimport json\nimport os\n\nimport datasets\n\n\nREPO_URL = \"https://huggingface.co/datasets/albertvillanova/tests-raw-jsonl/resolve/main/\"\nURLS = {\"train\": REPO_URL + \"wikiann-bn-train.jsonl\", \"validation\": REPO_URL + \"wikiann-bn-validation.jsonl\"}\n\n\nclass __DummyDataset1__(datasets.GeneratorBasedBuilder):\n\n    def _info(self):\n        features = datasets.Features(\n            {\n                \"tokens\": datasets.Sequence(datasets.Value(\"string\")),\n                \"ner_tags\": datasets.Sequence(\n                    datasets.features.ClassLabel(\n                        names=[\n                            \"O\",\n                            \"B-PER\",\n                            \"I-PER\",\n                            \"B-ORG\",\n                            \"I-ORG\",\n                            \"B-LOC\",\n                            \"I-LOC\",\n                        ]\n                    )\n                ),\n                \"langs\": datasets.Sequence(datasets.Value(\"string\")),\n                \"spans\": datasets.Sequence(datasets.Value(\"string\")),\n            }\n        )\n        return datasets.DatasetInfo(features=features)\n\n    def _split_generators(self, dl_manager):\n        dl_path = dl_manager.download(URLS)\n        return [\n            datasets.SplitGenerator(datasets.Split.TRAIN, gen_kwargs={\"filepath\": dl_path[\"train\"]}),\n            datasets.SplitGenerator(datasets.Split.VALIDATION, gen_kwargs={\"filepath\": dl_path[\"validation\"]}),\n        ]\n\n    def _generate_examples(self, filepath):\n        with open(filepath, \"r\", encoding=\"utf-8\") as f:\n            for i, line in enumerate(f):\n                yield i, json.loads(line)\n"





@pytest.fixture
def 					lowercase_ (    )     -> Optional[Any]:





				'''simple docstring'''


				return DATASET_LOADING_SCRIPT_NAME





@pytest.fixture
def 					lowercase_ (    )     -> Optional[int]:





				'''simple docstring'''


				return DATASET_LOADING_SCRIPT_CODE





@pytest.fixture
def 					lowercase_ (    __snake_case     :							Optional[int]	,	__snake_case     :							List[Any]	,	__snake_case     :							Any      )     -> Dict:





				'''simple docstring'''


				snake_case__      :Optional[Any]         =   dataset_loading_script_name
				snake_case__      :Optional[Any]         =   tmp_path / "datasets" / script_name
				script_dir.mkdir(parents=__snake_case      )
				snake_case__      :List[Any]         =   script_dir / F'{script_name}.py'
				with open(__snake_case	,	"w"      ) as f:
								f.write(__snake_case      )
				return str(__snake_case      )