



"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowercase__     :   Optional[int]			   =  logging.get_logger(__name__)

lowercase__     :   int			   =  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase__     :   Optional[int]			   =  {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__     :   Tuple			   =  {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

lowercase__     :   List[Any]			   =  {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}




class SCREAMING_SNAKE_CASE__					(		__snake_case      ):
						_lowerCAmelCase							=					VOCAB_FILES_NAMES
						_lowerCAmelCase							=					PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase							=					PRETRAINED_INIT_CONFIGURATION
						_lowerCAmelCase							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase							=					RealmTokenizer



						def __init__(self						,     _lowercase=None						,     _lowercase=None						,     _lowercase=True						,     _lowercase="[UNK]"						,     _lowercase="[SEP]"						,     _lowercase="[PAD]"						,     _lowercase="[CLS]"						,     _lowercase="[MASK]"						,     _lowercase=True						,     _lowercase=None						,     **_lowercase						,     ):





													'''simple docstring'''

													super().__init__(
													    _SCREAMING_SNAKE_CASE						,     tokenizer_file=_SCREAMING_SNAKE_CASE						,     do_lower_case=_SCREAMING_SNAKE_CASE						,     unk_token=_SCREAMING_SNAKE_CASE						,     sep_token=_SCREAMING_SNAKE_CASE						,     pad_token=_SCREAMING_SNAKE_CASE						,     cls_token=_SCREAMING_SNAKE_CASE						,     mask_token=_SCREAMING_SNAKE_CASE						,     tokenize_chinese_chars=_SCREAMING_SNAKE_CASE						,     strip_accents=_SCREAMING_SNAKE_CASE						,     **_SCREAMING_SNAKE_CASE						,     )

													__a		:		Dict									=       json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
													if (
													    normalizer_state.get("""lowercase"""						,     _SCREAMING_SNAKE_CASE							) != do_lower_case
													    or normalizer_state.get("""strip_accents"""						,     _SCREAMING_SNAKE_CASE							) != strip_accents
													    or normalizer_state.get("""handle_chinese_chars"""						,     _SCREAMING_SNAKE_CASE							) != tokenize_chinese_chars
													):
																				__a		:		List[Any]									=       getattr(_SCREAMING_SNAKE_CASE						,     normalizer_state.pop("""type"""							)							)
																				__a		:		str									=       do_lower_case
																				__a		:		List[Any]									=       strip_accents
																				__a		:		List[Any]									=       tokenize_chinese_chars
																				__a		:		Tuple									=       normalizer_class(**_SCREAMING_SNAKE_CASE							)

													__a		:		Optional[int]									=       do_lower_case



						def     lowerCAmelCase__(self						,     _lowercase						,     **_lowercase							):





													'''simple docstring'''

													__a		:		List[Any]									=       PaddingStrategy.MAX_LENGTH

													__a		:		Union[str, Any]									=       text
													__a		:		Optional[Any]									=       kwargs.pop("""text_pair"""						,     _SCREAMING_SNAKE_CASE							)
													__a		:		int									=       kwargs.pop("""return_tensors"""						,     _SCREAMING_SNAKE_CASE							)

													__a		:		Optional[Any]									=       {
													    """input_ids""": [],
													    """attention_mask""": [],
													    """token_type_ids""": [],
													}

													for idx, candidate_text in enumerate(_SCREAMING_SNAKE_CASE							):
																				if batch_text_pair is not None:
																											__a		:		str									=       batch_text_pair[idx]
																				else:
																											__a		:		Optional[Any]									=       None

																				__a		:		Optional[Any]									=       super().__call__(_SCREAMING_SNAKE_CASE						,     _SCREAMING_SNAKE_CASE						,     return_tensors=_SCREAMING_SNAKE_CASE						,     **_SCREAMING_SNAKE_CASE							)

																				__a		:		Union[str, Any]									=       encoded_candidates.get("""input_ids"""							)
																				__a		:		Any									=       encoded_candidates.get("""attention_mask"""							)
																				__a		:		Any									=       encoded_candidates.get("""token_type_ids"""							)

																				if encoded_input_ids is not None:
																											output_data["input_ids"].append(_SCREAMING_SNAKE_CASE							)
																				if encoded_attention_mask is not None:
																											output_data["attention_mask"].append(_SCREAMING_SNAKE_CASE							)
																				if encoded_token_type_ids is not None:
																											output_data["token_type_ids"].append(_SCREAMING_SNAKE_CASE							)

													__a		:		List[Any]									=       {key: item for key, item in output_data.items() if len(_SCREAMING_SNAKE_CASE							) != 0}

													return BatchEncoding(_SCREAMING_SNAKE_CASE						,     tensor_type=_SCREAMING_SNAKE_CASE							)



						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase=None							):





													'''simple docstring'''

													__a		:		Optional[int]									=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

													if token_ids_a:
																				output += token_ids_a + [self.sep_token_id]

													return output



						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase = None							):





													'''simple docstring'''

													__a		:		List[str]									=       [self.sep_token_id]
													__a		:		Tuple									=       [self.cls_token_id]
													if token_ids_a is None:
																				return len(cls + token_ids_a + sep							) * [0]
													return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase = None							):





													'''simple docstring'''

													__a		:		int									=       self._tokenizer.model.save(_SCREAMING_SNAKE_CASE						,     name=_SCREAMING_SNAKE_CASE							)
													return tuple(_SCREAMING_SNAKE_CASE							)






"""simple docstring"""

import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class SCREAMING_SNAKE_CASE__					(		__snake_case     ,					unittest.TestCase      ):
						_lowerCAmelCase							=					LEDTokenizer
						_lowerCAmelCase							=					LEDTokenizerFast
						_lowerCAmelCase							=					True



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													super().setUp()
													__a		:		str									=       [
													    """l""",
													    """o""",
													    """w""",
													    """e""",
													    """r""",
													    """s""",
													    """t""",
													    """i""",
													    """d""",
													    """n""",
													    """\u0120""",
													    """\u0120l""",
													    """\u0120n""",
													    """\u0120lo""",
													    """\u0120low""",
													    """er""",
													    """\u0120lowest""",
													    """\u0120newer""",
													    """\u0120wider""",
													    """<unk>""",
													]
													__a		:		int									=       dict(zip(_lowercase						,     range(len(_lowercase							)							)							)							)
													__a		:		Optional[int]									=       ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
													__a		:		List[Any]									=       {"""unk_token""": """<unk>"""}

													__a		:		Union[str, Any]									=       os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["""vocab_file"""]							)
													__a		:		Union[str, Any]									=       os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES["""merges_file"""]							)
													with open(self.vocab_file						,     """w"""						,     encoding="""utf-8"""							) as fp:
																				fp.write(json.dumps(_lowercase							) + """\n"""							)
													with open(self.merges_file						,     """w"""						,     encoding="""utf-8"""							) as fp:
																				fp.write("""\n""".join(_lowercase							)							)



						def     lowerCAmelCase__(self						,     **_lowercase							):





													'''simple docstring'''

													kwargs.update(self.special_tokens_map							)
													return self.tokenizer_class.from_pretrained(self.tmpdirname						,     **_lowercase							)



						def     lowerCAmelCase__(self						,     **_lowercase							):





													'''simple docstring'''

													kwargs.update(self.special_tokens_map							)
													return self.rust_tokenizer_class.from_pretrained(self.tmpdirname						,     **_lowercase							)



						def     lowerCAmelCase__(self						,     _lowercase							):





													'''simple docstring'''

													return "lower newer", "lower newer"



						@cached_property
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													return LEDTokenizer.from_pretrained("""allenai/led-base-16384"""							)



						@cached_property
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													return LEDTokenizerFast.from_pretrained("""allenai/led-base-16384"""							)



						@require_torch
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Any									=       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
													__a		:		List[str]									=       [0, 250, 251, 17818, 13, 39186, 1938, 4, 2]

													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Optional[int]									=       tokenizer(_lowercase						,     max_length=len(_lowercase							)						,     padding=_lowercase						,     return_tensors="""pt"""							)
																				self.assertIsInstance(_lowercase						,     _lowercase							)

																				self.assertEqual((2, 9)						,     batch.input_ids.shape							)
																				self.assertEqual((2, 9)						,     batch.attention_mask.shape							)
																				__a		:		Dict									=       batch.input_ids.tolist()[0]
																				self.assertListEqual(_lowercase						,     _lowercase							)



						@require_torch
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Union[str, Any]									=       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Tuple									=       tokenizer(_lowercase						,     padding=_lowercase						,     return_tensors="""pt"""							)
																				self.assertIn("""input_ids"""						,     _lowercase							)
																				self.assertIn("""attention_mask"""						,     _lowercase							)
																				self.assertNotIn("""labels"""						,     _lowercase							)
																				self.assertNotIn("""decoder_attention_mask"""						,     _lowercase							)



						@require_torch
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Optional[Any]									=       [
													    """Summary of the text.""",
													    """Another summary.""",
													]
													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Dict									=       tokenizer(text_target=_lowercase						,     max_length=32						,     padding="""max_length"""						,     return_tensors="""pt"""							)
																				self.assertEqual(32						,     targets["""input_ids"""].shape[1]							)



						@require_torch
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Optional[int]									=       tokenizer(
																				    ["""I am a small frog""" * 1024, """I am a small frog"""]						,     padding=_lowercase						,     truncation=_lowercase						,     return_tensors="""pt"""							)
																				self.assertIsInstance(_lowercase						,     _lowercase							)
																				self.assertEqual(batch.input_ids.shape						,     (2, 5122)							)



						@require_torch
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Tuple									=       ["""A long paragraph for summarization."""]
													__a		:		Dict									=       [
													    """Summary of the text.""",
													]
													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		int									=       tokenizer(_lowercase						,     return_tensors="""pt"""							)
																				__a		:		Dict									=       tokenizer(text_target=_lowercase						,     return_tensors="""pt"""							)
																				__a		:		List[str]									=       inputs["""input_ids"""]
																				__a		:		List[Any]									=       targets["""input_ids"""]
																				self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()							)
																				self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()							)
																				self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()							)
																				self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()							)



						@require_torch
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																				__a		:		Optional[Any]									=       ["""Summary of the text.""", """Another summary."""]
																				__a		:		List[Any]									=       [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

																				__a		:		Union[str, Any]									=       tokenizer(_lowercase						,     padding=_lowercase							)
																				__a		:		Tuple									=       [[0] * len(_lowercase							) for x in encoded_output["""input_ids"""]]
																				__a		:		Union[str, Any]									=       tokenizer.pad(_lowercase							)
																				self.assertSequenceEqual(outputs["""global_attention_mask"""]						,     _lowercase							)



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													pass





						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																											__a		:		Dict									=       self.rust_tokenizer_class.from_pretrained(_lowercase						,     **_lowercase							)
																											__a		:		Union[str, Any]									=       self.tokenizer_class.from_pretrained(_lowercase						,     **_lowercase							)
																											__a		:		Union[str, Any]									=       """A, <mask> AllenNLP sentence."""
																											__a		:		Dict									=       tokenizer_r.encode_plus(_lowercase						,     add_special_tokens=_lowercase						,     return_token_type_ids=_lowercase							)
																											__a		:		Tuple									=       tokenizer_p.encode_plus(_lowercase						,     add_special_tokens=_lowercase						,     return_token_type_ids=_lowercase							)
																											self.assertEqual(sum(tokens_r["""token_type_ids"""]							)						,     sum(tokens_p["""token_type_ids"""]							)							)
																											self.assertEqual(
																											    sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)						,     sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)						,     )

																											__a		:		Tuple									=       tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
																											__a		:		Any									=       tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)
																											self.assertSequenceEqual(tokens_p["""input_ids"""]						,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)
																											self.assertSequenceEqual(tokens_r["""input_ids"""]						,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)

																											self.assertSequenceEqual(
																											    _lowercase						,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
																											self.assertSequenceEqual(
																											    _lowercase						,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)

