




import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFGPTaLMHeadModel, is_keras_nlp_available, is_tf_available
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import require_keras_nlp, require_tf, slow


if is_tf_available():
     import tensorflow as tf

if is_keras_nlp_available():
     from transformers.models.gpta import TFGPTaTokenizer


_UpperCAmelCase	:						Optional[Any]			=     ["gpt2"]
_UpperCAmelCase	:						Optional[Any]			=     "gpt2"

if is_tf_available():




     class    __lowerCAmelCase       (							tf.Module):






            def __init__(   self:					Dict       ,					_lowerCAmelCase:					List[str]				):
              super().__init__()
              lowercase    :List[str]      	=						tokenizer
              lowercase    :Any      	=						AutoConfig.from_pretrained(_lowerCAmelCase				)
              lowercase    :List[str]      	=						TFGPTaLMHeadModel.from_config(_lowerCAmelCase				)






            @tf.function(input_signature=(tf.TensorSpec((None,)       ,					tf.string       ,					name="text"				),)				)
            def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]       ,					_lowerCAmelCase:					str				):
              lowercase    :Any      	=						self.tokenizer(_lowerCAmelCase				)
              lowercase    :Tuple      	=						tokenized["input_ids"].to_tensor()

              lowercase    :List[Any]      	=						tf.cast(input_ids_dense > 0       ,					tf.intaa				)
              # input_mask = tf.reshape(input_mask, [-1, MAX_SEQ_LEN])

              lowercase    :Union[str, Any]      	=						self.model(input_ids=_lowerCAmelCase       ,					attention_mask=_lowerCAmelCase				)["logits"]

              return outputs




@require_tf
@require_keras_nlp
class    __lowerCAmelCase       (							unittest.TestCase):






       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         super().setUp()

         lowercase    :List[str]      	=						[GPTaTokenizer.from_pretrained(_lowerCAmelCase				) for checkpoint in (TOKENIZER_CHECKPOINTS)]
         lowercase    :Tuple      	=						[TFGPTaTokenizer.from_pretrained(_lowerCAmelCase				) for checkpoint in TOKENIZER_CHECKPOINTS]
         assert len(self.tokenizers				) == len(self.tf_tokenizers				)

         lowercase    :Optional[Any]      	=						[
             "This is a straightforward English test sentence.",
             "This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.",
             "Now we're going to add some Chinese: 一 二 三 一二三",
             "And some much more rare Chinese: 齉 堃 齉堃",
             "Je vais aussi écrire en français pour tester les accents",
             "Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ",
         ]
         lowercase    :Optional[Any]      	=						list(zip(self.test_sentences       ,					self.test_sentences[::-1]				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         for tokenizer, tf_tokenizer in zip(self.tokenizers       ,					self.tf_tokenizers				):
           for test_inputs in self.test_sentences:
             lowercase    :Any      	=						tokenizer([test_inputs]       ,					return_tensors="tf"				)
             lowercase    :List[str]      	=						tf_tokenizer([test_inputs]				)

             for key in python_outputs.keys():
               # convert them to numpy to avoid messing with ragged tensors
               lowercase    :List[Any]      	=						python_outputs[key].numpy()
               lowercase    :str      	=						tf_outputs[key].numpy()

               self.assertTrue(tf.reduce_all(python_outputs_values.shape == tf_outputs_values.shape				)				)
               self.assertTrue(tf.reduce_all(tf.cast(_lowerCAmelCase       ,					tf.intaa				) == tf_outputs_values				)				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         for tf_tokenizer in self.tf_tokenizers:
           lowercase    :Any      	=						tf.function(_lowerCAmelCase				)
           for test_inputs in self.test_sentences:
             lowercase    :Dict      	=						tf.constant(_lowerCAmelCase				)
             lowercase    :str      	=						compiled_tokenizer(_lowerCAmelCase				)
             lowercase    :Optional[Any]      	=						tf_tokenizer(_lowerCAmelCase				)

             for key in eager_outputs.keys():
               self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]				)				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         for tf_tokenizer in self.tf_tokenizers:
           lowercase    :List[str]      	=						ModelToSave(tokenizer=_lowerCAmelCase				)
           lowercase    :Tuple      	=						tf.convert_to_tensor([self.test_sentences[0]]				)
           lowercase    :Optional[Any]      	=						model.serving(_lowerCAmelCase				)  # Build model with some sample inputs
           with TemporaryDirectory() as tempdir:
             lowercase    :Optional[Any]      	=						Path(_lowerCAmelCase				) / "saved.model"
             tf.saved_model.save(_lowerCAmelCase       ,					_lowerCAmelCase       ,					signatures={"serving_default": model.serving}				)
             lowercase    :Optional[Any]      	=						tf.saved_model.load(_lowerCAmelCase				)
           lowercase    :List[str]      	=						loaded_model.signatures["serving_default"](_lowerCAmelCase				)["output_0"]
           # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
           self.assertTrue(tf.reduce_all(out == loaded_output				)				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         for tf_tokenizer in self.tf_tokenizers:
           lowercase    :Dict      	=						tf.convert_to_tensor([self.test_sentences[0]]				)
           lowercase    :Optional[int]      	=						tf_tokenizer(_lowerCAmelCase				)  # Build model with some sample inputs

           lowercase    :List[str]      	=						tf_tokenizer.get_config()
           lowercase    :Any      	=						TFGPTaTokenizer.from_config(_lowerCAmelCase				)
           lowercase    :Optional[int]      	=						model_from_config(_lowerCAmelCase				)

           for key in from_config_output.keys():
             self.assertTrue(tf.reduce_all(from_config_output[key] == out[key]				)				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         for tf_tokenizer in self.tf_tokenizers:
           # for the test to run
           lowercase    :List[str]      	=						12_31_23

           for max_length in [3, 5, 10_24]:
             lowercase    :Optional[int]      	=						tf.convert_to_tensor([self.test_sentences[0]]				)
             lowercase    :str      	=						tf_tokenizer(_lowerCAmelCase       ,					max_length=_lowerCAmelCase				)

             lowercase    :str      	=						out["input_ids"].numpy().shape[1]

             assert out_length == max_length









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase	:						Optional[int]			=     logging.get_logger(__name__)

_UpperCAmelCase	:						Union[str, Any]			=     {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase	:						Optional[Any]			=     {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

_UpperCAmelCase	:						Tuple			=     {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

_UpperCAmelCase	:						Dict			=     {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						VOCAB_FILES_NAMES
       _a 					=						PRETRAINED_VOCAB_FILES_MAP
       _a 					=						PRETRAINED_INIT_CONFIGURATION
       _a 					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a 					=						RealmTokenizer






       def __init__(   self:					Tuple       ,					_lowerCAmelCase:					int=None       ,					_lowerCAmelCase:					List[Any]=None       ,					_lowerCAmelCase:					Optional[int]=True       ,					_lowerCAmelCase:					Optional[Any]="[UNK]"       ,					_lowerCAmelCase:					Dict="[SEP]"       ,					_lowerCAmelCase:					List[str]="[PAD]"       ,					_lowerCAmelCase:					Optional[int]="[CLS]"       ,					_lowerCAmelCase:					List[Any]="[MASK]"       ,					_lowerCAmelCase:					Tuple=True       ,					_lowerCAmelCase:					Tuple=None       ,					**_lowerCAmelCase:					Union[str, Any]       ,					):
         super().__init__(
             _lowerCAmelCase       ,					tokenizer_file=_lowerCAmelCase       ,					do_lower_case=_lowerCAmelCase       ,					unk_token=_lowerCAmelCase       ,					sep_token=_lowerCAmelCase       ,					pad_token=_lowerCAmelCase       ,					cls_token=_lowerCAmelCase       ,					mask_token=_lowerCAmelCase       ,					tokenize_chinese_chars=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)

         lowercase    :List[str]      	=						json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             normalizer_state.get("lowercase"       ,					_lowerCAmelCase				) != do_lower_case
             or normalizer_state.get("strip_accents"       ,					_lowerCAmelCase				) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,					_lowerCAmelCase				) != tokenize_chinese_chars
         ):
           lowercase    :List[str]      	=						getattr(_lowerCAmelCase       ,					normalizer_state.pop("type"				)				)
           lowercase    :List[str]      	=						do_lower_case
           lowercase    :Any      	=						strip_accents
           lowercase    :Tuple      	=						tokenize_chinese_chars
           lowercase    :Any      	=						normalizer_class(**_lowerCAmelCase				)

         lowercase    :Any      	=						do_lower_case






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Optional[Any]       ,					**_lowerCAmelCase:					Optional[int]				):
         lowercase    :List[str]      	=						PaddingStrategy.MAX_LENGTH

         lowercase    :Optional[Any]      	=						text
         lowercase    :str      	=						kwargs.pop("text_pair"       ,					_lowerCAmelCase				)
         lowercase    :Any      	=						kwargs.pop("return_tensors"       ,					_lowerCAmelCase				)

         lowercase    :List[str]      	=						{
             "input_ids": [],
             "attention_mask": [],
             "token_type_ids": [],
         }

         for idx, candidate_text in enumerate(_lowerCAmelCase				):
           if batch_text_pair is not None:
             lowercase    :Any      	=						batch_text_pair[idx]
           else:
             lowercase    :Optional[Any]      	=						None

           lowercase    :Optional[Any]      	=						super().__call__(_lowerCAmelCase       ,					_lowerCAmelCase       ,					return_tensors=_lowerCAmelCase       ,					**_lowerCAmelCase				)

           lowercase    :Tuple      	=						encoded_candidates.get("input_ids"				)
           lowercase    :Optional[Any]      	=						encoded_candidates.get("attention_mask"				)
           lowercase    :int      	=						encoded_candidates.get("token_type_ids"				)

           if encoded_input_ids is not None:
             output_data["input_ids"].append(_lowerCAmelCase				)
           if encoded_attention_mask is not None:
             output_data["attention_mask"].append(_lowerCAmelCase				)
           if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(_lowerCAmelCase				)

         lowercase    :str      	=						{key: item for key, item in output_data.items() if len(_lowerCAmelCase				) != 0}

         return BatchEncoding(_lowerCAmelCase       ,					tensor_type=_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Any       ,					_lowerCAmelCase:					Tuple=None				):
         lowercase    :Dict      	=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple       ,					_lowerCAmelCase:					List[int]       ,					_lowerCAmelCase:					Optional[List[int]] = None				):
         lowercase    :List[str]      	=						[self.sep_token_id]
         lowercase    :List[Any]      	=						[self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					str       ,					_lowerCAmelCase:					Optional[str] = None				):
         lowercase    :Tuple      	=						self._tokenizer.model.save(_lowerCAmelCase       ,					name=_lowerCAmelCase				)
         return tuple(_lowerCAmelCase				)



