'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_A								=   logging.get_logger(__name__)

_A								=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_A								=   {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

_A								=   {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

_A								=   {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}


class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   A							:       Tuple     =    VOCAB_FILES_NAMES
   A							:       Any     =    PRETRAINED_VOCAB_FILES_MAP
   A							:       Dict     =    PRETRAINED_INIT_CONFIGURATION
   A							:       int     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A							:       Optional[int]     =    RealmTokenizer
   def __init__(self       ,				_a=None       ,				_a=None       ,				_a=True       ,				_a="[UNK]"       ,				_a="[SEP]"       ,				_a="[PAD]"       ,				_a="[CLS]"       ,				_a="[MASK]"       ,				_a=True       ,				_a=None       ,				**_a       ,				)    ->     Optional[Any]:
        super().__init__(
            _a       ,				tokenizer_file=_a       ,				do_lower_case=_a       ,				unk_token=_a       ,				sep_token=_a       ,				pad_token=_a       ,				cls_token=_a       ,				mask_token=_a       ,				tokenize_chinese_chars=_a       ,				strip_accents=_a       ,				**_a       ,				)

        lowercase_					:       List[str]     				=					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
        if (
            normalizer_state.get('lowercase'       ,				_a				) != do_lower_case
            or normalizer_state.get('strip_accents'       ,				_a				) != strip_accents
            or normalizer_state.get('handle_chinese_chars'       ,				_a				) != tokenize_chinese_chars
        ):
             lowercase_					:       Dict     				=					getattr(_a       ,				normalizer_state.pop('type'				)				)
             lowercase_					:       int     				=					do_lower_case
             lowercase_					:       List[Any]     				=					strip_accents
             lowercase_					:       Optional[int]     				=					tokenize_chinese_chars
             lowercase_					:       List[str]     				=					normalizer_class(**_a				)

        lowercase_					:       Tuple     				=					do_lower_case
   def 		_lowerCamelCase (self       ,				_a       ,				**_a				)    ->     Dict:
        lowercase_					:       int     				=					PaddingStrategy.MAX_LENGTH

        lowercase_					:       Union[str, Any]     				=					text
        lowercase_					:       Optional[Any]     				=					kwargs.pop('text_pair'       ,				_a				)
        lowercase_					:       Any     				=					kwargs.pop('return_tensors'       ,				_a				)

        lowercase_					:       str     				=					{
            'input_ids': [],
            'attention_mask': [],
            'token_type_ids': [],
        }

        for idx, candidate_text in enumerate(_a				):
             if batch_text_pair is not None:
                  lowercase_					:       Any     				=					batch_text_pair[idx]
             else:
                  lowercase_					:       Union[str, Any]     				=					None

             lowercase_					:       Optional[int]     				=					super().__call__(_a       ,				_a       ,				return_tensors=_a       ,				**_a				)

             lowercase_					:       int     				=					encoded_candidates.get('input_ids'				)
             lowercase_					:       int     				=					encoded_candidates.get('attention_mask'				)
             lowercase_					:       Tuple     				=					encoded_candidates.get('token_type_ids'				)

             if encoded_input_ids is not None:
                  output_data["input_ids"].append(_a				)
             if encoded_attention_mask is not None:
                  output_data["attention_mask"].append(_a				)
             if encoded_token_type_ids is not None:
                  output_data["token_type_ids"].append(_a				)

        lowercase_					:       Tuple     				=					{key: item for key, item in output_data.items() if len(_a				) != 0}

        return BatchEncoding(_a       ,				tensor_type=_a				)
   def 		_lowerCamelCase (self       ,				_a       ,				_a=None				)    ->     Optional[int]:
        lowercase_					:       Optional[Any]     				=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

        return output
   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     List[int]:
        lowercase_					:       str     				=					[self.sep_token_id]
        lowercase_					:       Optional[Any]     				=					[self.cls_token_id]
        if token_ids_a is None:
             return len(cls + token_ids_a + sep				) * [0]
        return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     Tuple[str]:
        lowercase_					:       Dict     				=					self._tokenizer.model.save(_a       ,				name=_a				)
        return tuple(_a				)





'''simple docstring'''




from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
 import tensorflow as tf

 from transformers import TFBlipTextModel
 from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST


class       UpperCAmelCase__  :






   """simple docstring"""
   def __init__(self       ,				_a       ,				_a=12       ,				_a=7       ,				_a=True       ,				_a=True       ,				_a=True       ,				_a=99       ,				_a=32       ,				_a=32       ,				_a=2       ,				_a=4       ,				_a=37       ,				_a=0.1       ,				_a=0.1       ,				_a=512       ,				_a=0.02       ,				_a=0       ,				_a=None       ,				)    ->     Optional[Any]:
        lowercase_					:       List[Any]     				=					parent
        lowercase_					:       List[Any]     				=					batch_size
        lowercase_					:       Optional[Any]     				=					seq_length
        lowercase_					:       Optional[Any]     				=					is_training
        lowercase_					:       Optional[int]     				=					use_input_mask
        lowercase_					:       Any     				=					use_labels
        lowercase_					:       Union[str, Any]     				=					vocab_size
        lowercase_					:       Union[str, Any]     				=					hidden_size
        lowercase_					:       str     				=					projection_dim
        lowercase_					:       str     				=					num_hidden_layers
        lowercase_					:       List[str]     				=					num_attention_heads
        lowercase_					:       List[str]     				=					intermediate_size
        lowercase_					:       Optional[Any]     				=					dropout
        lowercase_					:       Tuple     				=					attention_dropout
        lowercase_					:       Optional[Any]     				=					max_position_embeddings
        lowercase_					:       Any     				=					initializer_range
        lowercase_					:       List[str]     				=					scope
        lowercase_					:       Optional[Any]     				=					bos_token_id
   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       Optional[Any]     				=					ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size				)

        lowercase_					:       Optional[int]     				=					None
        if self.use_input_mask:
             lowercase_					:       Dict     				=					random_attention_mask([self.batch_size, self.seq_length]				)

        if input_mask is not None:
             lowercase_					:       Dict     				=					input_mask.numpy()
             lowercase_				,lowercase_					:       int     				=					input_mask.shape
             lowercase_					:       int     				=					np.random.randint(1       ,				seq_length - 1       ,				size=(batch_size,)				)
             for batch_idx, start_index in enumerate(_a				):
                  lowercase_					:       Tuple     				=					1
                  lowercase_					:       List[Any]     				=					0

        lowercase_					:       List[Any]     				=					self.get_config()

        return config, input_ids, tf.convert_to_tensor(_a				)
   def 		_lowerCamelCase (self				)    ->     Any:
        return BlipTextConfig(
            vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				projection_dim=self.projection_dim       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				dropout=self.dropout       ,				attention_dropout=self.attention_dropout       ,				max_position_embeddings=self.max_position_embeddings       ,				initializer_range=self.initializer_range       ,				bos_token_id=self.bos_token_id       ,				)
   def 		_lowerCamelCase (self       ,				_a       ,				_a       ,				_a				)    ->     List[Any]:
        lowercase_					:       List[Any]     				=					TFBlipTextModel(config=_a				)
        lowercase_					:       List[str]     				=					model(_a       ,				attention_mask=_a       ,				training=_a				)
        lowercase_					:       Union[str, Any]     				=					model(_a       ,				training=_a				)
        self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)				)
        self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)				)





   def 		_lowerCamelCase (self				)    ->     Dict:
        lowercase_					:       Optional[Any]     				=					self.prepare_config_and_inputs()
        lowercase_				,lowercase_				,lowercase_					:       Dict     				=					config_and_inputs
        lowercase_					:       Optional[int]     				=					{'input_ids': input_ids, 'attention_mask': input_mask}
        return config, inputs_dict



@require_tf
class       UpperCAmelCase__  (       _snake_case					,     unittest.TestCase			):






   """simple docstring"""
   A							:       Any     =    (TFBlipTextModel,) if is_tf_available() else ()
   A							:       Union[str, Any]     =    False
   A							:       int     =    False
   A							:       str     =    False
   def 		_lowerCamelCase (self				)    ->     int:
        lowercase_					:       Tuple     				=					BlipTextModelTester(self				)
        lowercase_					:       str     				=					ConfigTester(self       ,				config_class=_a       ,				hidden_size=37				)
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        self.config_tester.run_common_tests()
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        lowercase_					:       str     				=					self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*_a				)
   def 		_lowerCamelCase (self				)    ->     Dict:
        pass
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        pass
   @unittest.skip(reason='Blip does not use inputs_embeds'				)
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        pass
   @unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'				)
   def 		_lowerCamelCase (self				)    ->     Any:
        pass
   @unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'				)
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        pass
   @slow
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowercase_					:       List[str]     				=					TFBlipTextModel.from_pretrained(_a				)
             self.assertIsNotNone(_a				)





   def 		_lowerCamelCase (self       ,				_a=True				)    ->     Optional[int]:
        super().test_pt_tf_model_equivalence(allow_missing_keys=_a				)




