






'''simple docstring'''







import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
							import tensorflow as tf

if is_tensorflow_text_available():
							from transformers.models.bert import TFBertTokenizer


UpperCamelCase    :      List[Any]  			=		['bert-base-uncased', 'bert-base-cased']
UpperCamelCase    :      List[str]  			=		'hf-internal-testing/tiny-bert-tf-only'

if is_tf_available():




							class 		UpperCamelCase__    (tf.keras.Model			):




									'''simple docstring'''





									def __init__(      self   ,_lowerCAmelCase       ):
															super().__init__()
															lowerCamelCase__         =  tokenizer
															lowerCamelCase__         =  AutoConfig.from_pretrained(_lowerCAmelCase       )
															lowerCamelCase__         =  TFAutoModel.from_config(_lowerCAmelCase       )






									def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
															lowerCamelCase__         =  self.tokenizer(_lowerCAmelCase       )
															lowerCamelCase__         =  self.bert(**_lowerCAmelCase       )
															return out["pooler_output"]




@require_tf
@require_tensorflow_text
class 		UpperCamelCase__    (unittest.TestCase			):




		'''simple docstring'''





		def 			UpperCamelCase_					(      self       ):
								super().setUp()

								lowerCamelCase__         =  [
								    BertTokenizer.from_pretrained(_lowerCAmelCase       ) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
								]  # repeat for when fast_bert_tokenizer=false
								lowerCamelCase__         =  [TFBertTokenizer.from_pretrained(_lowerCAmelCase       ) for checkpoint in TOKENIZER_CHECKPOINTS] + [
								    TFBertTokenizer.from_pretrained(_lowerCAmelCase   ,use_fast_bert_tokenizer=_lowerCAmelCase       )
								    for checkpoint in TOKENIZER_CHECKPOINTS
								]
								assert len(self.tokenizers       ) == len(self.tf_tokenizers       )

								lowerCamelCase__         =  [
								    """This is a straightforward English test sentence.""",
								    """This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.""",
								    """Now we're going to add some Chinese: 一 二 三 一二三""",
								    """And some much more rare Chinese: 齉 堃 齉堃""",
								    """Je vais aussi écrire en français pour tester les accents""",
								    """Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ""",
								]
								lowerCamelCase__         =  list(zip(self.test_sentences   ,self.test_sentences[::-1]       )       )






		def 			UpperCamelCase_					(      self       ):
								for tokenizer, tf_tokenizer in zip(self.tokenizers   ,self.tf_tokenizers       ):
														for test_inputs in (self.test_sentences, self.paired_sentences):
																				lowerCamelCase__         =  tokenizer(_lowerCAmelCase   ,return_tensors="""tf"""   ,padding="""longest"""       )
																				lowerCamelCase__         =  tf_tokenizer(_lowerCAmelCase       )

																				for key in python_outputs.keys():
																										self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape       )       )
																										self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]   ,tf.intaa       ) == tf_outputs[key]       )       )






		@slow
		def 			UpperCamelCase_					(      self       ):
								for tf_tokenizer in self.tf_tokenizers:
														lowerCamelCase__         =  tf_tokenizer(self.paired_sentences       )
														lowerCamelCase__         =  tf_tokenizer(
														    text=[sentence[0] for sentence in self.paired_sentences]   ,text_pair=[sentence[1] for sentence in self.paired_sentences]   ,)
														for key in merged_outputs.keys():
																				self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]   ,tf.intaa       ) == separated_outputs[key]       )       )






		@slow
		def 			UpperCamelCase_					(      self       ):
								for tf_tokenizer in self.tf_tokenizers:
														lowerCamelCase__         =  tf.function(_lowerCAmelCase       )
														for test_inputs in (self.test_sentences, self.paired_sentences):
																				lowerCamelCase__         =  tf.constant(_lowerCAmelCase       )
																				lowerCamelCase__         =  compiled_tokenizer(_lowerCAmelCase       )
																				lowerCamelCase__         =  tf_tokenizer(_lowerCAmelCase       )

																				for key in eager_outputs.keys():
																										self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]       )       )






		@slow
		def 			UpperCamelCase_					(      self       ):
								for tf_tokenizer in self.tf_tokenizers:
														lowerCamelCase__         =  ModelToSave(tokenizer=_lowerCAmelCase       )
														lowerCamelCase__         =  tf.convert_to_tensor(self.test_sentences       )
														lowerCamelCase__         =  model(_lowerCAmelCase       )  # Build model with some sample inputs
														with TemporaryDirectory() as tempdir:
																				lowerCamelCase__         =  Path(_lowerCAmelCase       ) / """saved.model"""
																				model.save(_lowerCAmelCase       )
																				lowerCamelCase__         =  tf.keras.models.load_model(_lowerCAmelCase       )
														lowerCamelCase__         =  loaded_model(_lowerCAmelCase       )
														# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
														self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output       )       )   ,1E-5       )









'''simple docstring'''







import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
							from PIL import Image

							from transformers import AutoProcessor, BertTokenizer, BlipImageProcessor, BlipProcessor, PreTrainedTokenizerFast




@require_vision
class 		UpperCamelCase__    (unittest.TestCase			):




		'''simple docstring'''





		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  tempfile.mkdtemp()

								lowerCamelCase__         =  BlipImageProcessor()
								lowerCamelCase__         =  BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-BertModel"""       )

								lowerCamelCase__         =  BlipProcessor(_lowerCAmelCase   ,_lowerCAmelCase       )

								processor.save_pretrained(self.tmpdirname       )






		def 			UpperCamelCase_					(      self   ,**_lowerCAmelCase       ):
								return AutoProcessor.from_pretrained(self.tmpdirname   ,**_lowerCAmelCase       ).tokenizer






		def 			UpperCamelCase_					(      self   ,**_lowerCAmelCase       ):
								return AutoProcessor.from_pretrained(self.tmpdirname   ,**_lowerCAmelCase       ).image_processor






		def 			UpperCamelCase_					(      self       ):
								shutil.rmtree(self.tmpdirname       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  [np.random.randint(2_55   ,size=(3, 30, 4_00)   ,dtype=np.uinta       )]

								lowerCamelCase__         =  [Image.fromarray(np.moveaxis(_lowerCAmelCase   ,0   ,-1       )       ) for x in image_inputs]

								return image_inputs






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  BlipProcessor(tokenizer=self.get_tokenizer()   ,image_processor=self.get_image_processor()       )
								processor.save_pretrained(self.tmpdirname       )

								lowerCamelCase__         =  self.get_tokenizer(bos_token="""(BOS)"""   ,eos_token="""(EOS)"""       )
								lowerCamelCase__         =  self.get_image_processor(do_normalize=_lowerCAmelCase   ,padding_value=1.0       )

								lowerCamelCase__         =  BlipProcessor.from_pretrained(
								    self.tmpdirname   ,bos_token="""(BOS)"""   ,eos_token="""(EOS)"""   ,do_normalize=_lowerCAmelCase   ,padding_value=1.0       )

								self.assertEqual(processor.tokenizer.get_vocab()   ,tokenizer_add_kwargs.get_vocab()       )
								self.assertIsInstance(processor.tokenizer   ,_lowerCAmelCase       )

								self.assertEqual(processor.image_processor.to_json_string()   ,image_processor_add_kwargs.to_json_string()       )
								self.assertIsInstance(processor.image_processor   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  self.prepare_image_inputs()

								lowerCamelCase__         =  image_processor(_lowerCAmelCase   ,return_tensors="""np"""       )
								lowerCamelCase__         =  processor(images=_lowerCAmelCase   ,return_tensors="""np"""       )

								for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()   ,input_processor[key].sum()   ,delta=1E-2       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  """lower newer"""

								lowerCamelCase__         =  processor(text=_lowerCAmelCase       )

								lowerCamelCase__         =  tokenizer(_lowerCAmelCase   ,return_token_type_ids=_lowerCAmelCase       )

								for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]   ,encoded_processor[key]       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  """lower newer"""
								lowerCamelCase__         =  self.prepare_image_inputs()

								lowerCamelCase__         =  processor(text=_lowerCAmelCase   ,images=_lowerCAmelCase       )

								self.assertListEqual(list(inputs.keys()       )   ,["""pixel_values""", """input_ids""", """attention_mask"""]       )

								# test if it raises when no input is passed
								with pytest.raises(_lowerCAmelCase       ):
														processor()






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								lowerCamelCase__         =  processor.batch_decode(_lowerCAmelCase       )
								lowerCamelCase__         =  tokenizer.batch_decode(_lowerCAmelCase       )

								self.assertListEqual(_lowerCAmelCase   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_image_processor()
								lowerCamelCase__         =  self.get_tokenizer()

								lowerCamelCase__         =  BlipProcessor(tokenizer=_lowerCAmelCase   ,image_processor=_lowerCAmelCase       )

								lowerCamelCase__         =  """lower newer"""
								lowerCamelCase__         =  self.prepare_image_inputs()

								lowerCamelCase__         =  processor(text=_lowerCAmelCase   ,images=_lowerCAmelCase       )

								# For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask']
								self.assertListEqual(list(inputs.keys()       )   ,["""pixel_values""", """input_ids""", """attention_mask"""]       )

