






'''simple docstring'''







import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 		UpperCamelCase__    (a   ,unittest.TestCase			):




		'''simple docstring'''





		_UpperCamelCase				  =					None
		_UpperCamelCase				  =					BloomTokenizerFast
		_UpperCamelCase				  =					BloomTokenizerFast
		_UpperCamelCase				  =					True
		_UpperCamelCase				  =					False
		_UpperCamelCase				  =					'tokenizer_file'
		_UpperCamelCase				  =					{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>'}






		def 			UpperCamelCase_					(      self       ):
								super().setUp()
								lowerCamelCase__         =  BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""       )
								tokenizer.save_pretrained(self.tmpdirname       )






		def 			UpperCamelCase_					(      self   ,**_lowerCAmelCase       ):
								kwargs.update(self.special_tokens_map       )
								return BloomTokenizerFast.from_pretrained(self.tmpdirname   ,**_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_rust_tokenizer()

								lowerCamelCase__         =  ["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
								lowerCamelCase__         =  [[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

								lowerCamelCase__         =  tokenizer.batch_encode_plus(_lowerCAmelCase       )["""input_ids"""]
								self.assertListEqual(_lowerCAmelCase   ,_lowerCAmelCase       )

								lowerCamelCase__         =  tokenizer.batch_decode(_lowerCAmelCase       )
								self.assertListEqual(_lowerCAmelCase   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase=6       ):
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																				lowerCamelCase__         =  self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase   ,**_lowerCAmelCase       )
																				# tokenizer_r.pad_token = None # Hotfixing padding = None
																				# Simple input
																				lowerCamelCase__         =  """This is a simple input"""
																				lowerCamelCase__         =  ["""This is a simple input 1""", """This is a simple input 2"""]
																				lowerCamelCase__         =  ("""This is a simple input""", """This is a pair""")
																				lowerCamelCase__         =  [
																				    ("""This is a simple input 1""", """This is a simple input 2"""),
																				    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																				]

																				# Simple input tests
																				try:
																										tokenizer_r.encode(_lowerCAmelCase   ,max_length=_lowerCAmelCase       )
																										tokenizer_r.encode_plus(_lowerCAmelCase   ,max_length=_lowerCAmelCase       )

																										tokenizer_r.batch_encode_plus(_lowerCAmelCase   ,max_length=_lowerCAmelCase       )
																										tokenizer_r.encode(_lowerCAmelCase   ,max_length=_lowerCAmelCase       )
																										tokenizer_r.batch_encode_plus(_lowerCAmelCase   ,max_length=_lowerCAmelCase       )
																				except ValueError:
																										self.fail("""Bloom Tokenizer should be able to deal with padding"""       )

																				lowerCamelCase__         =  None  # Hotfixing padding = None
																				self.assertRaises(_lowerCAmelCase   ,tokenizer_r.encode   ,_lowerCAmelCase   ,max_length=_lowerCAmelCase   ,padding="""max_length"""       )

																				# Simple input
																				self.assertRaises(_lowerCAmelCase   ,tokenizer_r.encode_plus   ,_lowerCAmelCase   ,max_length=_lowerCAmelCase   ,padding="""max_length"""       )

																				# Simple input
																				self.assertRaises(
																				    _lowerCAmelCase   ,tokenizer_r.batch_encode_plus   ,_lowerCAmelCase   ,max_length=_lowerCAmelCase   ,padding="""max_length"""   ,)

																				# Pair input
																				self.assertRaises(_lowerCAmelCase   ,tokenizer_r.encode   ,_lowerCAmelCase   ,max_length=_lowerCAmelCase   ,padding="""max_length"""       )

																				# Pair input
																				self.assertRaises(_lowerCAmelCase   ,tokenizer_r.encode_plus   ,_lowerCAmelCase   ,max_length=_lowerCAmelCase   ,padding="""max_length"""       )

																				# Pair input
																				self.assertRaises(
																				    _lowerCAmelCase   ,tokenizer_r.batch_encode_plus   ,_lowerCAmelCase   ,max_length=_lowerCAmelCase   ,padding="""max_length"""   ,)






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.get_rust_tokenizer()
								lowerCamelCase__         =  load_dataset("""xnli"""   ,"""all_languages"""   ,split="""test"""   ,streaming=_lowerCAmelCase       )

								lowerCamelCase__         =  next(iter(_lowerCAmelCase       )       )["""premise"""]  # pick up one data
								lowerCamelCase__         =  list(sample_data.values()       )

								lowerCamelCase__         =  list(map(tokenizer.encode   ,_lowerCAmelCase       )       )
								lowerCamelCase__         =  [tokenizer.decode(_lowerCAmelCase   ,clean_up_tokenization_spaces=_lowerCAmelCase       ) for x in output_tokens]
								self.assertListEqual(_lowerCAmelCase   ,_lowerCAmelCase       )






		def 			UpperCamelCase_					(      self       ):
								# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
								# any sequence length constraints. This test of the parent class will fail since it relies on the
								# maximum sequence length of the positoonal embeddings.
								self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map       )   ,1       )
								self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()       )[0]       )   ,1       )









'''simple docstring'''







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import LevitImageProcessor




class 		UpperCamelCase__    (unittest.TestCase			):




		'''simple docstring'''





		def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase=7   ,_lowerCAmelCase=3   ,_lowerCAmelCase=18   ,_lowerCAmelCase=30   ,_lowerCAmelCase=4_00   ,_lowerCAmelCase=True   ,_lowerCAmelCase=None   ,_lowerCAmelCase=True   ,_lowerCAmelCase=None   ,_lowerCAmelCase=True   ,_lowerCAmelCase=[0.5, 0.5, 0.5]   ,_lowerCAmelCase=[0.5, 0.5, 0.5]   ,):
								lowerCamelCase__         =  size if size is not None else {"""shortest_edge""": 18}
								lowerCamelCase__         =  crop_size if crop_size is not None else {"""height""": 18, """width""": 18}
								lowerCamelCase__         =  parent
								lowerCamelCase__         =  batch_size
								lowerCamelCase__         =  num_channels
								lowerCamelCase__         =  image_size
								lowerCamelCase__         =  min_resolution
								lowerCamelCase__         =  max_resolution
								lowerCamelCase__         =  do_resize
								lowerCamelCase__         =  size
								lowerCamelCase__         =  do_center_crop
								lowerCamelCase__         =  crop_size
								lowerCamelCase__         =  do_normalize
								lowerCamelCase__         =  image_mean
								lowerCamelCase__         =  image_std






		def 			UpperCamelCase_					(      self       ):
								return {
								    "image_mean": self.image_mean,
								    "image_std": self.image_std,
								    "do_normalize": self.do_normalize,
								    "do_resize": self.do_resize,
								    "do_center_crop": self.do_center_crop,
								    "size": self.size,
								    "crop_size": self.crop_size,
								}




@require_torch
@require_vision
class 		UpperCamelCase__    (a   ,unittest.TestCase			):




		'''simple docstring'''





		_UpperCamelCase				  =					LevitImageProcessor if is_vision_available() else None






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  LevitImageProcessingTester(self       )






		@property
		def 			UpperCamelCase_					(      self       ):
								return self.image_processor_tester.prepare_image_processor_dict()






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.image_processing_class(**self.image_processor_dict       )
								self.assertTrue(hasattr(_lowerCAmelCase   ,"""image_mean"""       )       )
								self.assertTrue(hasattr(_lowerCAmelCase   ,"""image_std"""       )       )
								self.assertTrue(hasattr(_lowerCAmelCase   ,"""do_normalize"""       )       )
								self.assertTrue(hasattr(_lowerCAmelCase   ,"""do_resize"""       )       )
								self.assertTrue(hasattr(_lowerCAmelCase   ,"""do_center_crop"""       )       )
								self.assertTrue(hasattr(_lowerCAmelCase   ,"""size"""       )       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.image_processing_class.from_dict(self.image_processor_dict       )
								self.assertEqual(image_processor.size   ,{"""shortest_edge""": 18}       )
								self.assertEqual(image_processor.crop_size   ,{"""height""": 18, """width""": 18}       )

								lowerCamelCase__         =  self.image_processing_class.from_dict(self.image_processor_dict   ,size=42   ,crop_size=84       )
								self.assertEqual(image_processor.size   ,{"""shortest_edge""": 42}       )
								self.assertEqual(image_processor.crop_size   ,{"""height""": 84, """width""": 84}       )






		def 			UpperCamelCase_					(      self       ):
								pass






		def 			UpperCamelCase_					(      self       ):
								# Initialize image_processing
								lowerCamelCase__         =  self.image_processing_class(**self.image_processor_dict       )
								# create random PIL images
								lowerCamelCase__         =  prepare_image_inputs(self.image_processor_tester   ,equal_resolution=_lowerCAmelCase       )
								for image in image_inputs:
														self.assertIsInstance(_lowerCAmelCase   ,Image.Image       )

								# Test not batched input
								lowerCamelCase__         =  image_processing(image_inputs[0]   ,return_tensors="""pt"""       ).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,)

								# Test batched
								lowerCamelCase__         =  image_processing(_lowerCAmelCase   ,return_tensors="""pt"""       ).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,)






		def 			UpperCamelCase_					(      self       ):
								# Initialize image_processing
								lowerCamelCase__         =  self.image_processing_class(**self.image_processor_dict       )
								# create random numpy tensors
								lowerCamelCase__         =  prepare_image_inputs(self.image_processor_tester   ,equal_resolution=_lowerCAmelCase   ,numpify=_lowerCAmelCase       )
								for image in image_inputs:
														self.assertIsInstance(_lowerCAmelCase   ,np.ndarray       )

								# Test not batched input
								lowerCamelCase__         =  image_processing(image_inputs[0]   ,return_tensors="""pt"""       ).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,)

								# Test batched
								lowerCamelCase__         =  image_processing(_lowerCAmelCase   ,return_tensors="""pt"""       ).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,)






		def 			UpperCamelCase_					(      self       ):
								# Initialize image_processing
								lowerCamelCase__         =  self.image_processing_class(**self.image_processor_dict       )
								# create random PyTorch tensors
								lowerCamelCase__         =  prepare_image_inputs(self.image_processor_tester   ,equal_resolution=_lowerCAmelCase   ,torchify=_lowerCAmelCase       )
								for image in image_inputs:
														self.assertIsInstance(_lowerCAmelCase   ,torch.Tensor       )

								# Test not batched input
								lowerCamelCase__         =  image_processing(image_inputs[0]   ,return_tensors="""pt"""       ).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,)

								# Test batched
								lowerCamelCase__         =  image_processing(_lowerCAmelCase   ,return_tensors="""pt"""       ).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,)

