





'''simple docstring'''



import unittest

import numpy as np

from transformers.testing_utils import is_flaky, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
	import torch

if is_vision_available():
	from PIL import Image

	from transformers import DonutImageProcessor






class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase=7     ,_lowerCamelCase=3     ,_lowerCamelCase=18     ,_lowerCamelCase=30     ,_lowerCamelCase=400     ,_lowerCamelCase=True     ,_lowerCamelCase=None     ,_lowerCamelCase=True     ,_lowerCamelCase=False     ,_lowerCamelCase=True     ,_lowerCamelCase=True     ,_lowerCamelCase=[0.5, 0.5, 0.5]     ,_lowerCamelCase=[0.5, 0.5, 0.5]     ,)							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							parent
				__lowercase										=							batch_size
				__lowercase										=							num_channels
				__lowercase										=							image_size
				__lowercase										=							min_resolution
				__lowercase										=							max_resolution
				__lowercase										=							do_resize
				__lowercase										=							size if size is not None else {'''height''': 18, '''width''': 20}
				__lowercase										=							do_thumbnail
				__lowercase										=							do_align_axis
				__lowercase										=							do_pad
				__lowercase										=							do_normalize
				__lowercase										=							image_mean
				__lowercase										=							image_std


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				return {
				    "do_resize": self.do_resize,
				    "size": self.size,
				    "do_thumbnail": self.do_thumbnail,
				    "do_align_long_axis": self.do_align_axis,
				    "do_pad": self.do_pad,
				    "do_normalize": self.do_normalize,
				    "image_mean": self.image_mean,
				    "image_std": self.image_std,
				}






@require_torch
@require_vision
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				List[Any]											=       DonutImageProcessor if is_vision_available() else None


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				__lowercase										=							DonutImageProcessingTester(self )


	@property
	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				return self.image_processor_tester.prepare_image_processor_dict()


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_resize''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''size''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_thumbnail''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_align_long_axis''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_pad''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_normalize''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''image_mean''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''image_std''' ) )


	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class.from_dict(self.image_processor_dict )
				self.assertEqual(image_processor.size     ,{'''height''': 18, '''width''': 20} )

				__lowercase										=							self.image_processing_class.from_dict(self.image_processor_dict     ,size=42 )
				self.assertEqual(image_processor.size     ,{'''height''': 42, '''width''': 42} )

				# Previous config had dimensions in (width, height) order
				__lowercase										=							self.image_processing_class.from_dict(self.image_processor_dict     ,size=(42, 84) )
				self.assertEqual(image_processor.size     ,{'''height''': 84, '''width''': 42} )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				pass


	@is_flaky()
	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )
				# create random PIL images
				__lowercase										=							prepare_image_inputs(self.image_processor_tester     ,equal_resolution=_lowerCamelCase )
				for image in image_inputs:
							self.assertIsInstance(_lowerCamelCase     ,Image.Image )

				# Test not batched input
				__lowercase										=							image_processing(image_inputs[0]     ,return_tensors='''pt''' ).pixel_values
				self.assertEqual(
				    encoded_images.shape     ,(
				        1,
				        self.image_processor_tester.num_channels,
				        self.image_processor_tester.size['''height'''],
				        self.image_processor_tester.size['''width'''],
				    )     ,)

				# Test batched
				__lowercase										=							image_processing(_lowerCamelCase     ,return_tensors='''pt''' ).pixel_values
				self.assertEqual(
				    encoded_images.shape     ,(
				        self.image_processor_tester.batch_size,
				        self.image_processor_tester.num_channels,
				        self.image_processor_tester.size['''height'''],
				        self.image_processor_tester.size['''width'''],
				    )     ,)


	@is_flaky()
	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )
				# create random numpy tensors
				__lowercase										=							prepare_image_inputs(self.image_processor_tester     ,equal_resolution=_lowerCamelCase     ,numpify=_lowerCamelCase )
				for image in image_inputs:
							self.assertIsInstance(_lowerCamelCase     ,np.ndarray )

				# Test not batched input
				__lowercase										=							image_processing(image_inputs[0]     ,return_tensors='''pt''' ).pixel_values
				self.assertEqual(
				    encoded_images.shape     ,(
				        1,
				        self.image_processor_tester.num_channels,
				        self.image_processor_tester.size['''height'''],
				        self.image_processor_tester.size['''width'''],
				    )     ,)

				# Test batched
				__lowercase										=							image_processing(_lowerCamelCase     ,return_tensors='''pt''' ).pixel_values
				self.assertEqual(
				    encoded_images.shape     ,(
				        self.image_processor_tester.batch_size,
				        self.image_processor_tester.num_channels,
				        self.image_processor_tester.size['''height'''],
				        self.image_processor_tester.size['''width'''],
				    )     ,)


	@is_flaky()
	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )
				# create random PyTorch tensors
				__lowercase										=							prepare_image_inputs(self.image_processor_tester     ,equal_resolution=_lowerCamelCase     ,torchify=_lowerCamelCase )
				for image in image_inputs:
							self.assertIsInstance(_lowerCamelCase     ,torch.Tensor )

				# Test not batched input
				__lowercase										=							image_processing(image_inputs[0]     ,return_tensors='''pt''' ).pixel_values
				self.assertEqual(
				    encoded_images.shape     ,(
				        1,
				        self.image_processor_tester.num_channels,
				        self.image_processor_tester.size['''height'''],
				        self.image_processor_tester.size['''width'''],
				    )     ,)

				# Test batched
				__lowercase										=							image_processing(_lowerCamelCase     ,return_tensors='''pt''' ).pixel_values
				self.assertEqual(
				    encoded_images.shape     ,(
				        self.image_processor_tester.batch_size,
				        self.image_processor_tester.num_channels,
				        self.image_processor_tester.size['''height'''],
				        self.image_processor_tester.size['''width'''],
				    )     ,)












'''simple docstring'''



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase = "▁"     ,_lowerCamelCase = True     ,_lowerCamelCase = "<unk>"     ,_lowerCamelCase = "</s>"     ,_lowerCamelCase = "<pad>"     ,)							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							{
				    '''pad''': {'''id''': 0, '''token''': pad_token},
				    '''eos''': {'''id''': 1, '''token''': eos_token},
				    '''unk''': {'''id''': 2, '''token''': unk_token},
				}

				__lowercase										=							[None] * len(self.special_tokens )
				for token_dict in self.special_tokens.values():
							__lowercase										=							token_dict['''token''']

				__lowercase										=							Tokenizer(Unigram() )

				__lowercase										=							normalizers.Sequence(
				    [
				        normalizers.Nmt(),
				        normalizers.NFKC(),
				        normalizers.Replace(Regex(''' {2,}''' )     ,''' ''' ),
				        normalizers.Lowercase(),
				    ] )
				__lowercase										=							pre_tokenizers.Sequence(
				    [
				        pre_tokenizers.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase ),
				        pre_tokenizers.Digits(individual_digits=_lowerCamelCase ),
				        pre_tokenizers.Punctuation(),
				    ] )
				__lowercase										=							decoders.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase )

				__lowercase										=							TemplateProcessing(
				    single=f"$A {self.special_tokens['eos']['token']}"     ,special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]     ,)

				__lowercase										=							{
				    '''model''': '''SentencePieceUnigram''',
				    '''replacement''': replacement,
				    '''add_prefix_space''': add_prefix_space,
				}

				super().__init__(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				if isinstance(_lowerCamelCase     ,_lowerCamelCase ):
							__lowercase										=							[files]
				self._tokenizer.train(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							List[str]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				self._tokenizer.train_from_iterator(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							json.loads(self._tokenizer.to_str() )

				__lowercase										=							self.special_tokens['''unk''']['''id''']

				__lowercase										=							Tokenizer.from_str(json.dumps(_lowerCamelCase ) )





