'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__snake_case:  Union[str, Any]								=			logging.get_logger(__name__)

__snake_case:  Union[str, Any]								=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__snake_case:  Union[str, Any]								=			{
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

__snake_case:  str								=			{
    "google/realm-cc-news-pretrained-embedder": 5_12,
    "google/realm-cc-news-pretrained-encoder": 5_12,
    "google/realm-cc-news-pretrained-scorer": 5_12,
    "google/realm-cc-news-pretrained-openqa": 5_12,
    "google/realm-orqa-nq-openqa": 5_12,
    "google/realm-orqa-nq-reader": 5_12,
    "google/realm-orqa-wq-openqa": 5_12,
    "google/realm-orqa-wq-reader": 5_12,
}

__snake_case:  Optional[Any]								=			{
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}


class  _UpperCAmelCase ( __A    ):

     """simple docstring"""

     a_    	=					VOCAB_FILES_NAMES
     a_    	=					PRETRAINED_VOCAB_FILES_MAP
     a_    	=					PRETRAINED_INIT_CONFIGURATION
     a_    	=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     a_    	=					RealmTokenizer


     def __init__(      self							,		lowerCAmelCase_=None							,		lowerCAmelCase_=None							,		lowerCAmelCase_=True							,		lowerCAmelCase_="[UNK]"							,		lowerCAmelCase_="[SEP]"							,		lowerCAmelCase_="[PAD]"							,		lowerCAmelCase_="[CLS]"							,		lowerCAmelCase_="[MASK]"							,		lowerCAmelCase_=True							,		lowerCAmelCase_=None							,		**lowerCAmelCase_							,		):

       '''simple docstring'''

       super().__init__(
           lowerCAmelCase_							,		tokenizer_file=lowerCAmelCase_							,		do_lower_case=lowerCAmelCase_							,		unk_token=lowerCAmelCase_							,		sep_token=lowerCAmelCase_							,		pad_token=lowerCAmelCase_							,		cls_token=lowerCAmelCase_							,		mask_token=lowerCAmelCase_							,		tokenize_chinese_chars=lowerCAmelCase_							,		strip_accents=lowerCAmelCase_							,		**lowerCAmelCase_							,		)

       a_      :	Dict      =							json.loads(self.backend_tokenizer.normalizer.__getstate__() )
       if (
           normalizer_state.get("""lowercase"""							,		lowerCAmelCase_ ) != do_lower_case
           or normalizer_state.get("""strip_accents"""							,		lowerCAmelCase_ ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""							,		lowerCAmelCase_ ) != tokenize_chinese_chars
       ):
         a_      :	Optional[int]      =							getattr(lowerCAmelCase_							,		normalizer_state.pop("""type""" ) )
         a_      :	Optional[Any]      =							do_lower_case
         a_      :	Union[str, Any]      =							strip_accents
         a_      :	Optional[int]      =							tokenize_chinese_chars
         a_      :	Union[str, Any]      =							normalizer_class(**lowerCAmelCase_ )

       a_      :	Tuple      =							do_lower_case


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		**lowerCAmelCase_ ):

       '''simple docstring'''

       a_      :	Optional[int]      =							PaddingStrategy.MAX_LENGTH

       a_      :	Optional[int]      =							text
       a_      :	int      =							kwargs.pop("""text_pair"""							,		lowerCAmelCase_ )
       a_      :	List[Any]      =							kwargs.pop("""return_tensors"""							,		lowerCAmelCase_ )

       a_      :	Any      =							{
           'input_ids': [],
           'attention_mask': [],
           'token_type_ids': [],
       }

       for idx, candidate_text in enumerate(lowerCAmelCase_ ):
         if batch_text_pair is not None:
           a_      :	Union[str, Any]      =							batch_text_pair[idx]
         else:
           a_      :	List[str]      =							None

         a_      :	Dict      =							super().__call__(lowerCAmelCase_							,		lowerCAmelCase_							,		return_tensors=lowerCAmelCase_							,		**lowerCAmelCase_ )

         a_      :	Tuple      =							encoded_candidates.get("""input_ids""" )
         a_      :	Dict      =							encoded_candidates.get("""attention_mask""" )
         a_      :	str      =							encoded_candidates.get("""token_type_ids""" )

         if encoded_input_ids is not None:
           output_data["input_ids"].append(lowerCAmelCase_ )
         if encoded_attention_mask is not None:
           output_data["attention_mask"].append(lowerCAmelCase_ )
         if encoded_token_type_ids is not None:
           output_data["token_type_ids"].append(lowerCAmelCase_ )

       a_      :	List[Any]      =							{key: item for key, item in output_data.items() if len(lowerCAmelCase_ ) != 0}

       return BatchEncoding(lowerCAmelCase_							,		tensor_type=lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_=None ):

       '''simple docstring'''

       a_      :	Union[str, Any]      =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_ = None ):

       '''simple docstring'''

       a_      :	Union[str, Any]      =							[self.sep_token_id]
       a_      :	Optional[Any]      =							[self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep ) * [0]
       return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_ = None ):

       '''simple docstring'''

       a_      :	str      =							self._tokenizer.model.save(lowerCAmelCase_							,		name=lowerCAmelCase_ )
       return tuple(lowerCAmelCase_ )




'''simple docstring'''

import unittest

import numpy as np

from transformers.testing_utils import is_flaky, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import DonutImageProcessor


class  _UpperCAmelCase ( unittest.TestCase    ):

     """simple docstring"""

     def __init__(      self							,		lowerCAmelCase_							,		lowerCAmelCase_=7							,		lowerCAmelCase_=3							,		lowerCAmelCase_=18							,		lowerCAmelCase_=30							,		lowerCAmelCase_=4_00							,		lowerCAmelCase_=True							,		lowerCAmelCase_=None							,		lowerCAmelCase_=True							,		lowerCAmelCase_=False							,		lowerCAmelCase_=True							,		lowerCAmelCase_=True							,		lowerCAmelCase_=[0.5, 0.5, 0.5]							,		lowerCAmelCase_=[0.5, 0.5, 0.5]							,		):

       '''simple docstring'''

       a_      :	Optional[int]      =							parent
       a_      :	Union[str, Any]      =							batch_size
       a_      :	Any      =							num_channels
       a_      :	Tuple      =							image_size
       a_      :	Any      =							min_resolution
       a_      :	Dict      =							max_resolution
       a_      :	Optional[Any]      =							do_resize
       a_      :	Tuple      =							size if size is not None else {"""height""": 18, """width""": 20}
       a_      :	int      =							do_thumbnail
       a_      :	Optional[int]      =							do_align_axis
       a_      :	Tuple      =							do_pad
       a_      :	int      =							do_normalize
       a_      :	Tuple      =							image_mean
       a_      :	Optional[Any]      =							image_std






     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return {
           "do_resize": self.do_resize,
           "size": self.size,
           "do_thumbnail": self.do_thumbnail,
           "do_align_long_axis": self.do_align_axis,
           "do_pad": self.do_pad,
           "do_normalize": self.do_normalize,
           "image_mean": self.image_mean,
           "image_std": self.image_std,
       }







@require_torch
@require_vision
class  _UpperCAmelCase ( lowerCAmelCase__	,unittest.TestCase    ):

     """simple docstring"""

     a_    	=					DonutImageProcessor if is_vision_available() else None


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[int]      =							DonutImageProcessingTester(self )


     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return self.image_processor_tester.prepare_image_processor_dict()


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Any      =							self.image_processing_class(**self.image_processor_dict )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""do_resize""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""size""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""do_thumbnail""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""do_align_long_axis""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""do_pad""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""do_normalize""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""image_mean""" ) )
       self.assertTrue(hasattr(lowerCAmelCase_							,		"""image_std""" ) )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[str]      =							self.image_processing_class.from_dict(self.image_processor_dict )
       self.assertEqual(image_processor.size							,		{"""height""": 18, """width""": 20} )

       a_      :	str      =							self.image_processing_class.from_dict(self.image_processor_dict							,		size=42 )
       self.assertEqual(image_processor.size							,		{"""height""": 42, """width""": 42} )

       # Previous config had dimensions in (width, height) order
       a_      :	Any      =							self.image_processing_class.from_dict(self.image_processor_dict							,		size=(42, 84) )
       self.assertEqual(image_processor.size							,		{"""height""": 84, """width""": 42} )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     @is_flaky()
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							self.image_processing_class(**self.image_processor_dict )
       # create random PIL images
       a_      :	str      =							prepare_image_inputs(self.image_processor_tester							,		equal_resolution=lowerCAmelCase_ )
       for image in image_inputs:
         self.assertIsInstance(lowerCAmelCase_							,		Image.Image )

       # Test not batched input
       a_      :	Dict      =							image_processing(image_inputs[0]							,		return_tensors="""pt""" ).pixel_values
       self.assertEqual(
           encoded_images.shape							,		(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size["""height"""],
               self.image_processor_tester.size["""width"""],
           )							,		)

       # Test batched
       a_      :	int      =							image_processing(lowerCAmelCase_							,		return_tensors="""pt""" ).pixel_values
       self.assertEqual(
           encoded_images.shape							,		(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size["""height"""],
               self.image_processor_tester.size["""width"""],
           )							,		)


     @is_flaky()
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[str]      =							self.image_processing_class(**self.image_processor_dict )
       # create random numpy tensors
       a_      :	int      =							prepare_image_inputs(self.image_processor_tester							,		equal_resolution=lowerCAmelCase_							,		numpify=lowerCAmelCase_ )
       for image in image_inputs:
         self.assertIsInstance(lowerCAmelCase_							,		np.ndarray )

       # Test not batched input
       a_      :	Optional[Any]      =							image_processing(image_inputs[0]							,		return_tensors="""pt""" ).pixel_values
       self.assertEqual(
           encoded_images.shape							,		(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size["""height"""],
               self.image_processor_tester.size["""width"""],
           )							,		)

       # Test batched
       a_      :	Dict      =							image_processing(lowerCAmelCase_							,		return_tensors="""pt""" ).pixel_values
       self.assertEqual(
           encoded_images.shape							,		(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size["""height"""],
               self.image_processor_tester.size["""width"""],
           )							,		)






     @is_flaky()
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Union[str, Any]      =							self.image_processing_class(**self.image_processor_dict )
       # create random PyTorch tensors
       a_      :	Optional[Any]      =							prepare_image_inputs(self.image_processor_tester							,		equal_resolution=lowerCAmelCase_							,		torchify=lowerCAmelCase_ )
       for image in image_inputs:
         self.assertIsInstance(lowerCAmelCase_							,		torch.Tensor )

       # Test not batched input
       a_      :	Optional[int]      =							image_processing(image_inputs[0]							,		return_tensors="""pt""" ).pixel_values
       self.assertEqual(
           encoded_images.shape							,		(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size["""height"""],
               self.image_processor_tester.size["""width"""],
           )							,		)

       # Test batched
       a_      :	Any      =							image_processing(lowerCAmelCase_							,		return_tensors="""pt""" ).pixel_values
       self.assertEqual(
           encoded_images.shape							,		(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.size["""height"""],
               self.image_processor_tester.size["""width"""],
           )							,		)


