




import json
import pathlib
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     from PIL import Image

     from transformers import YolosImageProcessor




class    __lowerCAmelCase       (							unittest.TestCase):






       def __init__(   self:					Optional[Any]       ,					_lowerCAmelCase:					List[str]       ,					_lowerCAmelCase:					int=7       ,					_lowerCAmelCase:					List[str]=3       ,					_lowerCAmelCase:					Optional[int]=30       ,					_lowerCAmelCase:					List[Any]=4_00       ,					_lowerCAmelCase:					Optional[int]=True       ,					_lowerCAmelCase:					Optional[Any]=None       ,					_lowerCAmelCase:					List[str]=True       ,					_lowerCAmelCase:					Union[str, Any]=[0.5, 0.5, 0.5]       ,					_lowerCAmelCase:					int=[0.5, 0.5, 0.5]       ,					_lowerCAmelCase:					int=True       ,					_lowerCAmelCase:					str=1 / 2_55       ,					_lowerCAmelCase:					Union[str, Any]=True       ,					):
         # by setting size["longest_edge"] > max_resolution we're effectively not testing this :p
         lowercase    :List[str]      	=						size if size is not None else {"shortest_edge": 18, "longest_edge": 13_33}
         lowercase    :Dict      	=						parent
         lowercase    :List[str]      	=						batch_size
         lowercase    :List[Any]      	=						num_channels
         lowercase    :Optional[Any]      	=						min_resolution
         lowercase    :Any      	=						max_resolution
         lowercase    :Optional[int]      	=						do_resize
         lowercase    :Any      	=						size
         lowercase    :Optional[int]      	=						do_normalize
         lowercase    :Optional[Any]      	=						image_mean
         lowercase    :Optional[Any]      	=						image_std
         lowercase    :str      	=						do_rescale
         lowercase    :Union[str, Any]      	=						rescale_factor
         lowercase    :List[Any]      	=						do_pad






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]				):
         return {
             "do_resize": self.do_resize,
             "size": self.size,
             "do_normalize": self.do_normalize,
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_rescale": self.do_rescale,
             "rescale_factor": self.rescale_factor,
             "do_pad": self.do_pad,
         }






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					List[str]       ,					_lowerCAmelCase:					Tuple=False				):
         if not batched:
           lowercase    :Optional[Any]      	=						image_inputs[0]
           if isinstance(_lowerCAmelCase       ,					Image.Image				):
             lowercase	,    lowercase    :str      	=						image.size
           else:
             lowercase	,    lowercase    :int      	=						image.shape[1], image.shape[2]
           if w < h:
             lowercase    :List[str]      	=						int(self.size["shortest_edge"] * h / w				)
             lowercase    :Optional[Any]      	=						self.size["shortest_edge"]
           elif w > h:
             lowercase    :Any      	=						self.size["shortest_edge"]
             lowercase    :Dict      	=						int(self.size["shortest_edge"] * w / h				)
           else:
             lowercase    :str      	=						self.size["shortest_edge"]
             lowercase    :List[str]      	=						self.size["shortest_edge"]

         else:
           lowercase    :Any      	=						[]
           for image in image_inputs:
             lowercase	,    lowercase    :List[Any]      	=						self.get_expected_values([image]				)
             expected_values.append((expected_height, expected_width)				)
           lowercase    :Optional[Any]      	=						max(_lowerCAmelCase       ,					key=lambda _lowerCAmelCase				: item[0]				)[0]
           lowercase    :Optional[int]      	=						max(_lowerCAmelCase       ,					key=lambda _lowerCAmelCase				: item[1]				)[1]

         return expected_height, expected_width







@require_torch
@require_vision
class    __lowerCAmelCase       (							lowerCAmelCase      , unittest.TestCase):
       _a 					=						YolosImageProcessor if is_vision_available() else None






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :List[Any]      	=						YolosImageProcessingTester(self				)






       @property
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]				):
         return self.image_processor_tester.prepare_image_processor_dict()






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :str      	=						self.image_processing_class(**self.image_processor_dict				)
         self.assertTrue(hasattr(_lowerCAmelCase       ,					"image_mean"				)				)
         self.assertTrue(hasattr(_lowerCAmelCase       ,					"image_std"				)				)
         self.assertTrue(hasattr(_lowerCAmelCase       ,					"do_normalize"				)				)
         self.assertTrue(hasattr(_lowerCAmelCase       ,					"do_resize"				)				)
         self.assertTrue(hasattr(_lowerCAmelCase       ,					"size"				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         lowercase    :List[str]      	=						self.image_processing_class.from_dict(self.image_processor_dict				)
         self.assertEqual(image_processor.size       ,					{"shortest_edge": 18, "longest_edge": 13_33}				)
         self.assertEqual(image_processor.do_pad       ,					_lowerCAmelCase				)

         lowercase    :List[Any]      	=						self.image_processing_class.from_dict(
             self.image_processor_dict       ,					size=42       ,					max_size=84       ,					pad_and_return_pixel_mask=_lowerCAmelCase				)
         self.assertEqual(image_processor.size       ,					{"shortest_edge": 42, "longest_edge": 84}				)
         self.assertEqual(image_processor.do_pad       ,					_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         pass






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         # Initialize image_processing
         lowercase    :str      	=						self.image_processing_class(**self.image_processor_dict				)
         # create random PIL images
         lowercase    :Union[str, Any]      	=						prepare_image_inputs(self.image_processor_tester       ,					equal_resolution=_lowerCAmelCase				)
         for image in image_inputs:
           self.assertIsInstance(_lowerCAmelCase       ,					Image.Image				)

         # Test not batched input
         lowercase    :Optional[int]      	=						image_processing(image_inputs[0]       ,					return_tensors="pt"				).pixel_values

         lowercase	,    lowercase    :int      	=						self.image_processor_tester.get_expected_values(_lowerCAmelCase				)

         self.assertEqual(
             encoded_images.shape       ,					(1, self.image_processor_tester.num_channels, expected_height, expected_width)       ,					)

         # Test batched
         lowercase	,    lowercase    :Union[str, Any]      	=						self.image_processor_tester.get_expected_values(_lowerCAmelCase       ,					batched=_lowerCAmelCase				)

         lowercase    :str      	=						image_processing(_lowerCAmelCase       ,					return_tensors="pt"				).pixel_values
         self.assertEqual(
             encoded_images.shape       ,					(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )       ,					)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         # Initialize image_processing
         lowercase    :Optional[Any]      	=						self.image_processing_class(**self.image_processor_dict				)
         # create random numpy tensors
         lowercase    :Optional[int]      	=						prepare_image_inputs(self.image_processor_tester       ,					equal_resolution=_lowerCAmelCase       ,					numpify=_lowerCAmelCase				)
         for image in image_inputs:
           self.assertIsInstance(_lowerCAmelCase       ,					np.ndarray				)

         # Test not batched input
         lowercase    :int      	=						image_processing(image_inputs[0]       ,					return_tensors="pt"				).pixel_values

         lowercase	,    lowercase    :Union[str, Any]      	=						self.image_processor_tester.get_expected_values(_lowerCAmelCase				)

         self.assertEqual(
             encoded_images.shape       ,					(1, self.image_processor_tester.num_channels, expected_height, expected_width)       ,					)

         # Test batched
         lowercase    :Tuple      	=						image_processing(_lowerCAmelCase       ,					return_tensors="pt"				).pixel_values

         lowercase	,    lowercase    :Union[str, Any]      	=						self.image_processor_tester.get_expected_values(_lowerCAmelCase       ,					batched=_lowerCAmelCase				)

         self.assertEqual(
             encoded_images.shape       ,					(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )       ,					)






       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         # Initialize image_processing
         lowercase    :str      	=						self.image_processing_class(**self.image_processor_dict				)
         # create random PyTorch tensors
         lowercase    :str      	=						prepare_image_inputs(self.image_processor_tester       ,					equal_resolution=_lowerCAmelCase       ,					torchify=_lowerCAmelCase				)
         for image in image_inputs:
           self.assertIsInstance(_lowerCAmelCase       ,					torch.Tensor				)

         # Test not batched input
         lowercase    :Optional[Any]      	=						image_processing(image_inputs[0]       ,					return_tensors="pt"				).pixel_values

         lowercase	,    lowercase    :Union[str, Any]      	=						self.image_processor_tester.get_expected_values(_lowerCAmelCase				)

         self.assertEqual(
             encoded_images.shape       ,					(1, self.image_processor_tester.num_channels, expected_height, expected_width)       ,					)

         # Test batched
         lowercase    :Any      	=						image_processing(_lowerCAmelCase       ,					return_tensors="pt"				).pixel_values

         lowercase	,    lowercase    :Tuple      	=						self.image_processor_tester.get_expected_values(_lowerCAmelCase       ,					batched=_lowerCAmelCase				)

         self.assertEqual(
             encoded_images.shape       ,					(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )       ,					)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         # Initialize image_processings
         lowercase    :List[str]      	=						self.image_processing_class(**self.image_processor_dict				)
         lowercase    :Any      	=						self.image_processing_class(do_resize=_lowerCAmelCase       ,					do_normalize=_lowerCAmelCase       ,					do_rescale=_lowerCAmelCase				)
         # create random PyTorch tensors
         lowercase    :Union[str, Any]      	=						prepare_image_inputs(self.image_processor_tester       ,					equal_resolution=_lowerCAmelCase       ,					torchify=_lowerCAmelCase				)
         for image in image_inputs:
           self.assertIsInstance(_lowerCAmelCase       ,					torch.Tensor				)

         # Test whether the method "pad" and calling the image processor return the same tensors
         lowercase    :Tuple      	=						image_processing_a.pad(_lowerCAmelCase       ,					return_tensors="pt"				)
         lowercase    :Dict      	=						image_processing_a(_lowerCAmelCase       ,					return_tensors="pt"				)

         self.assertTrue(
             torch.allclose(encoded_images_with_method["pixel_values"]       ,					encoded_images["pixel_values"]       ,					atol=1e-4				)				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         # prepare image and target
         lowercase    :List[str]      	=						Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"				)
         with open("./tests/fixtures/tests_samples/COCO/coco_annotations.txt"       ,					"r"				) as f:
           lowercase    :List[Any]      	=						json.loads(f.read()				)

         lowercase    :List[Any]      	=						{"image_id": 3_97_69, "annotations": target}

         # encode them
         lowercase    :Dict      	=						YolosImageProcessor.from_pretrained("hustvl/yolos-small"				)
         lowercase    :Union[str, Any]      	=						image_processing(images=_lowerCAmelCase       ,					annotations=_lowerCAmelCase       ,					return_tensors="pt"				)

         # verify pixel values
         lowercase    :Any      	=						torch.Size([1, 3, 8_00, 10_66]				)
         self.assertEqual(encoding["pixel_values"].shape       ,					_lowerCAmelCase				)

         lowercase    :str      	=						torch.tensor([0.27_96, 0.31_38, 0.34_81]				)
         self.assertTrue(torch.allclose(encoding["pixel_values"][0, 0, 0, :3]       ,					_lowerCAmelCase       ,					atol=1e-4				)				)

         # verify area
         lowercase    :Optional[int]      	=						torch.tensor([58_87.96_00, 1_12_50.20_61, 48_93_53.84_38, 83_71_22.75_00, 14_79_67.51_56, 16_57_32.34_38]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["area"]       ,					_lowerCAmelCase				)				)
         # verify boxes
         lowercase    :List[str]      	=						torch.Size([6, 4]				)
         self.assertEqual(encoding["labels"][0]["boxes"].shape       ,					_lowerCAmelCase				)
         lowercase    :Tuple      	=						torch.tensor([0.55_03, 0.27_65, 0.06_04, 0.22_15]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["boxes"][0]       ,					_lowerCAmelCase       ,					atol=1e-3				)				)
         # verify image_id
         lowercase    :int      	=						torch.tensor([3_97_69]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["image_id"]       ,					_lowerCAmelCase				)				)
         # verify is_crowd
         lowercase    :List[str]      	=						torch.tensor([0, 0, 0, 0, 0, 0]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["iscrowd"]       ,					_lowerCAmelCase				)				)
         # verify class_labels
         lowercase    :Optional[int]      	=						torch.tensor([75, 75, 63, 65, 17, 17]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["class_labels"]       ,					_lowerCAmelCase				)				)
         # verify orig_size
         lowercase    :int      	=						torch.tensor([4_80, 6_40]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["orig_size"]       ,					_lowerCAmelCase				)				)
         # verify size
         lowercase    :List[Any]      	=						torch.tensor([8_00, 10_66]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["size"]       ,					_lowerCAmelCase				)				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         # prepare image, target and masks_path
         lowercase    :Any      	=						Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"				)
         with open("./tests/fixtures/tests_samples/COCO/coco_panoptic_annotations.txt"       ,					"r"				) as f:
           lowercase    :List[Any]      	=						json.loads(f.read()				)

         lowercase    :List[str]      	=						{"file_name": "000000039769.png", "image_id": 3_97_69, "segments_info": target}

         lowercase    :int      	=						pathlib.Path("./tests/fixtures/tests_samples/COCO/coco_panoptic"				)

         # encode them
         lowercase    :Dict      	=						YolosImageProcessor(format="coco_panoptic"				)
         lowercase    :int      	=						image_processing(images=_lowerCAmelCase       ,					annotations=_lowerCAmelCase       ,					masks_path=_lowerCAmelCase       ,					return_tensors="pt"				)

         # verify pixel values
         lowercase    :Optional[Any]      	=						torch.Size([1, 3, 8_00, 10_66]				)
         self.assertEqual(encoding["pixel_values"].shape       ,					_lowerCAmelCase				)

         lowercase    :str      	=						torch.tensor([0.27_96, 0.31_38, 0.34_81]				)
         self.assertTrue(torch.allclose(encoding["pixel_values"][0, 0, 0, :3]       ,					_lowerCAmelCase       ,					atol=1e-4				)				)

         # verify area
         lowercase    :Optional[int]      	=						torch.tensor([14_79_79.68_75, 16_55_27.04_69, 48_46_38.59_38, 1_12_92.93_75, 58_79.65_62, 76_34.11_47]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["area"]       ,					_lowerCAmelCase				)				)
         # verify boxes
         lowercase    :str      	=						torch.Size([6, 4]				)
         self.assertEqual(encoding["labels"][0]["boxes"].shape       ,					_lowerCAmelCase				)
         lowercase    :str      	=						torch.tensor([0.26_25, 0.54_37, 0.46_88, 0.86_25]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["boxes"][0]       ,					_lowerCAmelCase       ,					atol=1e-3				)				)
         # verify image_id
         lowercase    :str      	=						torch.tensor([3_97_69]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["image_id"]       ,					_lowerCAmelCase				)				)
         # verify is_crowd
         lowercase    :Any      	=						torch.tensor([0, 0, 0, 0, 0, 0]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["iscrowd"]       ,					_lowerCAmelCase				)				)
         # verify class_labels
         lowercase    :Any      	=						torch.tensor([17, 17, 63, 75, 75, 93]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["class_labels"]       ,					_lowerCAmelCase				)				)
         # verify masks
         lowercase    :List[str]      	=						82_28_73
         self.assertEqual(encoding["labels"][0]["masks"].sum().item()       ,					_lowerCAmelCase				)
         # verify orig_size
         lowercase    :Tuple      	=						torch.tensor([4_80, 6_40]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["orig_size"]       ,					_lowerCAmelCase				)				)
         # verify size
         lowercase    :List[str]      	=						torch.tensor([8_00, 10_66]				)
         self.assertTrue(torch.allclose(encoding["labels"][0]["size"]       ,					_lowerCAmelCase				)				)









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase	:						Optional[int]			=     logging.get_logger(__name__)

_UpperCAmelCase	:						Union[str, Any]			=     {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase	:						Optional[Any]			=     {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

_UpperCAmelCase	:						Tuple			=     {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

_UpperCAmelCase	:						Dict			=     {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						VOCAB_FILES_NAMES
       _a 					=						PRETRAINED_VOCAB_FILES_MAP
       _a 					=						PRETRAINED_INIT_CONFIGURATION
       _a 					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a 					=						RealmTokenizer






       def __init__(   self:					Tuple       ,					_lowerCAmelCase:					int=None       ,					_lowerCAmelCase:					List[Any]=None       ,					_lowerCAmelCase:					Optional[int]=True       ,					_lowerCAmelCase:					Optional[Any]="[UNK]"       ,					_lowerCAmelCase:					Dict="[SEP]"       ,					_lowerCAmelCase:					List[str]="[PAD]"       ,					_lowerCAmelCase:					Optional[int]="[CLS]"       ,					_lowerCAmelCase:					List[Any]="[MASK]"       ,					_lowerCAmelCase:					Tuple=True       ,					_lowerCAmelCase:					Tuple=None       ,					**_lowerCAmelCase:					Union[str, Any]       ,					):
         super().__init__(
             _lowerCAmelCase       ,					tokenizer_file=_lowerCAmelCase       ,					do_lower_case=_lowerCAmelCase       ,					unk_token=_lowerCAmelCase       ,					sep_token=_lowerCAmelCase       ,					pad_token=_lowerCAmelCase       ,					cls_token=_lowerCAmelCase       ,					mask_token=_lowerCAmelCase       ,					tokenize_chinese_chars=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)

         lowercase    :List[str]      	=						json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             normalizer_state.get("lowercase"       ,					_lowerCAmelCase				) != do_lower_case
             or normalizer_state.get("strip_accents"       ,					_lowerCAmelCase				) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,					_lowerCAmelCase				) != tokenize_chinese_chars
         ):
           lowercase    :List[str]      	=						getattr(_lowerCAmelCase       ,					normalizer_state.pop("type"				)				)
           lowercase    :List[str]      	=						do_lower_case
           lowercase    :Any      	=						strip_accents
           lowercase    :Tuple      	=						tokenize_chinese_chars
           lowercase    :Any      	=						normalizer_class(**_lowerCAmelCase				)

         lowercase    :Any      	=						do_lower_case






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Optional[Any]       ,					**_lowerCAmelCase:					Optional[int]				):
         lowercase    :List[str]      	=						PaddingStrategy.MAX_LENGTH

         lowercase    :Optional[Any]      	=						text
         lowercase    :str      	=						kwargs.pop("text_pair"       ,					_lowerCAmelCase				)
         lowercase    :Any      	=						kwargs.pop("return_tensors"       ,					_lowerCAmelCase				)

         lowercase    :List[str]      	=						{
             "input_ids": [],
             "attention_mask": [],
             "token_type_ids": [],
         }

         for idx, candidate_text in enumerate(_lowerCAmelCase				):
           if batch_text_pair is not None:
             lowercase    :Any      	=						batch_text_pair[idx]
           else:
             lowercase    :Optional[Any]      	=						None

           lowercase    :Optional[Any]      	=						super().__call__(_lowerCAmelCase       ,					_lowerCAmelCase       ,					return_tensors=_lowerCAmelCase       ,					**_lowerCAmelCase				)

           lowercase    :Tuple      	=						encoded_candidates.get("input_ids"				)
           lowercase    :Optional[Any]      	=						encoded_candidates.get("attention_mask"				)
           lowercase    :int      	=						encoded_candidates.get("token_type_ids"				)

           if encoded_input_ids is not None:
             output_data["input_ids"].append(_lowerCAmelCase				)
           if encoded_attention_mask is not None:
             output_data["attention_mask"].append(_lowerCAmelCase				)
           if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(_lowerCAmelCase				)

         lowercase    :str      	=						{key: item for key, item in output_data.items() if len(_lowerCAmelCase				) != 0}

         return BatchEncoding(_lowerCAmelCase       ,					tensor_type=_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Any       ,					_lowerCAmelCase:					Tuple=None				):
         lowercase    :Dict      	=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple       ,					_lowerCAmelCase:					List[int]       ,					_lowerCAmelCase:					Optional[List[int]] = None				):
         lowercase    :List[str]      	=						[self.sep_token_id]
         lowercase    :List[Any]      	=						[self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					str       ,					_lowerCAmelCase:					Optional[str] = None				):
         lowercase    :Tuple      	=						self._tokenizer.model.save(_lowerCAmelCase       ,					name=_lowerCAmelCase				)
         return tuple(_lowerCAmelCase				)



