

'''simple docstring'''





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin





class A__    ( UpperCAmelCase_  ,		unittest.TestCase ):



 """simple docstring"""







 _lowercase     :					int      =						DebertaTokenizer
 _lowercase     :					Tuple      =						True
 _lowercase     :					Optional[Any]      =						DebertaTokenizerFast

 def    __magic_name__       (		self :	List[str]       ):






      '''simple docstring'''




      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      _lowerCAmelCase   :					Any        =    [
          'l',
          'o',
          'w',
          'e',
          'r',
          's',
          't',
          'i',
          'd',
          'n',
          '\u0120',
          '\u0120l',
          '\u0120n',
          '\u0120lo',
          '\u0120low',
          'er',
          '\u0120lowest',
          '\u0120newer',
          '\u0120wider',
          '[UNK]',
      ]
      _lowerCAmelCase   :					Any        =    dict(zip(_lowercase     ,						range(len(_lowercase       )       )       )       )
      _lowerCAmelCase   :					List[Any]        =    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
      _lowerCAmelCase   :					int        =    {'unk_token': '[UNK]'}

      _lowerCAmelCase   :					Tuple        =    os.path.join(self.tmpdirname     ,						VOCAB_FILES_NAMES["vocab_file"]       )
      _lowerCAmelCase   :					Any        =    os.path.join(self.tmpdirname     ,						VOCAB_FILES_NAMES["merges_file"]       )
      with open(self.vocab_file     ,						"w"     ,						encoding="utf-8"       ) as fp:
           fp.write(json.dumps(_lowercase       ) + "\n"       )
      with open(self.merges_file     ,						"w"     ,						encoding="utf-8"       ) as fp:
           fp.write("\n".join(_lowercase       )       )

 def    __magic_name__       (		self :	List[str]     ,						**A_ :	int       ):






      '''simple docstring'''




      kwargs.update(self.special_tokens_map       )
      return self.tokenizer_class.from_pretrained(self.tmpdirname     ,						**_lowercase       )

 def    __magic_name__       (		self :	List[Any]     ,						A_ :	List[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					str        =    'lower newer'
      _lowerCAmelCase   :					List[str]        =    'lower newer'
      return input_text, output_text

 def    __magic_name__       (		self :	List[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					int        =    self.get_tokenizer()
      _lowerCAmelCase   :					Any        =    'lower newer'
      _lowerCAmelCase   :					Optional[int]        =    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
      _lowerCAmelCase   :					Optional[Any]        =    tokenizer.tokenize(_lowercase       )
      self.assertListEqual(_lowercase     ,						_lowercase       )

      _lowerCAmelCase   :					Any        =    tokens + [tokenizer.unk_token]
      _lowerCAmelCase   :					List[str]        =    [0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase       )     ,						_lowercase       )

 def    __magic_name__       (		self :	str       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Dict        =    self.get_tokenizer()
      _lowerCAmelCase   :					int        =    tokenizer("Hello"     ,						"World"       )
      _lowerCAmelCase   :					Tuple        =    [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
      self.assertListEqual(tokd["token_type_ids"]     ,						_lowercase       )

 @slow
 def    __magic_name__       (		self :	Optional[Any]       ):






      '''simple docstring'''




      _lowerCAmelCase   :					List[Any]        =    self.tokenizer_class.from_pretrained("microsoft/deberta-base"       )

      _lowerCAmelCase   :					Union[str, Any]        =    tokenizer.encode("sequence builders"     ,						add_special_tokens=_lowercase       )
      _lowerCAmelCase   :					Dict        =    tokenizer.encode("multi-sequence build"     ,						add_special_tokens=_lowercase       )

      _lowerCAmelCase   :					Tuple        =    tokenizer.encode(
          "sequence builders"     ,						add_special_tokens=_lowercase     ,						add_prefix_space=_lowercase       )
      _lowerCAmelCase   :					Dict        =    tokenizer.encode(
          "sequence builders"     ,						"multi-sequence build"     ,						add_special_tokens=_lowercase     ,						add_prefix_space=_lowercase       )

      _lowerCAmelCase   :					Dict        =    tokenizer.build_inputs_with_special_tokens(_lowercase       )
      _lowerCAmelCase   :					Tuple        =    tokenizer.build_inputs_with_special_tokens(_lowercase     ,						_lowercase       )

      assert encoded_sentence == encoded_text_from_decode
      assert encoded_pair == encoded_pair_from_decode


 @slow
 def    __magic_name__       (		self :	int       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[Any]        =    [self.tokenizer_class]
      if self.test_rust_tokenizer:
           tokenizer_classes.append(self.rust_tokenizer_class       )

      for tokenizer_class in tokenizer_classes:
           _lowerCAmelCase   :					List[Any]        =    tokenizer_class.from_pretrained("microsoft/deberta-base"       )

           _lowerCAmelCase   :					str        =    [
               'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
               'ALBERT incorporates two parameter reduction techniques',
               'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
               ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
               ' vocabulary embedding.',
           ]

           _lowerCAmelCase   :					Union[str, Any]        =    tokenizer(_lowercase     ,						padding=_lowercase       )
           _lowerCAmelCase   :					Optional[int]        =    [tokenizer.decode(_lowercase     ,						skip_special_tokens=_lowercase       ) for seq in encoding['input_ids']]

           # fmt: off
           _lowerCAmelCase   :					Any        =    {
               'input_ids': [
                   [1, 2_1_1_8, 1_1_1_2_6, 5_6_5, 3_5, 8_3, 2_5_1_9_1, 1_6_3, 1_8_8_5_4, 1_3, 1_2_1_5_6, 1_2, 1_6_1_0_1, 2_5_3_7_6, 1_3_8_0_7, 9, 2_2_2_0_5, 2_7_8_9_3, 1_6_3_5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 2_1_1_8, 1_1_1_2_6, 5_6_5, 2_4_5_3_6, 8_0, 4_3_7_9_7, 4_8_7_8, 7_3_7_3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1_3_3, 7_8, 6_5, 1_6, 1_0, 3_7_2_4, 1_5_3_8, 3_3_1_8_3, 1_1_3_0_3, 4_3_7_9_7, 1_9_3_8, 4, 8_7_0, 2_4_1_6_5, 2_9_1_0_5, 5, 7_3_9, 3_2_6_4_4, 3_3_1_8_3, 1_1_3_0_3, 3_6_1_7_3, 8_8, 8_0, 6_5_0, 7_8_2_1, 4_5_9_4_0, 6, 5_2, 2_5_5_9, 5, 1_8_3_6, 9, 5, 7_3_9_7, 1_3_1_7_1, 3_1, 5, 1_8_3_6, 9, 3_2_6_4_4, 3_3_1_8_3, 1_1_3_0_3, 4, 2]
               ],
               'token_type_ids': [
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
               ],
               'attention_mask': [
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
               ]
           }
           # fmt: on

           _lowerCAmelCase   :					Any        =    [
               'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
               'ALBERT incorporates two parameter reduction techniques',
               'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
               ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
               ' vocabulary embedding.',
           ]

           self.assertDictEqual(encoding.data     ,						_lowercase       )

           for expected, decoded in zip(_lowercase     ,						_lowercase       ):
                self.assertEqual(_lowercase     ,						_lowercase       )









from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
  import torch


__UpperCAmelCase    				=     logging.get_logger(__name__)





class A__    ( A ):



 """simple docstring"""







 _lowercase     :					int      =						['''pixel_values''']

 def __init__(		self :	Union[str, Any]     ,						A_ :	bool = True     ,						A_ :	Optional[Dict[str, int]] = None     ,						A_ :	PILImageResampling = PILImageResampling.BILINEAR     ,						A_ :	bool = True     ,						A_ :	Dict[str, int] = None     ,						A_ :	bool = True     ,						A_ :	Union[int, float] = 1 / 2_5_5     ,						A_ :	bool = True     ,						A_ :	Optional[Union[float, List[float]]] = None     ,						A_ :	Optional[Union[float, List[float]]] = None     ,						**A_ :	Optional[Any]     ,						):






      '''simple docstring'''




      super().__init__(**A_       )
      _lowerCAmelCase   :					Optional[Any]        =    size if size is not None else {"shortest_edge": 2_5_6}
      _lowerCAmelCase   :					Optional[int]        =    get_size_dict(A_     ,						default_to_square=A_       )
      _lowerCAmelCase   :					Dict        =    crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
      _lowerCAmelCase   :					Tuple        =    get_size_dict(A_     ,						param_name="crop_size"       )
      _lowerCAmelCase   :					Dict        =    do_resize
      _lowerCAmelCase   :					Optional[Any]        =    size
      _lowerCAmelCase   :					Union[str, Any]        =    resample
      _lowerCAmelCase   :					List[Any]        =    do_center_crop
      _lowerCAmelCase   :					str        =    crop_size
      _lowerCAmelCase   :					int        =    do_rescale
      _lowerCAmelCase   :					Tuple        =    rescale_factor
      _lowerCAmelCase   :					str        =    do_normalize
      _lowerCAmelCase   :					List[Any]        =    image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
      _lowerCAmelCase   :					str        =    image_std if image_std is not None else IMAGENET_STANDARD_STD

 def    __magic_name__       (		self :	Dict     ,						A_ :	np.ndarray     ,						A_ :	Dict[str, int]     ,						A_ :	PILImageResampling = PILImageResampling.BICUBIC     ,						A_ :	Optional[Union[str, ChannelDimension]] = None     ,						**A_ :	Tuple     ,						):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[int]        =    get_size_dict(A_     ,						default_to_square=A_       )
      if "shortest_edge" not in size:
           raise ValueError(F'''The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'''       )
      _lowerCAmelCase   :					Optional[Any]        =    get_resize_output_image_size(A_     ,						size=size["shortest_edge"]     ,						default_to_square=A_       )
      return resize(A_     ,						size=A_     ,						resample=A_     ,						data_format=A_     ,						**A_       )

 def    __magic_name__       (		self :	List[Any]     ,						A_ :	np.ndarray     ,						A_ :	Dict[str, int]     ,						A_ :	Optional[Union[str, ChannelDimension]] = None     ,						**A_ :	Dict     ,						):






      '''simple docstring'''




      _lowerCAmelCase   :					Any        =    get_size_dict(A_       )
      if "height" not in size or "width" not in size:
           raise ValueError(F'''The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}'''       )
      return center_crop(A_     ,						size=(size["height"], size["width"])     ,						data_format=A_     ,						**A_       )

 def    __magic_name__       (		self :	int     ,						A_ :	np.ndarray     ,						A_ :	float     ,						A_ :	Optional[Union[str, ChannelDimension]] = None     ,						**A_ :	List[Any]       ):






      '''simple docstring'''




      return rescale(A_     ,						scale=A_     ,						data_format=A_     ,						**A_       )

 def    __magic_name__       (		self :	Tuple     ,						A_ :	np.ndarray     ,						A_ :	Union[float, List[float]]     ,						A_ :	Union[float, List[float]]     ,						A_ :	Optional[Union[str, ChannelDimension]] = None     ,						**A_ :	List[Any]     ,						):






      '''simple docstring'''




      return normalize(A_     ,						mean=A_     ,						std=A_     ,						data_format=A_     ,						**A_       )

 def    __magic_name__       (		self :	List[str]     ,						A_ :	ImageInput     ,						A_ :	Optional[bool] = None     ,						A_ :	Dict[str, int] = None     ,						A_ :	PILImageResampling = None     ,						A_ :	bool = None     ,						A_ :	Dict[str, int] = None     ,						A_ :	Optional[bool] = None     ,						A_ :	Optional[float] = None     ,						A_ :	Optional[bool] = None     ,						A_ :	Optional[Union[float, List[float]]] = None     ,						A_ :	Optional[Union[float, List[float]]] = None     ,						A_ :	Optional[Union[str, TensorType]] = None     ,						A_ :	Union[str, ChannelDimension] = ChannelDimension.FIRST     ,						**A_ :	int     ,						):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[int]        =    do_resize if do_resize is not None else self.do_resize
      _lowerCAmelCase   :					Optional[Any]        =    size if size is not None else self.size
      _lowerCAmelCase   :					Optional[Any]        =    get_size_dict(A_     ,						default_to_square=A_       )
      _lowerCAmelCase   :					Tuple        =    resample if resample is not None else self.resample
      _lowerCAmelCase   :					str        =    do_center_crop if do_center_crop is not None else self.do_center_crop
      _lowerCAmelCase   :					List[Any]        =    crop_size if crop_size is not None else self.crop_size
      _lowerCAmelCase   :					str        =    get_size_dict(A_     ,						param_name="crop_size"       )
      _lowerCAmelCase   :					Union[str, Any]        =    do_rescale if do_rescale is not None else self.do_rescale
      _lowerCAmelCase   :					List[str]        =    rescale_factor if rescale_factor is not None else self.rescale_factor
      _lowerCAmelCase   :					int        =    do_normalize if do_normalize is not None else self.do_normalize
      _lowerCAmelCase   :					Optional[Any]        =    image_mean if image_mean is not None else self.image_mean
      _lowerCAmelCase   :					Any        =    image_std if image_std is not None else self.image_std

      _lowerCAmelCase   :					str        =    make_list_of_images(A_       )

      if not valid_images(A_       ):
           raise ValueError(
               "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
               "torch.Tensor, tf.Tensor or jax.ndarray."       )

      if do_resize and size is None:
           raise ValueError("Size must be specified if do_resize is True."       )

      if do_center_crop and crop_size is None:
           raise ValueError("Crop size must be specified if do_center_crop is True."       )

      if do_rescale and rescale_factor is None:
           raise ValueError("Rescale factor must be specified if do_rescale is True."       )

      if do_normalize and (image_mean is None or image_std is None):
           raise ValueError("Image mean and std must be specified if do_normalize is True."       )

      # All transformations expect numpy arrays.
      _lowerCAmelCase   :					Any        =    [to_numpy_array(A_       ) for image in images]

      if do_resize:
           _lowerCAmelCase   :					str        =    [self.resize(image=A_     ,						size=A_     ,						resample=A_       ) for image in images]

      if do_center_crop:
           _lowerCAmelCase   :					List[Any]        =    [self.center_crop(image=A_     ,						size=A_       ) for image in images]

      if do_rescale:
           _lowerCAmelCase   :					Dict        =    [self.rescale(image=A_     ,						scale=A_       ) for image in images]

      if do_normalize:
           _lowerCAmelCase   :					Tuple        =    [self.normalize(image=A_     ,						mean=A_     ,						std=A_       ) for image in images]

      _lowerCAmelCase   :					List[Any]        =    [to_channel_dimension_format(A_     ,						A_       ) for image in images]

      _lowerCAmelCase   :					Tuple        =    {"pixel_values": images}
      return BatchFeature(data=A_     ,						tensor_type=A_       )


 def    __magic_name__       (		self :	Optional[int]     ,						A_ :	Tuple     ,						A_ :	List[Tuple] = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					int        =    outputs.logits

      # Resize logits and compute semantic segmentation maps
      if target_sizes is not None:
           if len(A_       ) != len(A_       ):
                raise ValueError(
                    "Make sure that you pass in as many target sizes as the batch dimension of the logits"       )

           if is_torch_tensor(A_       ):
                _lowerCAmelCase   :					List[str]        =    target_sizes.numpy()

           _lowerCAmelCase   :					Tuple        =    []

           for idx in range(len(A_       )       ):
                _lowerCAmelCase   :					Tuple        =    torch.nn.functional.interpolate(
                    logits[idx].unsqueeze(dim=0       )     ,						size=target_sizes[idx]     ,						mode="bilinear"     ,						align_corners=A_       )
                _lowerCAmelCase   :					str        =    resized_logits[0].argmax(dim=0       )
                semantic_segmentation.append(A_       )
      else:
           _lowerCAmelCase   :					List[Any]        =    logits.argmax(dim=1       )
           _lowerCAmelCase   :					str        =    [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]       )]

      return semantic_segmentation






