




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
  import PIL


_snake_case            = logging.get_logger(__name__)







class 		lowercase							(     UpperCamelCase__	):
     _a											=		["pixel_values"]

     def __init__(						self	,  _a = True	,  _a = None	,  _a = None	,  _a = PILImageResampling.BILINEAR	,  _a = True	,  _a = 1 / 255	,  _a = True	,  _a = None	,  _a = None	,  **_a	,  )		->							None:
           super().__init__(**_a       )
           _A							:  int    =		size if size is not None else {"""shortest_edge""": 384}
           _A							:  str    =		get_size_dict(_a	,  default_to_square=_a       )

           _A							:  str    =		do_resize
           _A							:  Optional[Any]    =		size
           # Default value set here for backwards compatibility where the value in config is None
           _A							:  List[str]    =		crop_pct if crop_pct is not None else 224 / 256
           _A							:  List[Any]    =		resample
           _A							:  Union[str, Any]    =		do_rescale
           _A							:  Optional[int]    =		rescale_factor
           _A							:  List[Any]    =		do_normalize
           _A							:  int    =		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
           _A							:  Union[str, Any]    =		image_std if image_std is not None else IMAGENET_STANDARD_STD

     def       a__			(						self	,  _a	,  _a	,  _a	,  _a = PILImageResampling.BICUBIC	,  _a = None	,  **_a	,  )		->							np.ndarray:
           _A							:  Dict    =		get_size_dict(_a	,  default_to_square=_a       )
           if "shortest_edge" not in size:
                 raise ValueError(F'''Size dictionary must contain \'shortest_edge\' key. Got {size.keys()}'''       )
           _A							:  List[str]    =		size["""shortest_edge"""]

           if shortest_edge < 384:
                 # maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
                 _A							:  Any    =		int(shortest_edge / crop_pct       )
                 _A							:  Union[str, Any]    =		get_resize_output_image_size(_a	,  size=_a	,  default_to_square=_a       )
                 _A							:  Any    =		resize(image=_a	,  size=_a	,  resample=_a	,  data_format=_a	,  **_a       )
                 # then crop to (shortest_edge, shortest_edge)
                 return center_crop(image=_a	,  size=(shortest_edge, shortest_edge)	,  data_format=_a	,  **_a       )
           else:
                 # warping (no cropping) when evaluated at 384 or larger
                 return resize(
                     _a	,  size=(shortest_edge, shortest_edge)	,  resample=_a	,  data_format=_a	,  **_a       )

     def       a__			(						self	,  _a	,  _a	,  _a = None	,  **_a	,  )		->							Dict:
           return rescale(_a	,  scale=_a	,  data_format=_a	,  **_a       )

     def       a__			(						self	,  _a	,  _a	,  _a	,  _a = None	,  **_a	,  )		->							np.ndarray:
           return normalize(_a	,  mean=_a	,  std=_a	,  data_format=_a	,  **_a       )



     def       a__			(						self	,  _a	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = None	,  _a = ChannelDimension.FIRST	,  **_a	,  )		->							PIL.Image.Image:
           _A							:  Optional[int]    =		do_resize if do_resize is not None else self.do_resize
           _A							:  str    =		crop_pct if crop_pct is not None else self.crop_pct
           _A							:  Tuple    =		resample if resample is not None else self.resample
           _A							:  Union[str, Any]    =		do_rescale if do_rescale is not None else self.do_rescale
           _A							:  Any    =		rescale_factor if rescale_factor is not None else self.rescale_factor
           _A							:  Union[str, Any]    =		do_normalize if do_normalize is not None else self.do_normalize
           _A							:  Tuple    =		image_mean if image_mean is not None else self.image_mean
           _A							:  List[str]    =		image_std if image_std is not None else self.image_std

           _A							:  Any    =		size if size is not None else self.size
           _A							:  str    =		get_size_dict(_a	,  default_to_square=_a       )

           _A							:  List[str]    =		make_list_of_images(_a       )

           if not valid_images(_a       ):
                 raise ValueError(
                     """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                     """torch.Tensor, tf.Tensor or jax.ndarray."""       )

           if do_resize and size is None or resample is None:
                 raise ValueError("""Size and resample must be specified if do_resize is True."""       )

           if do_resize and size["shortest_edge"] < 384 and crop_pct is None:
                 raise ValueError("""crop_pct must be specified if size < 384."""       )

           if do_rescale and rescale_factor is None:
                 raise ValueError("""Rescale factor must be specified if do_rescale is True."""       )

           if do_normalize and (image_mean is None or image_std is None):
                 raise ValueError("""Image mean and std must be specified if do_normalize is True."""       )

           # All transformations expect numpy arrays.
           _A							:  List[Any]    =		[to_numpy_array(_a       ) for image in images]

           if do_resize:
                 _A							:  Dict    =		[self.resize(image=_a	,  size=_a	,  crop_pct=_a	,  resample=_a       ) for image in images]

           if do_rescale:
                 _A							:  Optional[int]    =		[self.rescale(image=_a	,  scale=_a       ) for image in images]

           if do_normalize:
                 _A							:  Optional[int]    =		[self.normalize(image=_a	,  mean=_a	,  std=_a       ) for image in images]

           _A							:  Union[str, Any]    =		[to_channel_dimension_format(_a	,  _a       ) for image in images]

           _A							:  Tuple    =		{"""pixel_values""": images}
           return BatchFeature(data=_a	,  tensor_type=_a       )







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		lowercase							(     UpperCamelCase__,unittest.TestCase	):
     _a											=		CLIPTokenizer
     _a											=		CLIPTokenizerFast
     _a											=		True
     _a											=		{}
     _a											=		False

     def       a__			(						self       )		->							Optional[Any]:
           super().setUp()

           # fmt: off
           _A							:  int    =		["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
           # fmt: on
           _A							:  str    =		dict(zip(_a	,  range(len(_a       )       )       )       )
           _A							:  Optional[int]    =		["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
           _A							:  str    =		{"""unk_token""": """<unk>"""}

           _A							:  Union[str, Any]    =		os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["""vocab_file"""]       )
           _A							:  Union[str, Any]    =		os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["""merges_file"""]       )
           with open(self.vocab_file	,  """w"""	,  encoding="""utf-8"""       ) as fp:
                 fp.write(json.dumps(_a       ) + """\n"""       )
           with open(self.merges_file	,  """w"""	,  encoding="""utf-8"""       ) as fp:
                 fp.write("""\n""".join(_a       )       )

     def       a__			(						self	,  **_a       )		->							List[Any]:
           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizer.from_pretrained(self.tmpdirname	,  **_a       )

     def       a__			(						self	,  **_a       )		->							Union[str, Any]:
           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname	,  **_a       )

     def       a__			(						self	,  _a       )		->							str:
           _A							:  Tuple    =		"""lower newer"""
           _A							:  Optional[Any]    =		"""lower newer"""
           return input_text, output_text

     def       a__			(						self       )		->							List[Any]:
           _A							:  Optional[int]    =		CLIPTokenizer(self.vocab_file	,  self.merges_file	,  **self.special_tokens_map       )
           _A							:  str    =		"""lower newer"""
           _A							:  List[str]    =		["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
           _A							:  int    =		tokenizer.tokenize(_a       )
           self.assertListEqual(_a	,  _a       )

           _A							:  str    =		tokens + [tokenizer.unk_token]
           _A							:  Dict    =		[10, 2, 16, 9, 3, 2, 16, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(_a       )	,  _a       )

     @require_ftfy
     def       a__			(						self       )		->							Tuple:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                       _A							:  Dict    =		self.tokenizer_class.from_pretrained(_a	,  **_a       )
                       _A							:  str    =		self.rust_tokenizer_class.from_pretrained(_a	,  **_a       )

                       _A							:  List[str]    =		"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
                       _A							:  Dict    =		tokenizer_s.tokenize(_a       )
                       _A							:  Dict    =		tokenizer_r.tokenize(_a       )

                       self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                       # with Tilde) encoded in 2 different ways
                       _A							:  Tuple    =		"""xa\u0303y""" + """ """ + """x\xe3y"""
                       _A							:  Dict    =		tokenizer_s.tokenize(_a       )
                       _A							:  Dict    =		tokenizer_r.tokenize(_a       )

                       self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on unicode of space type
                       _A							:  Any    =		[
                           """\u0009""",  # (horizontal tab, '\t')
                           """\u000B""",  # (vertical tab)
                           """\u000C""",  # (form feed)
                           """\u0020""",  # (space, ' ')
                           """\u200E""",  # (left-to-right mark):w
                           """\u200F""",  # (right-to-left mark)
                       ]
                       for unicode_seq in spaces_unicodes:
                             _A							:  Optional[int]    =		tokenizer_s.tokenize(_a       )
                             _A							:  List[Any]    =		tokenizer_r.tokenize(_a       )

                             self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on unicode of line break type
                       _A							:  int    =		[
                           """\u000A""",  # (line feed, '\n')
                           """\r\n""",  # (carriage return and line feed, '\r\n')
                           """\u000D""",  # (carriage return, '\r')
                           """\r""",  # (carriage return, '\r')
                           """\u000D""",  # (carriage return, '\r')
                           """\u2028""",  # (line separator)
                           """\u2029""",  # (paragraph separator)
                           # "\u0085", # (next line)
                       ]

                       # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                       # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                       # space (and thus into an empty list).

                       for unicode_seq in line_break_unicodes:
                             _A							:  Optional[Any]    =		tokenizer_s.tokenize(_a       )
                             _A							:  Tuple    =		tokenizer_r.tokenize(_a       )

                             self.assertListEqual(_a	,  _a       )

     def       a__			(						self       )		->							Any:
           # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                       _A							:  List[Any]    =		"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                       _A							:  str    =		F'''{text_of_1_token} {text_of_1_token}'''

                       _A							:  str    =		self.rust_tokenizer_class.from_pretrained(
                           _a	,  use_fast=_a	,  )
                       _A							:  Dict    =		tokenizer_r(_a	,  return_offsets_mapping=_a	,  add_special_tokens=_a       )
                       self.assertEqual(encoding.offset_mapping[0]	,  (0, len(_a       ))       )
                       self.assertEqual(
                           encoding.offset_mapping[1]	,  (len(_a       ) + 1, len(_a       ) + 1 + len(_a       ))	,  )

                       _A							:  Any    =		F''' {text}'''

                       _A							:  List[Any]    =		self.rust_tokenizer_class.from_pretrained(
                           _a	,  use_fast=_a	,  )
                       _A							:  Optional[int]    =		tokenizer_r(_a	,  return_offsets_mapping=_a	,  add_special_tokens=_a       )
                       self.assertEqual(encoding.offset_mapping[0]	,  (1, 1 + len(_a       ))       )
                       self.assertEqual(
                           encoding.offset_mapping[1]	,  (1 + len(_a       ) + 1, 1 + len(_a       ) + 1 + len(_a       ))	,  )

     def       a__			(						self       )		->							List[Any]:
           # Test related to the breaking change introduced in transformers v4.17.0
           # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
           with self.assertRaises(_a       ) as context:
                 self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""       )

           self.assertTrue(
               context.exception.args[0].startswith(
                   """The `backend_tokenizer` provided does not match the expected format."""       )       )

     @require_ftfy
     def       a__			(						self       )		->							str:
           super().test_tokenization_python_rust_equals()



     def       a__			(						self       )		->							Union[str, Any]:
           # CLIP always lower cases letters
           pass

