

'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							_snake_case (       _a	,  unittest.TestCase						):
   _A       :     Optional[Any]       =					CLIPTokenizer
   _A       :     List[str]       =					CLIPTokenizerFast
   _A       :     int       =					True
   _A       :     Dict       =					{}
   _A       :     Union[str, Any]       =					False
   def     __UpperCamelCase						( self			:      Tuple							):
     super().setUp()

     # fmt: off
     SCREAMING_SNAKE_CASE:int						      =					["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
     # fmt: on
     SCREAMING_SNAKE_CASE:List[str]						      =					dict(zip(SCREAMING_SNAKE_CASE__     ,range(len(SCREAMING_SNAKE_CASE__							)							)							)							)
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
     SCREAMING_SNAKE_CASE:Any						      =					{"unk_token": "<unk>"}

     SCREAMING_SNAKE_CASE:str						      =					os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["vocab_file"]							)
     SCREAMING_SNAKE_CASE:Optional[int]						      =					os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["merges_file"]							)
     with open(self.vocab_file     ,"w"     ,encoding="utf-8"							) as fp:
       fp.write(json.dumps(SCREAMING_SNAKE_CASE__							) + "\n"							)
     with open(self.merges_file     ,"w"     ,encoding="utf-8"							) as fp:
       fp.write("\n".join(SCREAMING_SNAKE_CASE__							)							)
   def     __UpperCamelCase						( self			:      Optional[Any]     ,**SCREAMING_SNAKE_CASE__			:      List[Any]							):
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizer.from_pretrained(self.tmpdirname     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      List[str]     ,**SCREAMING_SNAKE_CASE__			:      Dict							):
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizerFast.from_pretrained(self.tmpdirname     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      Dict							):
     SCREAMING_SNAKE_CASE:str						      =					"lower newer"
     SCREAMING_SNAKE_CASE:List[Any]						      =					"lower newer"
     return input_text, output_text
   def     __UpperCamelCase						( self			:      Any							):
     SCREAMING_SNAKE_CASE:Optional[int]						      =					CLIPTokenizer(self.vocab_file     ,self.merges_file     ,**self.special_tokens_map							)
     SCREAMING_SNAKE_CASE:int						      =					"lower newer"
     SCREAMING_SNAKE_CASE:List[str]						      =					["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
     SCREAMING_SNAKE_CASE:List[str]						      =					tokenizer.tokenize(SCREAMING_SNAKE_CASE__							)
     self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Optional[int]						      =					tokens + [tokenizer.unk_token]
     SCREAMING_SNAKE_CASE:List[str]						      =					[10, 2, 16, 9, 3, 2, 16, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__							)     ,SCREAMING_SNAKE_CASE__							)
   @require_ftfy
   def     __UpperCamelCase						( self			:      Dict							):
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         SCREAMING_SNAKE_CASE:Dict						      =					self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
         SCREAMING_SNAKE_CASE:List[Any]						      =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)

         SCREAMING_SNAKE_CASE:int						      =					"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
         SCREAMING_SNAKE_CASE:Any						      =					tokenizer_s.tokenize(SCREAMING_SNAKE_CASE__							)
         SCREAMING_SNAKE_CASE:Dict						      =					tokenizer_r.tokenize(SCREAMING_SNAKE_CASE__							)

         self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)

         # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
         # with Tilde) encoded in 2 different ways
         SCREAMING_SNAKE_CASE:Dict						      =					"xa\u0303y" + " " + "x\xe3y"
         SCREAMING_SNAKE_CASE:Dict						      =					tokenizer_s.tokenize(SCREAMING_SNAKE_CASE__							)
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					tokenizer_r.tokenize(SCREAMING_SNAKE_CASE__							)

         self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)

         # Test that the tokenization is identical on unicode of space type
         SCREAMING_SNAKE_CASE:int						      =					[
             "\u0009",  # (horizontal tab, '\t')
             "\u000B",  # (vertical tab)
             "\u000C",  # (form feed)
             "\u0020",  # (space, ' ')
             "\u200E",  # (left-to-right mark):w
             "\u200F",  # (right-to-left mark)
         ]
         for unicode_seq in spaces_unicodes:
           SCREAMING_SNAKE_CASE:Tuple						      =					tokenizer_s.tokenize(SCREAMING_SNAKE_CASE__							)
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					tokenizer_r.tokenize(SCREAMING_SNAKE_CASE__							)

           self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)

         # Test that the tokenization is identical on unicode of line break type
         SCREAMING_SNAKE_CASE:Tuple						      =					[
             "\u000A",  # (line feed, '\n')
             "\r\n",  # (carriage return and line feed, '\r\n')
             "\u000D",  # (carriage return, '\r')
             "\r",  # (carriage return, '\r')
             "\u000D",  # (carriage return, '\r')
             "\u2028",  # (line separator)
             "\u2029",  # (paragraph separator)
             # "\u0085", # (next line)
         ]

         # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
         # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
         # space (and thus into an empty list).

         for unicode_seq in line_break_unicodes:
           SCREAMING_SNAKE_CASE:List[Any]						      =					tokenizer_s.tokenize(SCREAMING_SNAKE_CASE__							)
           SCREAMING_SNAKE_CASE:Optional[int]						      =					tokenizer_r.tokenize(SCREAMING_SNAKE_CASE__							)

           self.assertListEqual(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      int							):
     # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         SCREAMING_SNAKE_CASE:Dict						      =					"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
         SCREAMING_SNAKE_CASE:List[str]						      =					F'''{text_of_1_token} {text_of_1_token}'''

         SCREAMING_SNAKE_CASE:Optional[Any]						      =					self.rust_tokenizer_class.from_pretrained(
             SCREAMING_SNAKE_CASE__     ,use_fast=SCREAMING_SNAKE_CASE__     ,)
         SCREAMING_SNAKE_CASE:int						      =					tokenizer_r(SCREAMING_SNAKE_CASE__     ,return_offsets_mapping=SCREAMING_SNAKE_CASE__     ,add_special_tokens=SCREAMING_SNAKE_CASE__							)
         self.assertEqual(encoding.offset_mapping[0]     ,(0, len(SCREAMING_SNAKE_CASE__							))							)
         self.assertEqual(
             encoding.offset_mapping[1]     ,(len(SCREAMING_SNAKE_CASE__							) + 1, len(SCREAMING_SNAKE_CASE__							) + 1 + len(SCREAMING_SNAKE_CASE__							))     ,)

         SCREAMING_SNAKE_CASE:Tuple						      =					F''' {text}'''

         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					self.rust_tokenizer_class.from_pretrained(
             SCREAMING_SNAKE_CASE__     ,use_fast=SCREAMING_SNAKE_CASE__     ,)
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					tokenizer_r(SCREAMING_SNAKE_CASE__     ,return_offsets_mapping=SCREAMING_SNAKE_CASE__     ,add_special_tokens=SCREAMING_SNAKE_CASE__							)
         self.assertEqual(encoding.offset_mapping[0]     ,(1, 1 + len(SCREAMING_SNAKE_CASE__							))							)
         self.assertEqual(
             encoding.offset_mapping[1]     ,(1 + len(SCREAMING_SNAKE_CASE__							) + 1, 1 + len(SCREAMING_SNAKE_CASE__							) + 1 + len(SCREAMING_SNAKE_CASE__							))     ,)
   def     __UpperCamelCase						( self			:      Optional[Any]							):
     # Test related to the breaking change introduced in transformers v4.17.0
     # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
     with self.assertRaises(SCREAMING_SNAKE_CASE__							) as context:
       self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"							)

     self.assertTrue(
         context.exception.args[0].startswith(
             "The `backend_tokenizer` provided does not match the expected format."							)							)
   @require_ftfy
   def     __UpperCamelCase						( self			:      List[Any]							):
     super().test_tokenization_python_rust_equals()







   def     __UpperCamelCase						( self			:      Dict							):
     # CLIP always lower cases letters
     pass









'''simple docstring'''

from importlib import import_module

from .logging import get_logger


A_       						=							get_logger(__name__)





class 							_snake_case :
   def __init__( self			:      Dict     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Dict=None							):
     SCREAMING_SNAKE_CASE:Any						      =					attrs or []
     if module is not None:
       for key in module.__dict__:
         if key in attrs or not key.startswith("__"							):
           setattr(self     ,SCREAMING_SNAKE_CASE__     ,getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)							)
     SCREAMING_SNAKE_CASE:Any						      =					module._original_module if isinstance(SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj							) else module





class 							_snake_case :
   _A       :     Any       =					[]
   def __init__( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=None							):
     SCREAMING_SNAKE_CASE:Any						      =					obj
     SCREAMING_SNAKE_CASE:Any						      =					target
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					new
     SCREAMING_SNAKE_CASE:Any						      =					target.split("."							)[0]
     SCREAMING_SNAKE_CASE:str						      =					{}
     SCREAMING_SNAKE_CASE:List[Any]						      =					attrs or []
   def __enter__( self			:      Tuple							):
     *SCREAMING_SNAKE_CASE      ,    SCREAMING_SNAKE_CASE:int						      =					self.target.split("."							)

     # Patch modules:
     # it's used to patch attributes of submodules like "os.path.join";
     # in this case we need to patch "os" and "os.path"

     for i in range(len(SCREAMING_SNAKE_CASE__							)							):
       try:
         SCREAMING_SNAKE_CASE:int						      =					import_module(".".join(submodules[: i + 1]							)							)
       except ModuleNotFoundError:
         continue
       # We iterate over all the globals in self.obj in case we find "os" or "os.path"
       for attr in self.obj.__dir__():
         SCREAMING_SNAKE_CASE:str						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
         # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
         # This allows to patch renamed modules like "from os import path as ospath".
         if obj_attr is submodule or (
             (isinstance(SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj							) and obj_attr._original_module is submodule)
         ):
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					obj_attr
           # patch at top level
           setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj(SCREAMING_SNAKE_CASE__     ,attrs=self.attrs							)							)
           SCREAMING_SNAKE_CASE:int						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
           # construct lower levels patches
           for key in submodules[i + 1 :]:
             setattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj(getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)     ,attrs=self.attrs							)							)
             SCREAMING_SNAKE_CASE:Dict						      =					getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
           # finally set the target attribute
           setattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,self.new							)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

     if submodules:  # if it's an attribute of a submodule like "os.path.join"
       try:
         SCREAMING_SNAKE_CASE:Any						      =					getattr(import_module(".".join(SCREAMING_SNAKE_CASE__							)							)     ,SCREAMING_SNAKE_CASE__							)
       except (AttributeError, ModuleNotFoundError):
         return
       # We iterate over all the globals in self.obj in case we find "os.path.join"
       for attr in self.obj.__dir__():
         # We don't check for the name of the global, but rather if its value *is* "os.path.join".
         # This allows to patch renamed attributes like "from os.path import join as pjoin".
         if getattr(self.obj     ,SCREAMING_SNAKE_CASE__							) is attr_value:
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
           setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.new							)
     elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
       SCREAMING_SNAKE_CASE:int						      =					globals()["__builtins__"][target_attr]
       setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.new							)
     else:
       raise RuntimeError(F'''Tried to patch attribute {target_attr} instead of a submodule.'''							)
   def __exit__( self			:      List[Any]     ,*SCREAMING_SNAKE_CASE__			:      Union[str, Any]							):
     for attr in list(self.original							):
       setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.original.pop(SCREAMING_SNAKE_CASE__							)							)
   def     __UpperCamelCase						( self			:      Optional[int]							):
     self.__enter__()
     self._active_patches.append(self							)







   def     __UpperCamelCase						( self			:      Dict							):
     try:
       self._active_patches.remove(self							)
     except ValueError:
       # If the patch hasn't been started this will fail
       return None

     return self.__exit__()






