


"""simple docstring"""

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class    lowerCamelCase__						(		_a			,   unittest.TestCase    ):
      a :     List[str]											=		CLIPTokenizer
      a :     Union[str, Any]											=		CLIPTokenizerFast
      a :     str											=		True
      a :     List[str]											=		{}
      a :     Any											=		False
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       super().setUp()

       # fmt: off
       __lowercase									=	["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
       # fmt: on
       __lowercase									=	dict(zip(A_      ,       range(len(A_							)							)							)							)
       __lowercase									=	["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
       __lowercase									=	{"""unk_token""": """<unk>"""}

       __lowercase									=	os.path.join(self.tmpdirname      ,       VOCAB_FILES_NAMES["""vocab_file"""]							)
       __lowercase									=	os.path.join(self.tmpdirname      ,       VOCAB_FILES_NAMES["""merges_file"""]							)
       with open(self.vocab_file      ,       """w"""      ,       encoding="""utf-8"""							) as fp:
        fp.write(json.dumps(A_							) + """\n"""							)
       with open(self.merges_file      ,       """w"""      ,       encoding="""utf-8"""							) as fp:
        fp.write("""\n""".join(A_							)							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Any      ,       **A_	:	Union[str, Any]							):






       '''simple docstring'''


       kwargs.update(self.special_tokens_map							)
       return CLIPTokenizer.from_pretrained(self.tmpdirname      ,       **A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       **A_	:	Any							):






       '''simple docstring'''


       kwargs.update(self.special_tokens_map							)
       return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,       **A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	Union[str, Any]							):






       '''simple docstring'''


       __lowercase									=	"""lower newer"""
       __lowercase									=	"""lower newer"""
       return input_text, output_text
      def SCREAMING_SNAKE_CASE_	(		self	:	Union[str, Any]							):






       '''simple docstring'''


       __lowercase									=	CLIPTokenizer(self.vocab_file      ,       self.merges_file      ,       **self.special_tokens_map							)
       __lowercase									=	"""lower newer"""
       __lowercase									=	["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
       __lowercase									=	tokenizer.tokenize(A_							)
       self.assertListEqual(A_      ,       A_							)

       __lowercase									=	tokens + [tokenizer.unk_token]
       __lowercase									=	[1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(A_							)      ,       A_							)
      @require_ftfy
      def SCREAMING_SNAKE_CASE_	(		self	:	str							):






       '''simple docstring'''


       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         __lowercase									=	self.tokenizer_class.from_pretrained(A_      ,       **A_							)
         __lowercase									=	self.rust_tokenizer_class.from_pretrained(A_      ,       **A_							)

         __lowercase									=	"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
         __lowercase									=	tokenizer_s.tokenize(A_							)
         __lowercase									=	tokenizer_r.tokenize(A_							)

         self.assertListEqual(A_      ,       A_							)

         # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
         # with Tilde) encoded in 2 different ways
         __lowercase									=	"""xa\u0303y""" + """ """ + """x\xe3y"""
         __lowercase									=	tokenizer_s.tokenize(A_							)
         __lowercase									=	tokenizer_r.tokenize(A_							)

         self.assertListEqual(A_      ,       A_							)

         # Test that the tokenization is identical on unicode of space type
         __lowercase									=	[
             """\u0009""",  # (horizontal tab, '\t')
             """\u000B""",  # (vertical tab)
             """\u000C""",  # (form feed)
             """\u0020""",  # (space, ' ')
             """\u200E""",  # (left-to-right mark):w
             """\u200F""",  # (right-to-left mark)
         ]
         for unicode_seq in spaces_unicodes:
          __lowercase									=	tokenizer_s.tokenize(A_							)
          __lowercase									=	tokenizer_r.tokenize(A_							)

          self.assertListEqual(A_      ,       A_							)

         # Test that the tokenization is identical on unicode of line break type
         __lowercase									=	[
             """\u000A""",  # (line feed, '\n')
             """\r\n""",  # (carriage return and line feed, '\r\n')
             """\u000D""",  # (carriage return, '\r')
             """\r""",  # (carriage return, '\r')
             """\u000D""",  # (carriage return, '\r')
             """\u2028""",  # (line separator)
             """\u2029""",  # (paragraph separator)
             # "\u0085", # (next line)
         ]

         # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
         # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
         # space (and thus into an empty list).

         for unicode_seq in line_break_unicodes:
          __lowercase									=	tokenizer_s.tokenize(A_							)
          __lowercase									=	tokenizer_r.tokenize(A_							)

          self.assertListEqual(A_      ,       A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]							):






       '''simple docstring'''


       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         __lowercase									=	"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
         __lowercase									=	F'''{text_of_1_token} {text_of_1_token}'''

         __lowercase									=	self.rust_tokenizer_class.from_pretrained(
             A_      ,       use_fast=A_      ,       )
         __lowercase									=	tokenizer_r(A_      ,       return_offsets_mapping=A_      ,       add_special_tokens=A_							)
         self.assertEqual(encoding.offset_mapping[0]      ,       (0, len(A_							))							)
         self.assertEqual(
             encoding.offset_mapping[1]      ,       (len(A_							) + 1, len(A_							) + 1 + len(A_							))      ,       )

         __lowercase									=	F''' {text}'''

         __lowercase									=	self.rust_tokenizer_class.from_pretrained(
             A_      ,       use_fast=A_      ,       )
         __lowercase									=	tokenizer_r(A_      ,       return_offsets_mapping=A_      ,       add_special_tokens=A_							)
         self.assertEqual(encoding.offset_mapping[0]      ,       (1, 1 + len(A_							))							)
         self.assertEqual(
             encoding.offset_mapping[1]      ,       (1 + len(A_							) + 1, 1 + len(A_							) + 1 + len(A_							))      ,       )
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]							):






       '''simple docstring'''


       with self.assertRaises(A_							) as context:
        self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""							)

       self.assertTrue(
           context.exception.args[0].startswith(
               """The `backend_tokenizer` provided does not match the expected format."""							)							)
      @require_ftfy
      def SCREAMING_SNAKE_CASE_	(		self	:	Dict							):






       '''simple docstring'''


       super().test_tokenization_python_rust_equals()






      def SCREAMING_SNAKE_CASE_	(		self	:	Any							):






       '''simple docstring'''


       pass




"""simple docstring"""

from typing import List

from .keymap import KEYMAP, get_character


def lowerCAmelCase_      (				UpperCamelCase__	: str ):






 """simple docstring"""







 def decorator(UpperCamelCase__	: Tuple ):
  __lowercase									=	getattr(UpperCamelCase__				,   """handle_key"""				,   [] )
  handle += [key]
  setattr(UpperCamelCase__				,   """handle_key"""				,   UpperCamelCase__ )
  return func

 return decorator


def lowerCAmelCase_      (				*UpperCamelCase__	: List[str] ):






 """simple docstring"""







 def decorator(UpperCamelCase__	: Tuple ):
  __lowercase									=	getattr(UpperCamelCase__				,   """handle_key"""				,   [] )
  handle += keys
  setattr(UpperCamelCase__				,   """handle_key"""				,   UpperCamelCase__ )
  return func

 return decorator





class    lowerCamelCase__						(		_a    ):
      def __new__(		cls	:	str      ,       A_	:	Optional[Any]      ,       A_	:	Union[str, Any]      ,       A_	:	int							):






       '''simple docstring'''


       __lowercase									=	super().__new__(cls      ,       A_      ,       A_      ,       A_							)
       if not hasattr(A_      ,       """key_handler"""							):
        setattr(A_      ,       """key_handler"""      ,       {}							)
       setattr(A_      ,       """handle_input"""      ,       KeyHandler.handle_input							)

       for value in attrs.values():
        __lowercase									=	getattr(A_      ,       """handle_key"""      ,       []							)
        for key in handled_keys:
         __lowercase									=	value
       return new_cls






      @staticmethod
      def SCREAMING_SNAKE_CASE_	(		cls	:	Dict							):






       '''simple docstring'''


       __lowercase									=	get_character()
       if char != KEYMAP["undefined"]:
        __lowercase									=	ord(A_							)
       __lowercase									=	cls.key_handler.get(A_							)
       if handler:
        __lowercase									=	char
        return handler(cls							)
       else:
        return None






def lowerCAmelCase_      (				cls	: int ):






 """simple docstring"""







 return KeyHandler(cls.__name__				,   cls.__bases__				,   cls.__dict__.copy() )
