



import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 				_snake_case			(					lowercase__    ,	unittest.TestCase):
				UpperCamelCase__				:					Optional[Any]     		=CLIPTokenizer
				UpperCamelCase__				:					int     		=CLIPTokenizerFast
				UpperCamelCase__				:					Any     		=True
				UpperCamelCase__				:					Optional[int]     		={}
				UpperCamelCase__				:					Union[str, Any]     		=False




				def 							A__      (						self					:						List[str]						):
							super().setUp()

							# fmt: off
							lowercase__					  =				["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
							# fmt: on
							lowercase__					  =				dict(zip(__lowercase,					range(len(__lowercase						)						)						)						)
							lowercase__					  =				["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
							lowercase__					  =				{"unk_token": "<unk>"}

							lowercase__					  =				os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES["vocab_file"]						)
							lowercase__					  =				os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES["merges_file"]						)
							with open(self.vocab_file,					"w",					encoding="utf-8"						) as fp:
										fp.write(json.dumps(__lowercase						) + "\n"						)
							with open(self.merges_file,					"w",					encoding="utf-8"						) as fp:
										fp.write("\n".join(__lowercase						)						)




				def 							A__      (						self					:						Optional[Any],					**__lowercase					:						Any						):
							kwargs.update(self.special_tokens_map						)
							return CLIPTokenizer.from_pretrained(self.tmpdirname,					**__lowercase						)




				def 							A__      (						self					:						List[Any],					**__lowercase					:						List[str]						):
							kwargs.update(self.special_tokens_map						)
							return CLIPTokenizerFast.from_pretrained(self.tmpdirname,					**__lowercase						)




				def 							A__      (						self					:						int,					__lowercase					:						Any						):
							lowercase__					  =				"lower newer"
							lowercase__					  =				"lower newer"
							return input_text, output_text




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				CLIPTokenizer(self.vocab_file,					self.merges_file,					**self.special_tokens_map						)
							lowercase__					  =				"lower newer"
							lowercase__					  =				["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
							lowercase__					  =				tokenizer.tokenize(__lowercase						)
							self.assertListEqual(__lowercase,					__lowercase						)

							lowercase__					  =				tokens + [tokenizer.unk_token]
							lowercase__					  =				[10, 2, 16, 9, 3, 2, 16, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						),					__lowercase						)




				@require_ftfy
				def 							A__      (						self					:						Tuple						):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													lowercase__					  =				self.tokenizer_class.from_pretrained(__lowercase,					**__lowercase						)
													lowercase__					  =				self.rust_tokenizer_class.from_pretrained(__lowercase,					**__lowercase						)

													lowercase__					  =				"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
													lowercase__					  =				tokenizer_s.tokenize(__lowercase						)
													lowercase__					  =				tokenizer_r.tokenize(__lowercase						)

													self.assertListEqual(__lowercase,					__lowercase						)

													# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
													# with Tilde) encoded in 2 different ways
													lowercase__					  =				"xa\u0303y" + " " + "x\xe3y"
													lowercase__					  =				tokenizer_s.tokenize(__lowercase						)
													lowercase__					  =				tokenizer_r.tokenize(__lowercase						)

													self.assertListEqual(__lowercase,					__lowercase						)

													# Test that the tokenization is identical on unicode of space type
													lowercase__					  =				[
													    "\u0009",  # (horizontal tab, '\t')
													    "\u000B",  # (vertical tab)
													    "\u000C",  # (form feed)
													    "\u0020",  # (space, ' ')
													    "\u200E",  # (left-to-right mark):w
													    "\u200F",  # (right-to-left mark)
													]
													for unicode_seq in spaces_unicodes:
																lowercase__					  =				tokenizer_s.tokenize(__lowercase						)
																lowercase__					  =				tokenizer_r.tokenize(__lowercase						)

																self.assertListEqual(__lowercase,					__lowercase						)

													# Test that the tokenization is identical on unicode of line break type
													lowercase__					  =				[
													    "\u000A",  # (line feed, '\n')
													    "\r\n",  # (carriage return and line feed, '\r\n')
													    "\u000D",  # (carriage return, '\r')
													    "\r",  # (carriage return, '\r')
													    "\u000D",  # (carriage return, '\r')
													    "\u2028",  # (line separator)
													    "\u2029",  # (paragraph separator)
													    # "\u0085", # (next line)
													]

													# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
													# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
													# space (and thus into an empty list).

													for unicode_seq in line_break_unicodes:
																lowercase__					  =				tokenizer_s.tokenize(__lowercase						)
																lowercase__					  =				tokenizer_r.tokenize(__lowercase						)

																self.assertListEqual(__lowercase,					__lowercase						)




				def 							A__      (						self					:						Dict						):
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													lowercase__					  =				"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
													lowercase__					  =				F'''{text_of_1_token} {text_of_1_token}'''

													lowercase__					  =				self.rust_tokenizer_class.from_pretrained(
													    __lowercase,					use_fast=__lowercase,					)
													lowercase__					  =				tokenizer_r(__lowercase,					return_offsets_mapping=__lowercase,					add_special_tokens=__lowercase						)
													self.assertEqual(encoding.offset_mapping[0],					(0, len(__lowercase						))						)
													self.assertEqual(
													    encoding.offset_mapping[1],					(len(__lowercase						) + 1, len(__lowercase						) + 1 + len(__lowercase						)),					)

													lowercase__					  =				F''' {text}'''

													lowercase__					  =				self.rust_tokenizer_class.from_pretrained(
													    __lowercase,					use_fast=__lowercase,					)
													lowercase__					  =				tokenizer_r(__lowercase,					return_offsets_mapping=__lowercase,					add_special_tokens=__lowercase						)
													self.assertEqual(encoding.offset_mapping[0],					(1, 1 + len(__lowercase						))						)
													self.assertEqual(
													    encoding.offset_mapping[1],					(1 + len(__lowercase						) + 1, 1 + len(__lowercase						) + 1 + len(__lowercase						)),					)




				def 							A__      (						self					:						Tuple						):
							# Test related to the breaking change introduced in transformers v4.17.0
							# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
							with self.assertRaises(__lowercase						) as context:
										self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"						)

							self.assertTrue(
							    context.exception.args[0].startswith(
							        "The `backend_tokenizer` provided does not match the expected format."						)						)




				@require_ftfy
				def 							A__      (						self					:						Dict						):
							super().test_tokenization_python_rust_equals()




				def 							A__      (						self					:						Union[str, Any]						):
							# CLIP always lower cases letters
							pass









def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				0
			for ch in input_str:
						lowercase__					  =				ord(SCREAMING_SNAKE_CASE_  )
						lowercase__					  =				pow(2						,  SCREAMING_SNAKE_CASE_  )

						# If we already turned on bit for current character's unicode
						if bitmap >> ch_unicode & 1 == 1:
									return False
						bitmap |= ch_bit_index_on
			return True


if __name__ == "__main__":
							import doctest

							doctest.testmod()




