






'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							__magic_name__     (	a__,						unittest.TestCase				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      str        =  CLIPTokenizer
	SCREAMING_SNAKE_CASE_					:      Tuple        =  CLIPTokenizerFast
	SCREAMING_SNAKE_CASE_					:      List[str]        =  True
	SCREAMING_SNAKE_CASE_					:      int        =  {}
	SCREAMING_SNAKE_CASE_					:      int        =  False
	def        snake_case__		(							self       )						->  Any:
							super().setUp()

							# fmt: off
							lowercase_      :   Dict        =			["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
							# fmt: on
							lowercase_      :   Any        =			dict(zip(lowerCAmelCase__  ,      range(len(lowerCAmelCase__       )       )       )       )
							lowercase_      :   Optional[int]        =			["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
							lowercase_      :   List[str]        =			{"unk_token": "<unk>"}

							lowercase_      :   Dict        =			os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']       )
							lowercase_      :   List[str]        =			os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['merges_file']       )
							with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'       ) as fp:
													fp.write(json.dumps(lowerCAmelCase__       ) + '\n'       )
							with open(self.merges_file  ,      'w'  ,      encoding='utf-8'       ) as fp:
													fp.write('\n'.join(lowerCAmelCase__       )       )
	def        snake_case__		(							self  ,      **_lowercase       )						->  List[Any]:
							kwargs.update(self.special_tokens_map       )
							return CLIPTokenizer.from_pretrained(self.tmpdirname  ,      **lowerCAmelCase__       )
	def        snake_case__		(							self  ,      **_lowercase       )						->  int:
							kwargs.update(self.special_tokens_map       )
							return CLIPTokenizerFast.from_pretrained(self.tmpdirname  ,      **lowerCAmelCase__       )
	def        snake_case__		(							self  ,      _lowercase       )						->  Optional[int]:
							lowercase_      :   List[Any]        =			"lower newer"
							lowercase_      :   int        =			"lower newer"
							return input_text, output_text
	def        snake_case__		(							self       )						->  str:
							lowercase_      :   Optional[Any]        =			CLIPTokenizer(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map       )
							lowercase_      :   List[Any]        =			"lower newer"
							lowercase_      :   Optional[int]        =			["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
							lowercase_      :   List[str]        =			tokenizer.tokenize(lowerCAmelCase__       )
							self.assertListEqual(lowerCAmelCase__  ,      lowerCAmelCase__       )

							lowercase_      :   List[Any]        =			tokens + [tokenizer.unk_token]
							lowercase_      :   List[Any]        =			[10, 2, 16, 9, 3, 2, 16, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )  ,      lowerCAmelCase__       )
	@require_ftfy
	def        snake_case__		(							self       )						->  Dict:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
																			lowercase_      :   List[str]        =			self.tokenizer_class.from_pretrained(lowerCAmelCase__  ,      **lowerCAmelCase__       )
																			lowercase_      :   List[Any]        =			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__  ,      **lowerCAmelCase__       )

																			lowercase_      :   Any        =			"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
																			lowercase_      :   str        =			tokenizer_s.tokenize(lowerCAmelCase__       )
																			lowercase_      :   Union[str, Any]        =			tokenizer_r.tokenize(lowerCAmelCase__       )

																			self.assertListEqual(lowerCAmelCase__  ,      lowerCAmelCase__       )

																			# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																			# with Tilde) encoded in 2 different ways
																			lowercase_      :   List[Any]        =			"xa\u0303y" + " " + "x\xe3y"
																			lowercase_      :   str        =			tokenizer_s.tokenize(lowerCAmelCase__       )
																			lowercase_      :   Optional[int]        =			tokenizer_r.tokenize(lowerCAmelCase__       )

																			self.assertListEqual(lowerCAmelCase__  ,      lowerCAmelCase__       )

																			# Test that the tokenization is identical on unicode of space type
																			lowercase_      :   Any        =			[
																			    "\u0009",  # (horizontal tab, '\t')
																			    "\u000B",  # (vertical tab)
																			    "\u000C",  # (form feed)
																			    "\u0020",  # (space, ' ')
																			    "\u200E",  # (left-to-right mark):w
																			    "\u200F",  # (right-to-left mark)
																			]
																			for unicode_seq in spaces_unicodes:
																									lowercase_      :   Any        =			tokenizer_s.tokenize(lowerCAmelCase__       )
																									lowercase_      :   Dict        =			tokenizer_r.tokenize(lowerCAmelCase__       )

																									self.assertListEqual(lowerCAmelCase__  ,      lowerCAmelCase__       )

																			# Test that the tokenization is identical on unicode of line break type
																			lowercase_      :   Any        =			[
																			    "\u000A",  # (line feed, '\n')
																			    "\r\n",  # (carriage return and line feed, '\r\n')
																			    "\u000D",  # (carriage return, '\r')
																			    "\r",  # (carriage return, '\r')
																			    "\u000D",  # (carriage return, '\r')
																			    "\u2028",  # (line separator)
																			    "\u2029",  # (paragraph separator)
																			    # "\u0085", # (next line)
																			]

																			# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																			# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																			# space (and thus into an empty list).

																			for unicode_seq in line_break_unicodes:
																									lowercase_      :   List[str]        =			tokenizer_s.tokenize(lowerCAmelCase__       )
																									lowercase_      :   Tuple        =			tokenizer_r.tokenize(lowerCAmelCase__       )

																									self.assertListEqual(lowerCAmelCase__  ,      lowerCAmelCase__       )
	def        snake_case__		(							self       )						->  List[str]:
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
																			lowercase_      :   Tuple        =			"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																			lowercase_      :   Tuple        =			f"{text_of_1_token} {text_of_1_token}"

																			lowercase_      :   Union[str, Any]        =			self.rust_tokenizer_class.from_pretrained(
																			    lowerCAmelCase__  ,      use_fast=lowerCAmelCase__  ,      )
																			lowercase_      :   str        =			tokenizer_r(lowerCAmelCase__  ,      return_offsets_mapping=lowerCAmelCase__  ,      add_special_tokens=lowerCAmelCase__       )
																			self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(lowerCAmelCase__       ))       )
																			self.assertEqual(
																			    encoding.offset_mapping[1]  ,      (len(lowerCAmelCase__       ) + 1, len(lowerCAmelCase__       ) + 1 + len(lowerCAmelCase__       ))  ,      )

																			lowercase_      :   Optional[Any]        =			f" {text}"

																			lowercase_      :   Optional[int]        =			self.rust_tokenizer_class.from_pretrained(
																			    lowerCAmelCase__  ,      use_fast=lowerCAmelCase__  ,      )
																			lowercase_      :   Any        =			tokenizer_r(lowerCAmelCase__  ,      return_offsets_mapping=lowerCAmelCase__  ,      add_special_tokens=lowerCAmelCase__       )
																			self.assertEqual(encoding.offset_mapping[0]  ,      (1, 1 + len(lowerCAmelCase__       ))       )
																			self.assertEqual(
																			    encoding.offset_mapping[1]  ,      (1 + len(lowerCAmelCase__       ) + 1, 1 + len(lowerCAmelCase__       ) + 1 + len(lowerCAmelCase__       ))  ,      )
	def        snake_case__		(							self       )						->  Union[str, Any]:
							# Test related to the breaking change introduced in transformers v4.17.0
							# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
							with self.assertRaises(lowerCAmelCase__       ) as context:
													self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'       )

							self.assertTrue(
							    context.exception.args[0].startswith(
							        'The `backend_tokenizer` provided does not match the expected format.'       )       )
	@require_ftfy
	def        snake_case__		(							self       )						->  Union[str, Any]:
							super().test_tokenization_python_rust_equals()







	def        snake_case__		(							self       )						->  List[str]:
							# CLIP always lower cases letters
							pass








'''simple docstring'''

from pathlib import PurePosixPath
from typing import Optional

import fsspec
from fsspec import AbstractFileSystem
from huggingface_hub.hf_api import DatasetInfo

from ..utils.file_utils import get_authentication_headers_for_url
from ..utils.hub import hf_hub_url






class 							__magic_name__     (	UpperCAmelCase_				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  ''
	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  'hf-legacy'  # "hf://"" is reserved for hffs
	def __init__(							self  ,      _lowercase = None  ,      _lowercase = None  ,      **_lowercase  ,      )						->  Optional[Any]:
							super().__init__(self  ,      **_lowercase       )
							lowercase_      :   int        =			repo_info
							lowercase_      :   List[Any]        =			token
							lowercase_      :   Union[str, Any]        =			None
	def        lowerCamelCase__		(							self       )						->  Optional[Any]:
							if self.dir_cache is None:
													lowercase_      :   Optional[Any]        =			{}
													for hf_file in self.repo_info.siblings:
																			# TODO(QL): add sizes
																			lowercase_      :   str        =			{
																			    'name': hf_file.rfilename,
																			    'size': None,
																			    'type': 'file',
																			}
																			self.dir_cache.update(
																			    {
																			        str(_lowercase       ): {'name': str(_lowercase       ), 'size': None, 'type': 'directory'}
																			        for d in list(PurePosixPath(hf_file.rfilename       ).parents       )[:-1]
																			    }       )
	def        lowerCamelCase__		(							self  ,      _lowercase  ,      _lowercase = "rb"  ,      **_lowercase  ,      )						->  Dict:
							if not isinstance(self.repo_info  ,      _lowercase       ):
													raise NotImplementedError(f"Open is only implemented for dataset repositories, but got {self.repo_info}"       )
							lowercase_      :   Optional[int]        =			hf_hub_url(self.repo_info.id  ,      _lowercase  ,      revision=self.repo_info.sha       )
							return fsspec.open(
							    _lowercase  ,      mode=_lowercase  ,      headers=get_authentication_headers_for_url(_lowercase  ,      use_auth_token=self.token       )  ,      client_kwargs={'trust_env': True}  ,      ).open()
	def        lowerCamelCase__		(							self  ,      _lowercase  ,      **_lowercase       )						->  Tuple:
							self._get_dirs()
							lowercase_      :   str        =			self._strip_protocol(_lowercase       )
							if path in self.dir_cache:
													return self.dir_cache[path]
							else:
													raise FileNotFoundError(_lowercase       )







	def        lowerCamelCase__		(							self  ,      _lowercase  ,      _lowercase=False  ,      **_lowercase       )						->  List[str]:
							self._get_dirs()
							lowercase_      :   List[str]        =			PurePosixPath(path.strip('/'       )       )
							lowercase_      :   List[str]        =			{}
							for p, f in self.dir_cache.items():
													lowercase_      :   Tuple        =			PurePosixPath(p.strip('/'       )       )
													lowercase_      :   Optional[int]        =			p.parent
													if root == path:
																			lowercase_      :   List[str]        =			f
							lowercase_      :   List[str]        =			list(paths.values()       )
							if detail:
													return out
							else:
													return sorted(f['name'] for f in out       )
