

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				a  (	UpperCAmelCase							, unittest.TestCase				):
						_lowercase    						=			BertTokenizer
						_lowercase    						=			BertTokenizerFast
						_lowercase    						=			True
						_lowercase    						=			True
						_lowercase    						=			filter_non_english


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										super().setUp()

										_UpperCAmelCase       :		List[str] 		=    [
										    "[UNK]",
										    "[CLS]",
										    "[SEP]",
										    "[PAD]",
										    "[MASK]",
										    "want",
										    "##want",
										    "##ed",
										    "wa",
										    "un",
										    "runn",
										    "##ing",
										    ",",
										    "low",
										    "lowest",
										]
										_UpperCAmelCase       :		List[Any] 		=    os.path.join(self.tmpdirname						,					VOCAB_FILES_NAMES["vocab_file"]	)
										with open(self.vocab_file						,					"w"						,					encoding="utf-8"	) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)


						def        _UpperCAmelCase				(       self						,					A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[Any] 		=    "UNwant\u00E9d,running"
										_UpperCAmelCase       :		str 		=    "unwanted, running"
										return input_text, output_text


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Dict 		=    self.tokenizer_class(self.vocab_file	)

										_UpperCAmelCase       :		Tuple 		=    tokenizer.tokenize("UNwant\u00E9d,running"	)
										self.assertListEqual(A_						,					["un", "##want", "##ed", ",", "runn", "##ing"]	)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(A_	)						,					[9, 6, 7, 12, 10, 11]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										if not self.test_rust_tokenizer:
														return

										_UpperCAmelCase       :		Optional[int] 		=    self.get_tokenizer()
										_UpperCAmelCase       :		str 		=    self.get_rust_tokenizer()

										_UpperCAmelCase       :		int 		=    "UNwant\u00E9d,running"

										_UpperCAmelCase       :		Union[str, Any] 		=    tokenizer.tokenize(A_	)
										_UpperCAmelCase       :		Optional[int] 		=    rust_tokenizer.tokenize(A_	)
										self.assertListEqual(A_						,					A_	)

										_UpperCAmelCase       :		Tuple 		=    tokenizer.encode(A_						,					add_special_tokens=A_	)
										_UpperCAmelCase       :		Dict 		=    rust_tokenizer.encode(A_						,					add_special_tokens=A_	)
										self.assertListEqual(A_						,					A_	)

										_UpperCAmelCase       :		List[str] 		=    self.get_rust_tokenizer()
										_UpperCAmelCase       :		str 		=    tokenizer.encode(A_	)
										_UpperCAmelCase       :		List[str] 		=    rust_tokenizer.encode(A_	)
										self.assertListEqual(A_						,					A_	)

										# With lower casing
										_UpperCAmelCase       :		Any 		=    self.get_tokenizer(do_lower_case=A_	)
										_UpperCAmelCase       :		int 		=    self.get_rust_tokenizer(do_lower_case=A_	)

										_UpperCAmelCase       :		List[Any] 		=    "UNwant\u00E9d,running"

										_UpperCAmelCase       :		int 		=    tokenizer.tokenize(A_	)
										_UpperCAmelCase       :		List[str] 		=    rust_tokenizer.tokenize(A_	)
										self.assertListEqual(A_						,					A_	)

										_UpperCAmelCase       :		List[str] 		=    tokenizer.encode(A_						,					add_special_tokens=A_	)
										_UpperCAmelCase       :		Tuple 		=    rust_tokenizer.encode(A_						,					add_special_tokens=A_	)
										self.assertListEqual(A_						,					A_	)

										_UpperCAmelCase       :		str 		=    self.get_rust_tokenizer()
										_UpperCAmelCase       :		Tuple 		=    tokenizer.encode(A_	)
										_UpperCAmelCase       :		List[Any] 		=    rust_tokenizer.encode(A_	)
										self.assertListEqual(A_						,					A_	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)						,					["ah", "\u535A", "\u63A8", "zz"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    BasicTokenizer(do_lower_case=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)						,					["hello", "!", "how", "are", "you", "?"]	)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						,					["hello"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    BasicTokenizer(do_lower_case=A_						,					strip_accents=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						,					["hällo", "!", "how", "are", "you", "?"]	)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						,					["h\u00E9llo"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    BasicTokenizer(do_lower_case=A_						,					strip_accents=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						,					["hallo", "!", "how", "are", "you", "?"]	)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						,					["hello"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		str 		=    BasicTokenizer(do_lower_case=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						,					["hallo", "!", "how", "are", "you", "?"]	)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						,					["hello"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    BasicTokenizer(do_lower_case=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)						,					["HeLLo", "!", "how", "Are", "yoU", "?"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[int] 		=    BasicTokenizer(do_lower_case=A_						,					strip_accents=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						,					["HäLLo", "!", "how", "Are", "yoU", "?"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    BasicTokenizer(do_lower_case=A_						,					strip_accents=A_	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						,					["HaLLo", "!", "how", "Are", "yoU", "?"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    BasicTokenizer(do_lower_case=A_						,					never_split=["[UNK]"]	)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)						,					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    BasicTokenizer()
										_UpperCAmelCase       :		Any 		=    "a\n'll !!to?'d of, can't."
										_UpperCAmelCase       :		Union[str, Any] 		=    ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
										self.assertListEqual(tokenizer.tokenize(A_	)						,					A_	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

										_UpperCAmelCase       :		int 		=    {}
										for i, token in enumerate(A_	):
														_UpperCAmelCase       :		Union[str, Any] 		=    i
										_UpperCAmelCase       :		Dict 		=    WordpieceTokenizer(vocab=A_						,					unk_token="[UNK]"	)

										self.assertListEqual(tokenizer.tokenize(""	)						,					[]	)

										self.assertListEqual(tokenizer.tokenize("unwanted running"	)						,					["un", "##want", "##ed", "runn", "##ing"]	)

										self.assertListEqual(tokenizer.tokenize("unwantedX running"	)						,					["[UNK]", "runn", "##ing"]	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										self.assertTrue(_is_whitespace(" "	)	)
										self.assertTrue(_is_whitespace("\t"	)	)
										self.assertTrue(_is_whitespace("\r"	)	)
										self.assertTrue(_is_whitespace("\n"	)	)
										self.assertTrue(_is_whitespace("\u00A0"	)	)

										self.assertFalse(_is_whitespace("A"	)	)
										self.assertFalse(_is_whitespace("-"	)	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										self.assertTrue(_is_control("\u0005"	)	)

										self.assertFalse(_is_control("A"	)	)
										self.assertFalse(_is_control(" "	)	)
										self.assertFalse(_is_control("\t"	)	)
										self.assertFalse(_is_control("\r"	)	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										self.assertTrue(_is_punctuation("-"	)	)
										self.assertTrue(_is_punctuation("$"	)	)
										self.assertTrue(_is_punctuation("`"	)	)
										self.assertTrue(_is_punctuation("."	)	)

										self.assertFalse(_is_punctuation("A"	)	)
										self.assertFalse(_is_punctuation(" "	)	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    self.get_tokenizer()
										_UpperCAmelCase       :		Union[str, Any] 		=    self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(A_	) for t in ["Test", "\xad", "test"]]						,					[["[UNK]"], [], ["[UNK]"]]	)

										self.assertListEqual(
										    [rust_tokenizer.tokenize(A_	) for t in ["Test", "\xad", "test"]]						,					[["[UNK]"], [], ["[UNK]"]]	)


						@slow
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    self.tokenizer_class.from_pretrained("bert-base-uncased"	)

										_UpperCAmelCase       :		Optional[int] 		=    tokenizer.encode("sequence builders"						,					add_special_tokens=A_	)
										_UpperCAmelCase       :		Dict 		=    tokenizer.encode("multi-sequence build"						,					add_special_tokens=A_	)

										_UpperCAmelCase       :		Optional[int] 		=    tokenizer.build_inputs_with_special_tokens(A_	)
										_UpperCAmelCase       :		Optional[int] 		=    tokenizer.build_inputs_with_special_tokens(A_						,					A_	)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
																		_UpperCAmelCase       :		Optional[Any] 		=    self.rust_tokenizer_class.from_pretrained(A_						,					**A_	)

																		_UpperCAmelCase       :		Dict 		=    f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																		_UpperCAmelCase       :		Tuple 		=    tokenizer_r.encode_plus(
																		    A_						,					return_attention_mask=A_						,					return_token_type_ids=A_						,					return_offsets_mapping=A_						,					add_special_tokens=A_						,					)

																		_UpperCAmelCase       :		Optional[int] 		=    tokenizer_r.do_lower_case if hasattr(A_						,					"do_lower_case"	) else False
																		_UpperCAmelCase       :		List[Any] 		=    (
																		    [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), "A"),
																		        ((1, 2), ","),
																		        ((3, 5), "na"),
																		        ((5, 6), "##ï"),
																		        ((6, 8), "##ve"),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), "Allen"),
																		        ((21, 23), "##NL"),
																		        ((23, 24), "##P"),
																		        ((25, 33), "sentence"),
																		        ((33, 34), "."),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		    if not do_lower_case
																		    else [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), "a"),
																		        ((1, 2), ","),
																		        ((3, 8), "naive"),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), "allen"),
																		        ((21, 23), "##nl"),
																		        ((23, 24), "##p"),
																		        ((25, 33), "sentence"),
																		        ((33, 34), "."),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		)

																		self.assertEqual(
																		    [e[1] for e in expected_results]						,					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
																		self.assertEqual([e[0] for e in expected_results]						,					tokens["offset_mapping"]	)





						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    ["的", "人", "有"]
										_UpperCAmelCase       :		Union[str, Any] 		=    "".join(A_	)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
																		_UpperCAmelCase       :		Any 		=    True
																		_UpperCAmelCase       :		List[Any] 		=    self.tokenizer_class.from_pretrained(A_						,					**A_	)
																		_UpperCAmelCase       :		Dict 		=    self.rust_tokenizer_class.from_pretrained(A_						,					**A_	)

																		_UpperCAmelCase       :		List[str] 		=    tokenizer_p.encode(A_						,					add_special_tokens=A_	)
																		_UpperCAmelCase       :		Optional[int] 		=    tokenizer_r.encode(A_						,					add_special_tokens=A_	)

																		_UpperCAmelCase       :		Union[str, Any] 		=    tokenizer_r.convert_ids_to_tokens(A_	)
																		_UpperCAmelCase       :		Union[str, Any] 		=    tokenizer_p.convert_ids_to_tokens(A_	)

																		# it is expected that each Chinese character is not preceded by "##"
																		self.assertListEqual(A_						,					A_	)
																		self.assertListEqual(A_						,					A_	)

																		_UpperCAmelCase       :		str 		=    False
																		_UpperCAmelCase       :		Optional[Any] 		=    self.rust_tokenizer_class.from_pretrained(A_						,					**A_	)
																		_UpperCAmelCase       :		List[Any] 		=    self.tokenizer_class.from_pretrained(A_						,					**A_	)

																		_UpperCAmelCase       :		Any 		=    tokenizer_r.encode(A_						,					add_special_tokens=A_	)
																		_UpperCAmelCase       :		Tuple 		=    tokenizer_p.encode(A_						,					add_special_tokens=A_	)

																		_UpperCAmelCase       :		int 		=    tokenizer_r.convert_ids_to_tokens(A_	)
																		_UpperCAmelCase       :		Dict 		=    tokenizer_p.convert_ids_to_tokens(A_	)

																		# it is expected that only the first Chinese character is not preceded by "##".
																		_UpperCAmelCase       :		Tuple 		=    [
																		    f'##{token}' if idx != 0 else token for idx, token in enumerate(A_	)
																		]
																		self.assertListEqual(A_						,					A_	)
																		self.assertListEqual(A_						,					A_	)





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
					import PIL


SCREAMING_SNAKE_CASE_        =      logging.get_logger(__name__)

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Dict  ,      lowerCAmelCase:  Union[str, Any] )   ->       Union[str, Any]:
				_UpperCAmelCase       :		Dict 		=    b.T
				_UpperCAmelCase       :		Dict 		=    np.sum(np.square(lowerCAmelCase )  ,      axis=1 )
				_UpperCAmelCase       :		Optional[Any] 		=    np.sum(np.square(lowerCAmelCase )  ,      axis=0 )
				_UpperCAmelCase       :		str 		=    np.matmul(lowerCAmelCase  ,      lowerCAmelCase )
				_UpperCAmelCase       :		Any 		=    aa[:, None] - 2 * ab + ba[None, :]
				return d

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Union[str, Any]  ,      lowerCAmelCase:  Dict )   ->       int:
				_UpperCAmelCase       :		Any 		=    x.reshape(-1  ,      3 )
				_UpperCAmelCase       :		List[str] 		=    squared_euclidean_distance(lowerCAmelCase  ,      lowerCAmelCase )
				return np.argmin(lowerCAmelCase  ,      axis=1 )





class 				a  (	UpperCAmelCase				):
						_lowercase    						=			["pixel_values"]


						def __init__(       self						,					A_ = None						,					A_ = True						,					A_ = None						,					A_ = PILImageResampling.BILINEAR						,					A_ = True						,					A_ = True						,					**A_						,					):







										'''simple docstring'''


										super().__init__(**A_	)
										_UpperCAmelCase       :		Optional[Any] 		=    size if size is not None else {"height": 256, "width": 256}
										_UpperCAmelCase       :		Optional[int] 		=    get_size_dict(A_	)
										_UpperCAmelCase       :		Union[str, Any] 		=    np.array(A_	) if clusters is not None else None
										_UpperCAmelCase       :		int 		=    do_resize
										_UpperCAmelCase       :		Union[str, Any] 		=    size
										_UpperCAmelCase       :		Optional[Any] 		=    resample
										_UpperCAmelCase       :		str 		=    do_normalize
										_UpperCAmelCase       :		List[str] 		=    do_color_quantize


						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					A_ = PILImageResampling.BILINEAR						,					A_ = None						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    get_size_dict(A_	)
										if "height" not in size or "width" not in size:
														raise ValueError(f'Size dictionary must contain both height and width keys. Got {size.keys()}'	)
										return resize(
										    A_						,					size=(size["height"], size["width"])						,					resample=A_						,					data_format=A_						,					**A_	)


						def        _UpperCAmelCase				(       self						,					A_						,					A_ = None						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Dict 		=    rescale(image=A_						,					scale=1 / 1_27.5						,					data_format=A_	)
										_UpperCAmelCase       :		List[Any] 		=    image - 1
										return image





						def        _UpperCAmelCase				(       self						,					A_						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = None						,					A_ = ChannelDimension.FIRST						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[int] 		=    do_resize if do_resize is not None else self.do_resize
										_UpperCAmelCase       :		Any 		=    size if size is not None else self.size
										_UpperCAmelCase       :		Dict 		=    get_size_dict(A_	)
										_UpperCAmelCase       :		List[Any] 		=    resample if resample is not None else self.resample
										_UpperCAmelCase       :		Any 		=    do_normalize if do_normalize is not None else self.do_normalize
										_UpperCAmelCase       :		Optional[Any] 		=    do_color_quantize if do_color_quantize is not None else self.do_color_quantize
										_UpperCAmelCase       :		Any 		=    clusters if clusters is not None else self.clusters
										_UpperCAmelCase       :		Optional[int] 		=    np.array(A_	)

										_UpperCAmelCase       :		List[str] 		=    make_list_of_images(A_	)

										if not valid_images(A_	):
														raise ValueError(
														    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
														    "torch.Tensor, tf.Tensor or jax.ndarray."	)

										if do_resize and size is None or resample is None:
														raise ValueError("Size and resample must be specified if do_resize is True."	)

										if do_color_quantize and clusters is None:
														raise ValueError("Clusters must be specified if do_color_quantize is True."	)

										# All transformations expect numpy arrays.
										_UpperCAmelCase       :		List[str] 		=    [to_numpy_array(A_	) for image in images]

										if do_resize:
														_UpperCAmelCase       :		int 		=    [self.resize(image=A_						,					size=A_						,					resample=A_	) for image in images]

										if do_normalize:
														_UpperCAmelCase       :		List[str] 		=    [self.normalize(image=A_	) for image in images]

										if do_color_quantize:
														_UpperCAmelCase       :		Tuple 		=    [to_channel_dimension_format(A_						,					ChannelDimension.LAST	) for image in images]
														# color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
														_UpperCAmelCase       :		List[str] 		=    np.array(A_	)
														_UpperCAmelCase       :		List[Any] 		=    color_quantize(A_						,					A_	).reshape(images.shape[:-1]	)

														# flatten to (batch_size, height*width)
														_UpperCAmelCase       :		Any 		=    images.shape[0]
														_UpperCAmelCase       :		List[Any] 		=    images.reshape(A_						,					-1	)

														# We need to convert back to a list of images to keep consistent behaviour across processors.
														_UpperCAmelCase       :		Union[str, Any] 		=    list(A_	)
										else:
														_UpperCAmelCase       :		Optional[Any] 		=    [to_channel_dimension_format(A_						,					A_	) for image in images]

										_UpperCAmelCase       :		List[Any] 		=    {"input_ids": images}
										return BatchFeature(data=A_						,					tensor_type=A_	)


