

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 					_a		(__magic_name__	,     unittest.TestCase       ):


					'''simple docstring'''






					UpperCAmelCase__:     Tuple													= BertTokenizer
					UpperCAmelCase__:     Tuple													= BertTokenizerFast
					UpperCAmelCase__:     Dict													= True
					UpperCAmelCase__:     List[Any]													= True
					UpperCAmelCase__:     List[Any]													= filter_non_english






					def 						__A    (				self    ):
						super().setUp()

						A__							:					Dict       =	[
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """[PAD]""",
						    """[MASK]""",
						    """want""",
						    """##want""",
						    """##ed""",
						    """wa""",
						    """un""",
						    """runn""",
						    """##ing""",
						    """,""",
						    """low""",
						    """lowest""",
						]
						A__							:					Optional[int]       =	os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES["""vocab_file"""]    )
						with open(self.vocab_file					, """w"""					, encoding="""utf-8"""    ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )






					def 						__A    (				self					, A__    ):
						A__							:					Dict       =	"""UNwant\u00E9d,running"""
						A__							:					Union[str, Any]       =	"""unwanted, running"""
						return input_text, output_text






					def 						__A    (				self    ):
						A__							:					int       =	self.tokenizer_class(self.vocab_file    )

						A__							:					List[str]       =	tokenizer.tokenize("""UNwant\u00E9d,running"""    )
						self.assertListEqual(A__					, ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__    )					, [9, 6, 7, 12, 10, 11]    )






					def 						__A    (				self    ):







						if not self.test_rust_tokenizer:
							return

						A__							:					Tuple       =	self.get_tokenizer()
						A__							:					List[str]       =	self.get_rust_tokenizer()

						A__							:					List[str]       =	"""UNwant\u00E9d,running"""

						A__							:					List[str]       =	tokenizer.tokenize(A__    )
						A__							:					List[str]       =	rust_tokenizer.tokenize(A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					Tuple       =	tokenizer.encode(A__					, add_special_tokens=A__    )
						A__							:					Dict       =	rust_tokenizer.encode(A__					, add_special_tokens=A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					int       =	self.get_rust_tokenizer()
						A__							:					Union[str, Any]       =	tokenizer.encode(A__    )
						A__							:					int       =	rust_tokenizer.encode(A__    )
						self.assertListEqual(A__					, A__    )

						# With lower casing
						A__							:					Dict       =	self.get_tokenizer(do_lower_case=A__    )
						A__							:					Dict       =	self.get_rust_tokenizer(do_lower_case=A__    )

						A__							:					Union[str, Any]       =	"""UNwant\u00E9d,running"""

						A__							:					List[Any]       =	tokenizer.tokenize(A__    )
						A__							:					Union[str, Any]       =	rust_tokenizer.tokenize(A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					Optional[Any]       =	tokenizer.encode(A__					, add_special_tokens=A__    )
						A__							:					Optional[Any]       =	rust_tokenizer.encode(A__					, add_special_tokens=A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					int       =	self.get_rust_tokenizer()
						A__							:					Union[str, Any]       =	tokenizer.encode(A__    )
						A__							:					List[Any]       =	rust_tokenizer.encode(A__    )
						self.assertListEqual(A__					, A__    )






					def 						__A    (				self    ):
						A__							:					Optional[int]       =	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )					, ["""ah""", """\u535A""", """\u63A8""", """zz"""]    )






					def 						__A    (				self    ):
						A__							:					Dict       =	BasicTokenizer(do_lower_case=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )					, ["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""hello"""]    )






					def 						__A    (				self    ):
						A__							:					Any       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""h\u00E9llo"""]    )






					def 						__A    (				self    ):
						A__							:					str       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""hello"""]    )






					def 						__A    (				self    ):
						A__							:					str       =	BasicTokenizer(do_lower_case=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""hello"""]    )






					def 						__A    (				self    ):
						A__							:					int       =	BasicTokenizer(do_lower_case=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )					, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )






					def 						__A    (				self    ):
						A__							:					Any       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )






					def 						__A    (				self    ):
						A__							:					List[str]       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )






					def 						__A    (				self    ):
						A__							:					Any       =	BasicTokenizer(do_lower_case=A__					, never_split=["""[UNK]"""]    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )					, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )






					def 						__A    (				self    ):
						A__							:					str       =	BasicTokenizer()
						A__							:					Dict       =	"""a\n\'ll !!to?\'d of, can\'t."""
						A__							:					Any       =	["""a""", """\'""", """ll""", """!""", """!""", """to""", """?""", """\'""", """d""", """of""", """,""", """can""", """\'""", """t""", """."""]
						self.assertListEqual(tokenizer.tokenize(A__    )					, A__    )






					def 						__A    (				self    ):
						A__							:					Optional[Any]       =	["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						A__							:					Optional[Any]       =	{}
						for i, token in enumerate(A__    ):
							A__							:					Any       =	i
						A__							:					int       =	WordpieceTokenizer(vocab=A__					, unk_token="""[UNK]"""    )

						self.assertListEqual(tokenizer.tokenize(""""""    )					, []    )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )					, ["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )					, ["""[UNK]""", """runn""", """##ing"""]    )






					def 						__A    (				self    ):
						self.assertTrue(_is_whitespace(""" """    )    )
						self.assertTrue(_is_whitespace("""\t"""    )    )
						self.assertTrue(_is_whitespace("""\r"""    )    )
						self.assertTrue(_is_whitespace("""\n"""    )    )
						self.assertTrue(_is_whitespace("""\u00A0"""    )    )

						self.assertFalse(_is_whitespace("""A"""    )    )
						self.assertFalse(_is_whitespace("""-"""    )    )






					def 						__A    (				self    ):
						self.assertTrue(_is_control("""\u0005"""    )    )

						self.assertFalse(_is_control("""A"""    )    )
						self.assertFalse(_is_control(""" """    )    )
						self.assertFalse(_is_control("""\t"""    )    )
						self.assertFalse(_is_control("""\r"""    )    )






					def 						__A    (				self    ):
						self.assertTrue(_is_punctuation("""-"""    )    )
						self.assertTrue(_is_punctuation("""$"""    )    )
						self.assertTrue(_is_punctuation("""`"""    )    )
						self.assertTrue(_is_punctuation("""."""    )    )

						self.assertFalse(_is_punctuation("""A"""    )    )
						self.assertFalse(_is_punctuation(""" """    )    )






					def 						__A    (				self    ):
						A__							:					List[Any]       =	self.get_tokenizer()
						A__							:					List[str]       =	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__    ) for t in ["""Test""", """\xad""", """test"""]]					, [["""[UNK]"""], [], ["""[UNK]"""]]    )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(A__    ) for t in ["""Test""", """\xad""", """test"""]]					, [["""[UNK]"""], [], ["""[UNK]"""]]    )






					@slow
					def 						__A    (				self    ):
						A__							:					Optional[Any]       =	self.tokenizer_class.from_pretrained("""bert-base-uncased"""    )

						A__							:					Union[str, Any]       =	tokenizer.encode("""sequence builders"""					, add_special_tokens=A__    )
						A__							:					Any       =	tokenizer.encode("""multi-sequence build"""					, add_special_tokens=A__    )

						A__							:					int       =	tokenizer.build_inputs_with_special_tokens(A__    )
						A__							:					List[Any]       =	tokenizer.build_inputs_with_special_tokens(A__					, A__    )

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]






					def 						__A    (				self    ):







						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
								A__							:					Any       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					int       =	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
								A__							:					Tuple       =	tokenizer_r.encode_plus(
								    A__					, return_attention_mask=A__					, return_token_type_ids=A__					, return_offsets_mapping=A__					, add_special_tokens=A__					, )

								A__							:					Any       =	tokenizer_r.do_lower_case if hasattr(A__					, """do_lower_case"""    ) else False
								A__							:					Optional[int]       =	(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """A"""),
								        ((1, 2), ""","""),
								        ((3, 5), """na"""),
								        ((5, 6), """##ï"""),
								        ((6, 8), """##ve"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """Allen"""),
								        ((21, 23), """##NL"""),
								        ((23, 24), """##P"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """a"""),
								        ((1, 2), ""","""),
								        ((3, 8), """naive"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """allen"""),
								        ((21, 23), """##nl"""),
								        ((23, 24), """##p"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]					, tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
								self.assertEqual([e[0] for e in expected_results]					, tokens["""offset_mapping"""]    )






					def 						__A    (				self    ):
						A__							:					List[Any]       =	["""的""", """人""", """有"""]
						A__							:					Optional[Any]       =	"""""".join(A__    )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
								A__							:					Dict       =	True
								A__							:					str       =	self.tokenizer_class.from_pretrained(A__					, **A__    )
								A__							:					List[str]       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					List[str]       =	tokenizer_p.encode(A__					, add_special_tokens=A__    )
								A__							:					Optional[Any]       =	tokenizer_r.encode(A__					, add_special_tokens=A__    )

								A__							:					Optional[Any]       =	tokenizer_r.convert_ids_to_tokens(A__    )
								A__							:					Tuple       =	tokenizer_p.convert_ids_to_tokens(A__    )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(A__					, A__    )
								self.assertListEqual(A__					, A__    )

								A__							:					Union[str, Any]       =	False
								A__							:					Any       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )
								A__							:					Union[str, Any]       =	self.tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					List[Any]       =	tokenizer_r.encode(A__					, add_special_tokens=A__    )
								A__							:					Optional[int]       =	tokenizer_p.encode(A__					, add_special_tokens=A__    )

								A__							:					Union[str, Any]       =	tokenizer_r.convert_ids_to_tokens(A__    )
								A__							:					Optional[Any]       =	tokenizer_p.convert_ids_to_tokens(A__    )

								# it is expected that only the first Chinese character is not preceded by "##".
								A__							:					str       =	[
								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__    )
								]
								self.assertListEqual(A__					, A__    )
								self.assertListEqual(A__					, A__    )






import inspect
import warnings
from typing import Any, Dict, Optional, Union

from packaging import version

def 		UpperCamelCase      (*lowercase_:				Optional[int]					,   lowercase_:				Optional[Union[Dict, Any]] = None					,   lowercase_:				Dict=True					,   lowercase_:				Tuple=2      )  ->   Dict:
	from .. import __version__

	A__							:					Dict       =	take_from
	A__							:					str       =	()
	if not isinstance(args[0]					,   lowercase_      ):
		A__							:					int       =	(args,)

	for attribute, version_name, message in args:
		if version.parse(version.parse(lowercase_      ).base_version      ) >= version.parse(lowercase_      ):
			raise ValueError(
			    f"""The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers'"""
			    f""" version {__version__} is >= {version_name}"""      )

		A__							:					Any       =	None
		if isinstance(lowercase_					,   lowercase_      ) and attribute in deprecated_kwargs:
			values += (deprecated_kwargs.pop(lowercase_      ),)
			A__							:					List[str]       =	f"""The `{attribute}` argument is deprecated and will be removed in version {version_name}."""
		elif hasattr(lowercase_					,   lowercase_      ):
			values += (getattr(lowercase_					,   lowercase_      ),)
			A__							:					Optional[Any]       =	f"""The `{attribute}` attribute is deprecated and will be removed in version {version_name}."""
		elif deprecated_kwargs is None:
			A__							:					int       =	f"""`{attribute}` is deprecated and will be removed in version {version_name}."""

		if warning is not None:
			A__							:					int       =	warning + """ """ if standard_warn else """"""
			warnings.warn(warning + message					,   lowercase_					,   stacklevel=lowercase_      )

	if isinstance(lowercase_					,   lowercase_      ) and len(lowercase_      ) > 0:
		A__							:					Union[str, Any]       =	inspect.getouterframes(inspect.currentframe()      )[1]
		A__							:					Optional[Any]       =	call_frame.filename
		A__							:					Optional[int]       =	call_frame.lineno
		A__							:					Any       =	call_frame.function
		A__     ,     A__							:					List[str]       =	next(iter(deprecated_kwargs.items()      )      )
		raise TypeError(f"""{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`"""      )

	if len(lowercase_      ) == 0:
		return
	elif len(lowercase_      ) == 1:
		return values[0]
	return values



