






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class     UpperCamelCase(  _a     ,							unittest.TestCase   ):
					snake_case_		:				Optional[Any]										=	RobertaTokenizer
					snake_case_		:				Union[str, Any]										=	RobertaTokenizerFast
					snake_case_		:				Union[str, Any]										=	True
					snake_case_		:				Tuple										=	{"cls_token": "<s>"}


					def 							SCREAMING_SNAKE_CASE_  (						self	:		int						)			->	Optional[int]:




											'''simple docstring'''

											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											__snake_case				=		[
											    "l",
											    "o",
											    "w",
											    "e",
											    "r",
											    "s",
											    "t",
											    "i",
											    "d",
											    "n",
											    "\u0120",
											    "\u0120l",
											    "\u0120n",
											    "\u0120lo",
											    "\u0120low",
											    "er",
											    "\u0120lowest",
											    "\u0120newer",
											    "\u0120wider",
											    "<unk>",
											]
											__snake_case				=		dict(zip(SCREAMING_SNAKE_CASE     ,    range(len(SCREAMING_SNAKE_CASE						)						)						)						)
											__snake_case				=		["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
											__snake_case				=		{"unk_token": "<unk>"}

											__snake_case				=		os.path.join(self.tmpdirname     ,    VOCAB_FILES_NAMES["vocab_file"]						)
											__snake_case				=		os.path.join(self.tmpdirname     ,    VOCAB_FILES_NAMES["merges_file"]						)
											with open(self.vocab_file     ,    "w"     ,    encoding="utf-8"						) as fp:
																	fp.write(json.dumps(SCREAMING_SNAKE_CASE						) + "\n"						)
											with open(self.merges_file     ,    "w"     ,    encoding="utf-8"						) as fp:
																	fp.write("\n".join(SCREAMING_SNAKE_CASE						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple     ,    **SCREAMING_SNAKE_CASE	:		Optional[Any]						)			->	Union[str, Any]:




											'''simple docstring'''

											kwargs.update(self.special_tokens_map						)
											return self.tokenizer_class.from_pretrained(self.tmpdirname     ,    **SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]     ,    **SCREAMING_SNAKE_CASE	:		List[Any]						)			->	List[str]:




											'''simple docstring'''

											kwargs.update(self.special_tokens_map						)
											return RobertaTokenizerFast.from_pretrained(self.tmpdirname     ,    **SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		str     ,    SCREAMING_SNAKE_CASE	:		Union[str, Any]						)			->	List[str]:




											'''simple docstring'''

											__snake_case				=		"lower newer"
											__snake_case				=		"lower newer"
											return input_text, output_text


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple						)			->	Optional[int]:




											'''simple docstring'''

											__snake_case				=		self.tokenizer_class(self.vocab_file     ,    self.merges_file     ,    **self.special_tokens_map						)
											__snake_case				=		"lower newer"
											__snake_case				=		["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
											__snake_case				=		tokenizer.tokenize(SCREAMING_SNAKE_CASE						)  # , add_prefix_space=True)
											self.assertListEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)

											__snake_case				=		tokens + [tokenizer.unk_token]
											__snake_case				=		[0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE						)     ,    SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple						)			->	str:




											'''simple docstring'''

											__snake_case				=		self.get_tokenizer()

											self.assertListEqual(tokenizer.encode("Hello world!"     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)     ,    [0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]						)
											self.assertListEqual(
											    tokenizer.encode("Hello world! cécé herlolip 418"     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)     ,    [0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]     ,    )


					@slow
					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[Any]						)			->	Optional[Any]:




											'''simple docstring'''

											__snake_case				=		self.tokenizer_class.from_pretrained("roberta-base"						)

											__snake_case				=		tokenizer.encode("sequence builders"     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.encode("multi-sequence build"     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)

											__snake_case				=		tokenizer.encode(
											    "sequence builders"     ,    add_special_tokens=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.encode(
											    "sequence builders"     ,    "multi-sequence build"     ,    add_special_tokens=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE						)

											__snake_case				=		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)

											assert encoded_sentence == encoded_text_from_decode
											assert encoded_pair == encoded_pair_from_decode


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]						)			->	Tuple:




											'''simple docstring'''

											__snake_case				=		self.get_tokenizer()

											__snake_case				=		"Encode this sequence."
											__snake_case				=		tokenizer.byte_encoder[" ".encode("utf-8"						)[0]]

											# Testing encoder arguments
											__snake_case				=		tokenizer.encode(SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
											self.assertNotEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)

											__snake_case				=		tokenizer.encode(SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
											self.assertEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)

											tokenizer.add_special_tokens({"bos_token": "<s>"}						)
											__snake_case				=		tokenizer.encode(SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.convert_ids_to_tokens(encoded[1]						)[0]
											self.assertNotEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)

											# Testing spaces after special tokens
											__snake_case				=		"<mask>"
											tokenizer.add_special_tokens(
											    {"mask_token": AddedToken(SCREAMING_SNAKE_CASE     ,    lstrip=SCREAMING_SNAKE_CASE     ,    rstrip=SCREAMING_SNAKE_CASE						)}						)  # mask token has a left space
											__snake_case				=		tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE						)

											__snake_case				=		"Encode <mask> sequence"
											__snake_case				=		"Encode <mask>sequence"

											__snake_case				=		tokenizer.encode(SCREAMING_SNAKE_CASE						)
											__snake_case				=		encoded.index(SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
											self.assertEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)

											__snake_case				=		tokenizer.encode(SCREAMING_SNAKE_CASE						)
											__snake_case				=		encoded.index(SCREAMING_SNAKE_CASE						)
											__snake_case				=		tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
											self.assertNotEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[Any]						)			->	int:




											'''simple docstring'''

											pass


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]						)			->	Union[str, Any]:




											'''simple docstring'''

											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     ,    **SCREAMING_SNAKE_CASE						)
																							__snake_case				=		self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     ,    **SCREAMING_SNAKE_CASE						)
																							__snake_case				=		"A, <mask> AllenNLP sentence."
																							__snake_case				=		tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE     ,    return_token_type_ids=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_p.encode_plus(SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE     ,    return_token_type_ids=SCREAMING_SNAKE_CASE						)

																							# token_type_ids should put 0 everywhere
																							self.assertEqual(sum(tokens_r["token_type_ids"]						)     ,    sum(tokens_p["token_type_ids"]						)						)

																							# attention_mask should put 1 everywhere, so sum over length should be 1
																							self.assertEqual(
																							    sum(tokens_r["attention_mask"]						) / len(tokens_r["attention_mask"]						)     ,    sum(tokens_p["attention_mask"]						) / len(tokens_p["attention_mask"]						)     ,    )

																							__snake_case				=		tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]						)
																							__snake_case				=		tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]						)

																							# Rust correctly handles the space before the mask while python doesnt
																							self.assertSequenceEqual(tokens_p["input_ids"]     ,    [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]						)
																							self.assertSequenceEqual(tokens_r["input_ids"]     ,    [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]						)

																							self.assertSequenceEqual(
																							    SCREAMING_SNAKE_CASE     ,    ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)
																							self.assertSequenceEqual(
																							    SCREAMING_SNAKE_CASE     ,    ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict						)			->	Dict:




											'''simple docstring'''

											for trim_offsets, add_prefix_space in itertools.product([True, False]     ,    repeat=2						):
																	__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																	    self.tmpdirname     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)

																	__snake_case				=		json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()						)
																	__snake_case				=		json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()						)

																	self.assertEqual(pre_tokenizer_state["add_prefix_space"]     ,    SCREAMING_SNAKE_CASE						)

																	self.assertEqual(post_processor_state["add_prefix_space"]     ,    SCREAMING_SNAKE_CASE						)
																	self.assertEqual(post_processor_state["trim_offsets"]     ,    SCREAMING_SNAKE_CASE						)






					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict						)			->	Union[str, Any]:




											'''simple docstring'''

											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
																							__snake_case				=		"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																							__snake_case				=		f'''{text_of_1_token} {text_of_1_token}'''

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (0, len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (len(SCREAMING_SNAKE_CASE						) + 1, len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (0, len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (len(SCREAMING_SNAKE_CASE						) + 1, len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (0, len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (len(SCREAMING_SNAKE_CASE						), len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (0, len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (len(SCREAMING_SNAKE_CASE						), len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )

																							__snake_case				=		f''' {text}'''

																							# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																							#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																							# )
																							# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																							# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																							# self.assertEqual(
																							#     encoding.offset_mapping[1],
																							#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																							# )

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (1, 1 + len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (1 + len(SCREAMING_SNAKE_CASE						) + 1, 1 + len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (0, 1 + len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (1 + len(SCREAMING_SNAKE_CASE						), 1 + len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )

																							__snake_case				=		self.rust_tokenizer_class.from_pretrained(
																							    SCREAMING_SNAKE_CASE     ,    use_fast=SCREAMING_SNAKE_CASE     ,    add_prefix_space=SCREAMING_SNAKE_CASE     ,    trim_offsets=SCREAMING_SNAKE_CASE						)
																							__snake_case				=		tokenizer_r(SCREAMING_SNAKE_CASE     ,    return_offsets_mapping=SCREAMING_SNAKE_CASE     ,    add_special_tokens=SCREAMING_SNAKE_CASE						)
																							self.assertEqual(encoding.offset_mapping[0]     ,    (0, 1 + len(SCREAMING_SNAKE_CASE						))						)
																							self.assertEqual(
																							    encoding.offset_mapping[1]     ,    (1 + len(SCREAMING_SNAKE_CASE						), 1 + len(SCREAMING_SNAKE_CASE						) + 1 + len(SCREAMING_SNAKE_CASE						))     ,    )









def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	int:


						'''simple docstring'''






						__snake_case				=		abs(_lowerCAmelCase )
						__snake_case				=		0
						while n > 0:
												res += n % 10
												n //= 10
						return res





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	int:


						'''simple docstring'''






						__snake_case				=		abs(_lowerCAmelCase )
						return n if n < 10 else n % 10 + sum_of_digits(n // 10 )





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	int:


						'''simple docstring'''






						return sum(int(_lowerCAmelCase ) for c in str(abs(_lowerCAmelCase ) ) )





def 	_lowerCAmelCase       (    )						->	None:


						'''simple docstring'''






						from collections.abc import Callable
						from timeit import timeit

						def benchmark_a_function(_lowerCAmelCase      ,	_lowerCAmelCase ) -> None:
												__snake_case				=		F'''{func.__name__}({value})'''
												__snake_case				=		timeit(F'''__main__.{call}'''      ,	setup="import __main__" )
												print(F'''{call:56} = {func(_lowerCAmelCase )} -- {timing:.4f} seconds''' )

						for value in (26_2144, 1125_8999_0684_2624, 126_7650_6002_2822_9401_4967_0320_5376):
												for func in (sum_of_digits, sum_of_digits_recursion, sum_of_digits_compact):
																		benchmark_a_function(_lowerCAmelCase      ,	_lowerCAmelCase )
												print()


if __name__ == "__main__":
	import doctest

	doctest.testmod()
	benchmark()





