




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 			UpperCAmelCase_      (   A		,      unittest.TestCase       ):




							'''simple docstring'''



							a__						  =     GPTSanJapaneseTokenizer
							a__						  =     False
							a__						  =     {'''do_clean_text''': False, '''add_prefix_space''': False}





							def    _UpperCAmelCase     (					self       :	Any			)			->					List[str]:
										super().setUp()

										# fmt: off
										SCREAMING_SNAKE_CASE           =					["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
										# fmt: on
										SCREAMING_SNAKE_CASE           =					{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
										SCREAMING_SNAKE_CASE           =					{"""unk_token""": """<unk>"""}

										SCREAMING_SNAKE_CASE           =					os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES["""vocab_file"""]			)
										SCREAMING_SNAKE_CASE           =					os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES["""emoji_file"""]			)
										with open(self.vocab_file       ,     """w"""       ,     encoding="""utf-8"""			) as vocab_writer:
													vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)
										with open(self.emoji_file       ,     """w"""			) as emoji_writer:
													emoji_writer.write(json.dumps(a			)			)





							def    _UpperCAmelCase     (					self       :	str       ,     **a       :	str			)			->					Dict:
										kwargs.update(self.special_tokens_map			)
										return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname       ,     **a			)





							def    _UpperCAmelCase     (					self       :	List[str]       ,     a       :	Dict			)			->					Any:
										SCREAMING_SNAKE_CASE           =					"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
										SCREAMING_SNAKE_CASE           =					"""こんにちは、世界。 \nこんばんは、世界。😀"""
										return input_text, output_text





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	Union[str, Any]			)			->					List[Any]:
										SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					self.get_input_output_texts(a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.encode(a       ,     add_special_tokens=a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(a       ,     clean_up_tokenization_spaces=a			)
										return text, ids





							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										pass  # TODO add if relevant





							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					List[Any]:
										pass  # TODO add if relevant





							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					List[str]:
										pass  # TODO add if relevant





							def    _UpperCAmelCase     (					self       :	Optional[Any]			)			->					int:
										SCREAMING_SNAKE_CASE           =					self.get_tokenizer()

										# Testing tokenization
										SCREAMING_SNAKE_CASE           =					"""こんにちは、世界。　こんばんは、㔺界。"""
										SCREAMING_SNAKE_CASE           =					["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
										SCREAMING_SNAKE_CASE           =					tokenizer.tokenize(a			)
										self.assertListEqual(a       ,     a			)

										# Testing conversion to ids without special tokens
										SCREAMING_SNAKE_CASE           =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
										SCREAMING_SNAKE_CASE           =					tokenizer.convert_tokens_to_ids(a			)
										self.assertListEqual(a       ,     a			)

										# Testing conversion to ids with special tokens
										SCREAMING_SNAKE_CASE           =					tokens + [tokenizer.unk_token]
										SCREAMING_SNAKE_CASE           =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
										SCREAMING_SNAKE_CASE           =					tokenizer.convert_tokens_to_ids(a			)
										self.assertListEqual(a       ,     a			)





							def    _UpperCAmelCase     (					self       :	str			)			->					Tuple:
										SCREAMING_SNAKE_CASE           =					self.get_tokenizer()

										# Testing tokenization
										SCREAMING_SNAKE_CASE           =					"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
										SCREAMING_SNAKE_CASE           =					"""こんにちは、、、、世界。こんばんは、、、、世界。"""
										SCREAMING_SNAKE_CASE           =					tokenizer.encode(a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(a			)
										self.assertEqual(a       ,     a			)





							@slow
							def    _UpperCAmelCase     (					self       :	Tuple			)			->					List[str]:
										SCREAMING_SNAKE_CASE           =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""			)

										# Testing tokenization
										SCREAMING_SNAKE_CASE           =					"""こんにちは、世界。"""
										SCREAMING_SNAKE_CASE           =					"""こんばんは、㔺界。😀"""
										SCREAMING_SNAKE_CASE           =					"""こんにちは、世界。こんばんは、世界。😀"""
										SCREAMING_SNAKE_CASE           =					tokenizer.encode(prefix_text + input_text			)
										SCREAMING_SNAKE_CASE           =					tokenizer.encode(""""""       ,     prefix_text=prefix_text + input_text			)
										SCREAMING_SNAKE_CASE           =					tokenizer.encode(a       ,     prefix_text=a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(a			)
										self.assertEqual(a       ,     a			)
										self.assertEqual(a       ,     a			)
										self.assertEqual(a       ,     a			)





							@slow
							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Any:
										SCREAMING_SNAKE_CASE           =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""			)

										# Testing tokenization
										SCREAMING_SNAKE_CASE           =					"""こんにちは、世界。"""
										SCREAMING_SNAKE_CASE           =					"""こんばんは、㔺界。😀"""

										SCREAMING_SNAKE_CASE           =					len(tokenizer.encode(a			)			) - 2
										SCREAMING_SNAKE_CASE           =					len(tokenizer.encode(a			)			) - 2

										SCREAMING_SNAKE_CASE           =					[1] + [0] * (len_prefix + len_text + 1)
										SCREAMING_SNAKE_CASE           =					[1] * (len_prefix + len_text + 1) + [0]
										SCREAMING_SNAKE_CASE           =					[1] + [1] * (len_prefix) + [0] * (len_text + 1)

										SCREAMING_SNAKE_CASE           =					tokenizer(prefix_text + input_text			).token_type_ids
										SCREAMING_SNAKE_CASE           =					tokenizer(""""""       ,     prefix_text=prefix_text + input_text			).token_type_ids
										SCREAMING_SNAKE_CASE           =					tokenizer(a       ,     prefix_text=a			).token_type_ids
										self.assertListEqual(a       ,     a			)
										self.assertListEqual(a       ,     a			)
										self.assertListEqual(a       ,     a			)





							@slow
							def    _UpperCAmelCase     (					self       :	Any			)			->					Optional[int]:
										SCREAMING_SNAKE_CASE           =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""			)

										SCREAMING_SNAKE_CASE           =					tokenizer.encode("""あンいワ"""			)
										SCREAMING_SNAKE_CASE           =					tokenizer.encode(""""""       ,     prefix_text="""あンいワ"""			)
										SCREAMING_SNAKE_CASE           =					tokenizer.encode("""いワ"""       ,     prefix_text="""あン"""			)

										self.assertEqual(tokenizer.decode(a			)       ,     tokenizer.decode(a			)			)
										self.assertEqual(tokenizer.decode(a			)       ,     tokenizer.decode(a			)			)
										self.assertNotEqual(a       ,     a			)
										self.assertNotEqual(a       ,     a			)
										self.assertEqual(x_token_a[1]       ,     x_token_a[-1]			)  # SEG token
										self.assertEqual(x_token_a[1]       ,     x_token_a[3]			)  # SEG token





							@slow
							def    _UpperCAmelCase     (					self       :	Dict			)			->					Union[str, Any]:
										SCREAMING_SNAKE_CASE           =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""			)

										SCREAMING_SNAKE_CASE           =					[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
										SCREAMING_SNAKE_CASE           =					tokenizer(a       ,     padding=a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.batch_encode_plus(a       ,     padding=a			)

										# fmt: off
										SCREAMING_SNAKE_CASE           =					[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
										SCREAMING_SNAKE_CASE           =					[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
										SCREAMING_SNAKE_CASE           =					[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
										# fmt: on
										self.assertListEqual(x_token.input_ids       ,     a			)
										self.assertListEqual(x_token.token_type_ids       ,     a			)
										self.assertListEqual(x_token.attention_mask       ,     a			)
										self.assertListEqual(x_token_a.input_ids       ,     a			)
										self.assertListEqual(x_token_a.token_type_ids       ,     a			)
										self.assertListEqual(x_token_a.attention_mask       ,     a			)





							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										# Intentionally convert some words to accommodate character fluctuations unique to Japanese
										pass







							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Optional[Any]:
										# tokenizer has no padding token
										pass












from __future__ import annotations

import typing
from collections.abc import Iterable

import numpy as np

__A						:							Dict							=  typing.Union[Iterable[float], Iterable[int], np.ndarray]  # noqa: UP007
__A						:							Tuple							=  typing.Union[np.floataa, int, float]  # noqa: UP007
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			return np.sqrt(np.sum((np.asarray(SCREAMING_SNAKE_CASE   ) - np.asarray(SCREAMING_SNAKE_CASE   )) ** 2   )   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			return sum((va - va) ** 2 for va, va in zip(SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   )   ) ** (1 / 2)


if __name__ == "__main__":
		def 					lowerCamelCase_    (    ):
					'''simple docstring'''
					from timeit import timeit

					print("""Without Numpy"""   )
					print(
					    timeit(
					        """euclidean_distance_no_np([1, 2, 3], [4, 5, 6])"""					,	number=1_00_00					,	globals=globals()					,	)   )
					print("""With Numpy"""   )
					print(
					    timeit(
					        """euclidean_distance([1, 2, 3], [4, 5, 6])"""					,	number=1_00_00					,	globals=globals()					,	)   )




		benchmark()






