

'''simple docstring'''


import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     a			(				SCREAMING_SNAKE_CASE			,			unittest.TestCase       ):






						"""simple docstring"""




						__UpperCAmelCase			     =    GPTSanJapaneseTokenizer
						__UpperCAmelCase			     =    False
						__UpperCAmelCase			     =    {"""do_clean_text""": False, """add_prefix_space""": False}







						def       __magic_name__							(							self					:      int  ):



								'''simple docstring'''



								super().setUp()

								# fmt: off
								snake_case__						:       List[Any]       						=  ['''こん''', '''こんに''', '''にちは''', '''ばんは''', '''世界,㔺界''', '''、''', '''。''', '''<BR>''', '''<SP>''', '''<TAB>''', '''<URL>''', '''<EMAIL>''', '''<TEL>''', '''<DATE>''', '''<PRICE>''', '''<BLOCK>''', '''<KIGOU>''', '''<U2000U2BFF>''', '''<|emoji1|>''', '''<unk>''', '''<|bagoftoken|>''', '''<|endoftext|>''']
								# fmt: on
								snake_case__						:       List[Any]       						=  {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
								snake_case__						:       Optional[int]       						=  {'''unk_token''': '''<unk>'''}

								snake_case__						:       Tuple       						=  os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['''vocab_file''']  )
								snake_case__						:       int       						=  os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['''emoji_file''']  )
								with open(self.vocab_file			,			'''w'''			,			encoding='''utf-8'''  ) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )
								with open(self.emoji_file			,			'''w'''  ) as emoji_writer:
										emoji_writer.write(json.dumps(snake_case_  )  )







						def       __magic_name__							(							self					:      Union[str, Any]			,			**snake_case_					:      Tuple  ):



								'''simple docstring'''



								kwargs.update(self.special_tokens_map  )
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname			,			**snake_case_  )







						def       __magic_name__							(							self					:      Optional[int]			,			snake_case_					:      str  ):



								'''simple docstring'''



								snake_case__						:       Tuple       						=  '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
								snake_case__						:       Union[str, Any]       						=  '''こんにちは、世界。 \nこんばんは、世界。😀'''
								return input_text, output_text







						def       __magic_name__							(							self					:      Any			,			snake_case_					:      Tuple  ):



								'''simple docstring'''



								snake_case__						,       snake_case__						:       Optional[Any]       						=  self.get_input_output_texts(snake_case_  )
								snake_case__						:       int       						=  tokenizer.encode(snake_case_			,			add_special_tokens=snake_case_  )
								snake_case__						:       int       						=  tokenizer.decode(snake_case_			,			clean_up_tokenization_spaces=snake_case_  )
								return text, ids







						def       __magic_name__							(							self					:      int  ):



								'''simple docstring'''



								pass  # TODO add if relevant







						def       __magic_name__							(							self					:      List[Any]  ):



								'''simple docstring'''



								pass  # TODO add if relevant







						def       __magic_name__							(							self					:      Tuple  ):



								'''simple docstring'''



								pass  # TODO add if relevant







						def       __magic_name__							(							self					:      Tuple  ):



								'''simple docstring'''



								snake_case__						:       str       						=  self.get_tokenizer()

								# Testing tokenization
								snake_case__						:       Any       						=  '''こんにちは、世界。　こんばんは、㔺界。'''
								snake_case__						:       Dict       						=  ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
								snake_case__						:       Optional[Any]       						=  tokenizer.tokenize(snake_case_  )
								self.assertListEqual(snake_case_			,			snake_case_  )

								# Testing conversion to ids without special tokens
								snake_case__						:       str       						=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								snake_case__						:       List[Any]       						=  tokenizer.convert_tokens_to_ids(snake_case_  )
								self.assertListEqual(snake_case_			,			snake_case_  )

								# Testing conversion to ids with special tokens
								snake_case__						:       Dict       						=  tokens + [tokenizer.unk_token]
								snake_case__						:       int       						=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
								snake_case__						:       Union[str, Any]       						=  tokenizer.convert_tokens_to_ids(snake_case_  )
								self.assertListEqual(snake_case_			,			snake_case_  )







						def       __magic_name__							(							self					:      int  ):



								'''simple docstring'''



								snake_case__						:       Union[str, Any]       						=  self.get_tokenizer()

								# Testing tokenization
								snake_case__						:       Optional[int]       						=  '''こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'''
								snake_case__						:       int       						=  '''こんにちは、、、、世界。こんばんは、、、、世界。'''
								snake_case__						:       Tuple       						=  tokenizer.encode(snake_case_  )
								snake_case__						:       Any       						=  tokenizer.decode(snake_case_  )
								self.assertEqual(snake_case_			,			snake_case_  )







						@slow
						def       __magic_name__							(							self					:      List[Any]  ):



								'''simple docstring'''



								snake_case__						:       Dict       						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''  )

								# Testing tokenization
								snake_case__						:       Optional[Any]       						=  '''こんにちは、世界。'''
								snake_case__						:       Any       						=  '''こんばんは、㔺界。😀'''
								snake_case__						:       Optional[Any]       						=  '''こんにちは、世界。こんばんは、世界。😀'''
								snake_case__						:       Tuple       						=  tokenizer.encode(prefix_text + input_text  )
								snake_case__						:       str       						=  tokenizer.encode(''''''			,			prefix_text=prefix_text + input_text  )
								snake_case__						:       List[Any]       						=  tokenizer.encode(snake_case_			,			prefix_text=snake_case_  )
								snake_case__						:       List[str]       						=  tokenizer.decode(snake_case_  )
								snake_case__						:       Optional[Any]       						=  tokenizer.decode(snake_case_  )
								snake_case__						:       str       						=  tokenizer.decode(snake_case_  )
								self.assertEqual(snake_case_			,			snake_case_  )
								self.assertEqual(snake_case_			,			snake_case_  )
								self.assertEqual(snake_case_			,			snake_case_  )







						@slow
						def       __magic_name__							(							self					:      Optional[int]  ):



								'''simple docstring'''



								snake_case__						:       Any       						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''  )

								# Testing tokenization
								snake_case__						:       Dict       						=  '''こんにちは、世界。'''
								snake_case__						:       List[str]       						=  '''こんばんは、㔺界。😀'''

								snake_case__						:       List[Any]       						=  len(tokenizer.encode(snake_case_  )  ) - 2
								snake_case__						:       str       						=  len(tokenizer.encode(snake_case_  )  ) - 2

								snake_case__						:       int       						=  [1] + [0] * (len_prefix + len_text + 1)
								snake_case__						:       int       						=  [1] * (len_prefix + len_text + 1) + [0]
								snake_case__						:       int       						=  [1] + [1] * (len_prefix) + [0] * (len_text + 1)

								snake_case__						:       Union[str, Any]       						=  tokenizer(prefix_text + input_text  ).token_type_ids
								snake_case__						:       int       						=  tokenizer(''''''			,			prefix_text=prefix_text + input_text  ).token_type_ids
								snake_case__						:       Any       						=  tokenizer(snake_case_			,			prefix_text=snake_case_  ).token_type_ids
								self.assertListEqual(snake_case_			,			snake_case_  )
								self.assertListEqual(snake_case_			,			snake_case_  )
								self.assertListEqual(snake_case_			,			snake_case_  )







						@slow
						def       __magic_name__							(							self					:      Optional[Any]  ):



								'''simple docstring'''



								snake_case__						:       Any       						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''  )

								snake_case__						:       Dict       						=  tokenizer.encode('''あンいワ'''  )
								snake_case__						:       int       						=  tokenizer.encode(''''''			,			prefix_text='''あンいワ'''  )
								snake_case__						:       Tuple       						=  tokenizer.encode('''いワ'''			,			prefix_text='''あン'''  )

								self.assertEqual(tokenizer.decode(snake_case_  )			,			tokenizer.decode(snake_case_  )  )
								self.assertEqual(tokenizer.decode(snake_case_  )			,			tokenizer.decode(snake_case_  )  )
								self.assertNotEqual(snake_case_			,			snake_case_  )
								self.assertNotEqual(snake_case_			,			snake_case_  )
								self.assertEqual(x_token_a[1]			,			x_token_a[-1]  )  # SEG token
								self.assertEqual(x_token_a[1]			,			x_token_a[3]  )  # SEG token







						@slow
						def       __magic_name__							(							self					:      Dict  ):



								'''simple docstring'''



								snake_case__						:       Any       						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''  )

								snake_case__						:       int       						=  [['''武田信玄''', '''は、'''], ['''織田信長''', '''の配下の、''']]
								snake_case__						:       Optional[Any]       						=  tokenizer(snake_case_			,			padding=snake_case_  )
								snake_case__						:       Tuple       						=  tokenizer.batch_encode_plus(snake_case_			,			padding=snake_case_  )

								# fmt: off
								snake_case__						:       Any       						=  [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
								snake_case__						:       Dict       						=  [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								snake_case__						:       List[Any]       						=  [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids			,			snake_case_  )
								self.assertListEqual(x_token.token_type_ids			,			snake_case_  )
								self.assertListEqual(x_token.attention_mask			,			snake_case_  )
								self.assertListEqual(x_token_a.input_ids			,			snake_case_  )
								self.assertListEqual(x_token_a.token_type_ids			,			snake_case_  )
								self.assertListEqual(x_token_a.attention_mask			,			snake_case_  )







						def       __magic_name__							(							self					:      Dict  ):



								'''simple docstring'''



								pass







						def       __magic_name__							(							self					:      Optional[Any]  ):



								'''simple docstring'''



								pass






'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__						:       Dict										=						logging.get_logger(__name__)

lowerCAmelCase__						:       Tuple										=						{"""ctrl""": """https://huggingface.co/ctrl/resolve/main/config.json"""}




class     a			(				SCREAMING_SNAKE_CASE       ):






						"""simple docstring"""




						__UpperCAmelCase			     =    """ctrl"""
						__UpperCAmelCase			     =    ["""past_key_values"""]
						__UpperCAmelCase			     =    {
						    """max_position_embeddings""": """n_positions""",
						    """hidden_size""": """n_embd""",
						    """num_attention_heads""": """n_head""",
						    """num_hidden_layers""": """n_layer""",
						}







						def __init__(							self					:      Tuple			,			snake_case_					:      Dict=2_4_6_5_3_4			,			snake_case_					:      Optional[int]=2_5_6			,			snake_case_					:      Dict=1_2_8_0			,			snake_case_					:      Union[str, Any]=8_1_9_2			,			snake_case_					:      Any=4_8			,			snake_case_					:      List[Any]=1_6			,			snake_case_					:      Optional[Any]=0.1			,			snake_case_					:      Union[str, Any]=0.1			,			snake_case_					:      Optional[Any]=1e-6			,			snake_case_					:      List[Any]=0.0_2			,			snake_case_					:      Dict=True			,			**snake_case_					:      List[Any]			,			):



								'''simple docstring'''



								snake_case__						:       Any       						=  vocab_size
								snake_case__						:       int       						=  n_positions
								snake_case__						:       Optional[int]       						=  n_embd
								snake_case__						:       str       						=  n_layer
								snake_case__						:       Any       						=  n_head
								snake_case__						:       str       						=  dff
								snake_case__						:       Any       						=  resid_pdrop
								snake_case__						:       Tuple       						=  embd_pdrop
								snake_case__						:       List[str]       						=  layer_norm_epsilon
								snake_case__						:       int       						=  initializer_range

								snake_case__						:       Optional[int]       						=  use_cache

								super().__init__(**snake_case_  )



