

"""simple docstring"""





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class __lowerCAmelCase				(					__a     ,				unittest.TestCase      ):




			"""simple docstring"""







			snake_case	      =				GPTSanJapaneseTokenizer
			snake_case	      =				False
			snake_case	      =				{"do_clean_text": False, "add_prefix_space": False}




			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> int:







								"""simple docstring"""
								super().setUp()

								# fmt: off
								A_				 =						["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
								# fmt: on
								A_				 =						{"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
								A_				 =						{"unk_token": "<unk>"}

								A_				 =						os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES["vocab_file"]	)
								A_				 =						os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES["emoji_file"]	)
								with open(self.vocab_file       ,   "w"       ,   encoding="utf-8"	) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
								with open(self.emoji_file       ,   "w"	) as emoji_writer:
													emoji_writer.write(json.dumps(A__	)	)




			def      lowerCamelCase__     (				self						:					Tuple       ,   **_snake_case						:					List[Any]	)      -> Union[str, Any]:







								"""simple docstring"""
								kwargs.update(self.special_tokens_map	)
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname       ,   **A__	)




			def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					Any	)      -> Any:







								"""simple docstring"""
								A_				 =						"こんにちは、世界。 \nこんばんは、㔺界。😀"
								A_				 =						"こんにちは、世界。 \nこんばんは、世界。😀"
								return input_text, output_text




			def      lowerCamelCase__     (				self						:					Optional[int]       ,   _snake_case						:					Optional[int]	)      -> str:







								"""simple docstring"""
								A_ ,    A_				 =						self.get_input_output_texts(A__	)
								A_				 =						tokenizer.encode(A__       ,   add_special_tokens=A__	)
								A_				 =						tokenizer.decode(A__       ,   clean_up_tokenization_spaces=A__	)
								return text, ids




			def      lowerCamelCase__     (				self						:					Dict	)      -> List[Any]:







								"""simple docstring"""
								pass  # TODO add if relevant




			def      lowerCamelCase__     (				self						:					Dict	)      -> int:







								"""simple docstring"""
								pass  # TODO add if relevant




			def      lowerCamelCase__     (				self						:					Any	)      -> Union[str, Any]:







								"""simple docstring"""
								pass  # TODO add if relevant




			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Dict:







								"""simple docstring"""
								A_				 =						self.get_tokenizer()

								# Testing tokenization
								A_				 =						"こんにちは、世界。　こんばんは、㔺界。"
								A_				 =						["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
								A_				 =						tokenizer.tokenize(A__	)
								self.assertListEqual(A__       ,   A__	)

								# Testing conversion to ids without special tokens
								A_				 =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								A_				 =						tokenizer.convert_tokens_to_ids(A__	)
								self.assertListEqual(A__       ,   A__	)

								# Testing conversion to ids with special tokens
								A_				 =						tokens + [tokenizer.unk_token]
								A_				 =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								A_				 =						tokenizer.convert_tokens_to_ids(A__	)
								self.assertListEqual(A__       ,   A__	)




			def      lowerCamelCase__     (				self						:					str	)      -> Dict:







								"""simple docstring"""
								A_				 =						self.get_tokenizer()

								# Testing tokenization
								A_				 =						"こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
								A_				 =						"こんにちは、、、、世界。こんばんは、、、、世界。"
								A_				 =						tokenizer.encode(A__	)
								A_				 =						tokenizer.decode(A__	)
								self.assertEqual(A__       ,   A__	)




			@slow
			def      lowerCamelCase__     (				self						:					Dict	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								# Testing tokenization
								A_				 =						"こんにちは、世界。"
								A_				 =						"こんばんは、㔺界。😀"
								A_				 =						"こんにちは、世界。こんばんは、世界。😀"
								A_				 =						tokenizer.encode(prefix_text + input_text	)
								A_				 =						tokenizer.encode(""       ,   prefix_text=prefix_text + input_text	)
								A_				 =						tokenizer.encode(A__       ,   prefix_text=A__	)
								A_				 =						tokenizer.decode(A__	)
								A_				 =						tokenizer.decode(A__	)
								A_				 =						tokenizer.decode(A__	)
								self.assertEqual(A__       ,   A__	)
								self.assertEqual(A__       ,   A__	)
								self.assertEqual(A__       ,   A__	)




			@slow
			def      lowerCamelCase__     (				self						:					Dict	)      -> int:







								"""simple docstring"""
								A_				 =						self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								# Testing tokenization
								A_				 =						"こんにちは、世界。"
								A_				 =						"こんばんは、㔺界。😀"

								A_				 =						len(tokenizer.encode(A__	)	) - 2
								A_				 =						len(tokenizer.encode(A__	)	) - 2

								A_				 =						[1] + [0] * (len_prefix + len_text + 1)
								A_				 =						[1] * (len_prefix + len_text + 1) + [0]
								A_				 =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								A_				 =						tokenizer(prefix_text + input_text	).token_type_ids
								A_				 =						tokenizer(""       ,   prefix_text=prefix_text + input_text	).token_type_ids
								A_				 =						tokenizer(A__       ,   prefix_text=A__	).token_type_ids
								self.assertListEqual(A__       ,   A__	)
								self.assertListEqual(A__       ,   A__	)
								self.assertListEqual(A__       ,   A__	)




			@slow
			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								A_				 =						tokenizer.encode("あンいワ"	)
								A_				 =						tokenizer.encode(""       ,   prefix_text="あンいワ"	)
								A_				 =						tokenizer.encode("いワ"       ,   prefix_text="あン"	)

								self.assertEqual(tokenizer.decode(A__	)       ,   tokenizer.decode(A__	)	)
								self.assertEqual(tokenizer.decode(A__	)       ,   tokenizer.decode(A__	)	)
								self.assertNotEqual(A__       ,   A__	)
								self.assertNotEqual(A__       ,   A__	)
								self.assertEqual(x_token_a[1]       ,   x_token_a[-1]	)  # SEG token
								self.assertEqual(x_token_a[1]       ,   x_token_a[3]	)  # SEG token




			@slow
			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								A_				 =						[["武田信玄", "は、"], ["織田信長", "の配下の、"]]
								A_				 =						tokenizer(A__       ,   padding=A__	)
								A_				 =						tokenizer.batch_encode_plus(A__       ,   padding=A__	)

								# fmt: off
								A_				 =						[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
								A_				 =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								A_				 =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids       ,   A__	)
								self.assertListEqual(x_token.token_type_ids       ,   A__	)
								self.assertListEqual(x_token.attention_mask       ,   A__	)
								self.assertListEqual(x_token_a.input_ids       ,   A__	)
								self.assertListEqual(x_token_a.token_type_ids       ,   A__	)
								self.assertListEqual(x_token_a.attention_mask       ,   A__	)




			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Optional[int]:







								"""simple docstring"""
								# Intentionally convert some words to accommodate character fluctuations unique to Japanese
								pass






			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Optional[int]:







								"""simple docstring"""
								# tokenizer has no padding token
								pass






"""simple docstring"""





from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand
def 							A_     (__a				):

					'''simple docstring'''





					return ConvertCommand(
					    args.model_type				,   args.tf_checkpoint				,   args.pytorch_dump_output				,   args.config				,   args.finetuning_task_name				)


UpperCamelCase_					:						Tuple				= '''
transformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires
TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.
'''




class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			@staticmethod
			def      lowerCamelCase__     (				_snake_case						:					ArgumentParser	)      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						parser.add_parser(
								    "convert"       ,   help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."       ,   )
								train_parser.add_argument("--model_type"       ,   type=_snake_case       ,   required=_snake_case       ,   help="Model's type."	)
								train_parser.add_argument(
								    "--tf_checkpoint"       ,   type=_snake_case       ,   required=_snake_case       ,   help="TensorFlow checkpoint path or folder."	)
								train_parser.add_argument(
								    "--pytorch_dump_output"       ,   type=_snake_case       ,   required=_snake_case       ,   help="Path to the PyTorch saved model output."	)
								train_parser.add_argument("--config"       ,   type=_snake_case       ,   default=""       ,   help="Configuration file path or folder."	)
								train_parser.add_argument(
								    "--finetuning_task_name"       ,   type=_snake_case       ,   default=_snake_case       ,   help="Optional fine-tuning task name if the TF model was a finetuned model."       ,   )
								train_parser.set_defaults(func=_snake_case	)




			def __init__(				self						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   _snake_case						:					str       ,   *_snake_case						:					Optional[int]       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						logging.get_logger("transformers-cli/converting"	)

								self._logger.info(F'Loading model {model_type}'	)
								A_				 =						model_type
								A_				 =						tf_checkpoint
								A_				 =						pytorch_dump_output
								A_				 =						config
								A_				 =						finetuning_task_name






			def      lowerCamelCase__     (				self						:					Tuple	)      -> Dict:







								"""simple docstring"""
								if self._model_type == "albert":
													try:
																		from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
																		    convert_tf_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "bert":
													try:
																		from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
																		    convert_tf_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "funnel":
													try:
																		from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
																		    convert_tf_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "t5":
													try:
																		from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "gpt":
													from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
													    convert_openai_checkpoint_to_pytorch,
													)

													convert_openai_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "transfo_xl":
													try:
																		from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
																		    convert_transfo_xl_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													if "ckpt" in self._tf_checkpoint.lower():
																		A_				 =						self._tf_checkpoint
																		A_				 =						""
													else:
																		A_				 =						self._tf_checkpoint
																		A_				 =						""
													convert_transfo_xl_checkpoint_to_pytorch(
													    _snake_case       ,   self._config       ,   self._pytorch_dump_output       ,   _snake_case	)
								elif self._model_type == "gpt2":
													try:
																		from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
																		    convert_gpta_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								elif self._model_type == "xlnet":
													try:
																		from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
																		    convert_xlnet_checkpoint_to_pytorch,
																		)
													except ImportError:
																		raise ImportError(_snake_case	)

													convert_xlnet_checkpoint_to_pytorch(
													    self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output       ,   self._finetuning_task_name	)
								elif self._model_type == "xlm":
													from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
													    convert_xlm_checkpoint_to_pytorch,
													)

													convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._pytorch_dump_output	)
								elif self._model_type == "lxmert":
													from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
													    convert_lxmert_checkpoint_to_pytorch,
													)

													convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._pytorch_dump_output	)
								elif self._model_type == "rembert":
													from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
													    convert_rembert_tf_checkpoint_to_pytorch,
													)

													convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,   self._config       ,   self._pytorch_dump_output	)
								else:
													raise ValueError(
													    "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]"	)




