import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch





def 					lowercase_ (    __snake_case     :							str      )     -> List[Any]:





				'''simple docstring'''


				snake_case__      :Union[str, Any]         =   os.path.join(args.tf_model_dir	,	"parameters.json"      )
				snake_case__      :List[Any]         =   json.loads(open(__snake_case      ).read()      )
				if not params:
								raise ValueError(
								    F'It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file.'      )
				if not args.output.endswith(".pt"      ):
								snake_case__      :Any         =   args.output + ".pt"
				snake_case__      :Optional[Any]         =   OrderedDict()
				with tf.device("/CPU:0"      ):
								snake_case__      :Union[str, Any]         =   tf.train.load_checkpoint(args.tf_model_dir      )
								snake_case__      :List[Any]         =   reader.get_variable_to_shape_map()
								for key_name in shapes.keys():
												snake_case__      :Tuple         =   reader.get_tensor(__snake_case      ).astype(np.floataa      )
												if key_name.endswith("/adam_m"      ) or key_name.endswith("/adam_v"      ):
																continue
												if key_name.startswith("pasts/"      ):
																if key_name.startswith("pasts/mlp"      ):
																				snake_case__      :List[str]         =   int(key_name[9]      )
																elif key_name.startswith("pasts/out"      ):
																				snake_case__      :List[str]         =   8
																snake_case__      :Union[str, Any]         =   "model.sqout.%d.weight" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
																snake_case__      :Optional[int]         =   vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
																snake_case__      :List[Any]         =   torch.tensor(__snake_case      )
												elif key_name.startswith("model/moe"      ):
																snake_case__      :Tuple         =   int(key_name[9:].split("/"      )[0]      )
																if key_name.endswith("/switch_gating/kernel"      ):
																				snake_case__      :List[str]         =   "model.blocks.%d.feed_forward.mlp.router.classifier.weight" % player
																				snake_case__      :int         =   vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :Any         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/softmlp/kernel"      ):
																				snake_case__      :Tuple         =   "model.blocks.%d.feed_forward.soft_bypass_mlp.weight" % player
																				snake_case__      :List[Any]         =   vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :List[str]         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/wo/kernel"      ) or key_name.endswith("/wi/kernel"      ):
																				snake_case__      :List[str]         =   key_name[-9:-7]
																				for i in range(16      ):
																								snake_case__      :List[Any]         =   "model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight" % (player, i, nlayer)
																								snake_case__      :Optional[int]         =   (
																								    vnp[i].transpose([1, 0]      ).copy()
																								)  # In Mesh-Tensorflow, it is one array, so it is divided
																								snake_case__      :Optional[int]         =   torch.tensor(__snake_case      )
												elif key_name.startswith("model/mlp"      ):
																snake_case__      :Any         =   int(key_name[9:].split("/"      )[0]      )
																if key_name.endswith("/p1/kernel"      ):
																				snake_case__      :Dict         =   "model.blocks.%d.feed_forward.mlp.wi.weight" % player
																				snake_case__      :Optional[int]         =   vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :Tuple         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/p1/bias"      ):
																				snake_case__      :Dict         =   "model.blocks.%d.feed_forward.mlp.wi.bias" % player
																				snake_case__      :Any         =   vnp.copy()  # same because it is one dimensional
																				snake_case__      :Union[str, Any]         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/p2/kernel"      ):
																				snake_case__      :Optional[Any]         =   "model.blocks.%d.feed_forward.mlp.wo.weight" % player
																				snake_case__      :Dict         =   vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :str         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/p2/bias"      ):
																				snake_case__      :Union[str, Any]         =   "model.blocks.%d.feed_forward.mlp.wo.bias" % player
																				snake_case__      :Optional[Any]         =   vnp.copy()  # same because it is one dimensional
																				snake_case__      :List[str]         =   torch.tensor(__snake_case      )
												elif key_name.startswith("model/ln"      ):
																snake_case__      :Optional[Any]         =   int(key_name[8:].split("/"      )[0]      )
																if key_name.endswith("/b"      ):
																				snake_case__      :Tuple         =   "model.blocks.%d.feed_forward.norm.bias" % player
																				snake_case__      :List[str]         =   vnp.copy()  # same because it is one dimensional
																				snake_case__      :Dict         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/g"      ):
																				snake_case__      :Optional[int]         =   "model.blocks.%d.feed_forward.norm.weight" % player
																				snake_case__      :str         =   vnp.copy()  # same because it is one dimensional
																				snake_case__      :Optional[int]         =   torch.tensor(__snake_case      )
												elif key_name.startswith("model/att"      ):
																snake_case__      :Tuple         =   int(key_name[9:].split("/"      )[0]      )
																if key_name.endswith("/qkv/kernel"      ):
																				snake_case__      :List[str]         =   vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
																				snake_case__      :str         =   state[:, 0, :, :]
																				snake_case__      :Optional[int]         =   state[:, 1, :, :]
																				snake_case__      :Any         =   state[:, 2, :, :]
																				snake_case__      :Dict         =   (
																				    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]      )
																				    .transpose([1, 0]      )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :Dict         =   (
																				    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]      )
																				    .transpose([1, 0]      )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :List[str]         =   (
																				    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]      )
																				    .transpose([1, 0]      )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :int         =   "model.blocks.%d.self_attn.self_attn.q_proj.weight" % player
																				snake_case__      :List[Any]         =   torch.tensor(__snake_case      )
																				snake_case__      :Optional[int]         =   "model.blocks.%d.self_attn.self_attn.k_proj.weight" % player
																				snake_case__      :Union[str, Any]         =   torch.tensor(__snake_case      )
																				snake_case__      :Any         =   "model.blocks.%d.self_attn.self_attn.v_proj.weight" % player
																				snake_case__      :int         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/o/kernel"      ):
																				snake_case__      :Dict         =   "model.blocks.%d.self_attn.self_attn.out_proj.weight" % player
																				snake_case__      :Union[str, Any]         =   (
																				    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]      ).transpose([1, 0]      ).copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				snake_case__      :Tuple         =   torch.tensor(__snake_case      )
												elif key_name.startswith("model/an"      ):
																snake_case__      :int         =   int(key_name[8:].split("/"      )[0]      )
																if key_name.endswith("/b"      ):
																				snake_case__      :str         =   "model.blocks.%d.self_attn.norm.bias" % player
																				snake_case__      :Tuple         =   vnp.copy()  # same because it is one dimensional
																				snake_case__      :Tuple         =   torch.tensor(__snake_case      )
																elif key_name.endswith("/g"      ):
																				snake_case__      :Any         =   "model.blocks.%d.self_attn.norm.weight" % player
																				snake_case__      :List[str]         =   vnp.copy()  # same because it is one dimensional
																				snake_case__      :Optional[Any]         =   torch.tensor(__snake_case      )
												elif (
												    key_name.startswith("model/wte"      )
												    or key_name.startswith("model/wpe"      )
												    or key_name.startswith("model/ete"      )
												):
																snake_case__      :List[str]         =   {"wte": "embed_tokens", "wpe": "position_embeddings", "ete": "extra_position_embeddings"}[
																    key_name[-3:]
																]
																snake_case__      :Optional[Any]         =   "model.%s.weight" % nlayer
																snake_case__      :str         =   vnp.copy()  # same in embedded
																snake_case__      :str         =   torch.tensor(__snake_case      )
																if key_name.startswith("model/wte"      ):
																				snake_case__      :str         =   "lm_head.weight"
																				snake_case__      :str         =   vnp.copy()  # same in embedded
																				snake_case__      :Optional[int]         =   torch.tensor(__snake_case      )
												elif key_name.startswith("model/wob"      ):
																snake_case__      :List[str]         =   "final_logits_bias"
																snake_case__      :Any         =   vnp.copy()  # same in embedded
																snake_case__      :int         =   state.reshape((1, -1)      )
																snake_case__      :List[str]         =   torch.tensor(__snake_case      )
												elif key_name == "model/dense/kernel":
																snake_case__      :Optional[int]         =   "model.last_project.weight"
																snake_case__      :Tuple         =   vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
																snake_case__      :Tuple         =   torch.tensor(__snake_case      )
												elif key_name == "model/dense_1/bias":
																snake_case__      :str         =   "model.last_project.bias"
																snake_case__      :List[str]         =   vnp.copy()  # same because it is one dimensional
																snake_case__      :int         =   torch.tensor(__snake_case      )
				torch.save(__snake_case	,	args.output      )


if __name__ == "__main__":
	__UpperCAmelCase							:   Optional[int]							=  argparse.ArgumentParser(
	    description="model converter.", formatter_class=argparse.ArgumentDefaultsHelpFormatter
	)
	parser.add_argument("--tf_model_dir", metavar="PATH", type=str, required=True, help="import model")
	parser.add_argument("--output", metavar="PATH", type=str, required=True, help="output model")
	__UpperCAmelCase							:   Union[str, Any]							=  parser.parse_args()
	convert_tf_gptsan_to_pt(args)







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast


class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					# A mock response for an HTTP head request to emulate server down
					snake_case__      :Tuple         =   mock.Mock()
					snake_case__      :List[str]         =   500
					snake_case__      :Any         =   {}
					snake_case__      :Union[str, Any]         =   HTTPError
					snake_case__      :Tuple         =   {}

					# Download this model to make sure it's in the cache.
					snake_case__      :Any         =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch("requests.Session.request"					,return_value=UpperCamelCase       ) as mock_head:
									snake_case__      :Dict         =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
									# This check we did call the fake head request
									mock_head.assert_called()






	@require_tokenizers
	def    lowerCAmelCase_							(		self       )						->    Dict:
					# A mock response for an HTTP head request to emulate server down
					snake_case__      :Union[str, Any]         =   mock.Mock()
					snake_case__      :int         =   500
					snake_case__      :Any         =   {}
					snake_case__      :Dict         =   HTTPError
					snake_case__      :List[Any]         =   {}

					# Download this model to make sure it's in the cache.
					snake_case__      :Optional[int]         =   GPTaTokenizerFast.from_pretrained("gpt2"       )

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch("requests.Session.request"					,return_value=UpperCamelCase       ) as mock_head:
									snake_case__      :Any         =   GPTaTokenizerFast.from_pretrained("gpt2"       )
									# This check we did call the fake head request
									mock_head.assert_called()






	def    lowerCAmelCase_							(		self       )						->    int:
					# This test is for deprecated behavior and can be removed in v5
					try:
									snake_case__      :Union[str, Any]         =   tempfile.mktemp()
									with open(UpperCamelCase					,"wb"       ) as f:
													http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"					,UpperCamelCase       )

									snake_case__      :Tuple         =   AlbertTokenizer.from_pretrained(UpperCamelCase       )
					finally:
									os.remove(UpperCamelCase       )

					# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
					# the current folder and have the right name.
					if os.path.isfile("tokenizer.json"       ):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
					try:
									with open("tokenizer.json"					,"wb"       ) as f:
													http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"					,UpperCamelCase       )
									snake_case__      :Dict         =   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size					,1_000       )
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

					finally:
									os.remove("tokenizer.json"       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					# This test is for deprecated behavior and can be removed in v5
					snake_case__      :Union[str, Any]         =   AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )




@is_staging_test
class 					_snake_case						(			unittest.TestCase ):
	_A          =   ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






	@classmethod
	def    lowerCAmelCase_							(		cls       )						->    Optional[int]:
					snake_case__      :List[str]         =   TOKEN
					HfFolder.save_token(UpperCamelCase       )






	@classmethod
	def    lowerCAmelCase_							(		cls       )						->    Union[str, Any]:
					try:
									delete_repo(token=cls._token					,repo_id="test-tokenizer"       )
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token					,repo_id="valid_org/test-tokenizer-org"       )
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token					,repo_id="test-dynamic-tokenizer"       )
					except HTTPError:
									pass






	def    lowerCAmelCase_							(		self       )						->    Optional[Any]:
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :List[str]         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :str         =   BertTokenizer(UpperCamelCase       )

					tokenizer.push_to_hub("test-tokenizer"					,use_auth_token=self._token       )
					snake_case__      :Dict         =   BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )

					# Reset repo
					delete_repo(token=self._token					,repo_id="test-tokenizer"       )

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(UpperCamelCase					,repo_id="test-tokenizer"					,push_to_hub=UpperCamelCase					,use_auth_token=self._token       )

					snake_case__      :List[str]         =   BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )






	def    lowerCAmelCase_							(		self       )						->    Optional[int]:
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :List[Any]         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :Any         =   BertTokenizer(UpperCamelCase       )

					tokenizer.push_to_hub("valid_org/test-tokenizer-org"					,use_auth_token=self._token       )
					snake_case__      :Any         =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )

					# Reset repo
					delete_repo(token=self._token					,repo_id="valid_org/test-tokenizer-org"       )

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    UpperCamelCase					,repo_id="valid_org/test-tokenizer-org"					,push_to_hub=UpperCamelCase					,use_auth_token=self._token       )

					snake_case__      :Union[str, Any]         =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
					self.assertDictEqual(new_tokenizer.vocab					,tokenizer.vocab       )






	@require_tokenizers
	def    lowerCAmelCase_							(		self       )						->    Any:
					CustomTokenizer.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :str         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
									snake_case__      :Optional[int]         =   CustomTokenizer(UpperCamelCase       )

					# No fast custom tokenizer
					tokenizer.push_to_hub("test-dynamic-tokenizer"					,use_auth_token=self._token       )

					snake_case__      :Union[str, Any]         =   AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizer"       )

					# Fast and slow custom tokenizer
					CustomTokenizerFast.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__      :int         =   os.path.join(UpperCamelCase					,"vocab.txt"       )
									with open(UpperCamelCase					,"w"					,encoding="utf-8"       ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

									snake_case__      :Tuple         =   BertTokenizerFast.from_pretrained(UpperCamelCase       )
									bert_tokenizer.save_pretrained(UpperCamelCase       )
									snake_case__      :List[Any]         =   CustomTokenizerFast.from_pretrained(UpperCamelCase       )

					tokenizer.push_to_hub("test-dynamic-tokenizer"					,use_auth_token=self._token       )

					snake_case__      :List[Any]         =   AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizerFast"       )
					snake_case__      :List[str]         =   AutoTokenizer.from_pretrained(
					    f'{USER}/test-dynamic-tokenizer'					,use_fast=UpperCamelCase					,trust_remote_code=UpperCamelCase       )
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__					,"CustomTokenizer"       )




class 					_snake_case						(			unittest.TestCase ):






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					snake_case__      :int         =   Trie()
					trie.add("Hello 友達"       )
					self.assertEqual(trie.data					,{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
					trie.add("Hello"       )
					trie.data
					self.assertEqual(trie.data					,{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )






	def    lowerCAmelCase_							(		self       )						->    int:
					snake_case__      :List[str]         =   Trie()
					self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )					,["[CLS] This is a extra_id_100"]       )
					trie.add("[CLS]"       )
					trie.add("extra_id_1"       )
					trie.add("extra_id_100"       )
					self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )					,["[CLS]", " This is a ", "extra_id_100"]       )






	def    lowerCAmelCase_							(		self       )						->    str:
					snake_case__      :Optional[Any]         =   Trie()
					trie.add("A"       )
					self.assertEqual(trie.split("ABC"       )					,["A", "BC"]       )
					self.assertEqual(trie.split("BCA"       )					,["BC", "A"]       )






	def    lowerCAmelCase_							(		self       )						->    Dict:
					snake_case__      :Any         =   Trie()
					trie.add("TOKEN]"       )
					trie.add("[SPECIAL_TOKEN]"       )
					self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )					,["This is something ", "[SPECIAL_TOKEN]"]       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :List[Any]         =   Trie()
					trie.add("A"       )
					trie.add("P"       )
					trie.add("[SPECIAL_TOKEN]"       )
					self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )					,["This is something ", "[SPECIAL_TOKEN]"]       )






	def    lowerCAmelCase_							(		self       )						->    Tuple:
					snake_case__      :str         =   Trie()
					trie.add("AB"       )
					trie.add("B"       )
					trie.add("C"       )
					self.assertEqual(trie.split("ABC"       )					,["AB", "C"]       )






	def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
					snake_case__      :Dict         =   Trie()
					trie.add("ABC"       )
					trie.add("B"       )
					trie.add("CD"       )
					self.assertEqual(trie.split("ABCD"       )					,["ABC", "D"]       )






	def    lowerCAmelCase_							(		self       )						->    int:
					# Even if the offsets are wrong, we necessarily output correct string
					# parts.
					snake_case__      :Optional[int]         =   Trie()
					snake_case__      :Union[str, Any]         =   trie.cut_text("ABC"					,[0, 0, 2, 1, 2, 3]       )
					self.assertEqual(UpperCamelCase					,["AB", "C"]       )