

import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 					_a		(__magic_name__	,     unittest.TestCase       ):


					'''simple docstring'''






					UpperCAmelCase__:     Any													= MobileBertTokenizer
					UpperCAmelCase__:     Any													= MobileBertTokenizerFast
					UpperCAmelCase__:     int													= True
					UpperCAmelCase__:     int													= True
					UpperCAmelCase__:     Dict													= filter_non_english
					UpperCAmelCase__:     Union[str, Any]													= '''google/mobilebert-uncased'''






					def 						__A    (				self    ):
						super().setUp()

						A__							:					Optional[int]       =	[
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """[PAD]""",
						    """[MASK]""",
						    """want""",
						    """##want""",
						    """##ed""",
						    """wa""",
						    """un""",
						    """runn""",
						    """##ing""",
						    """,""",
						    """low""",
						    """lowest""",
						]
						A__							:					List[str]       =	os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES["""vocab_file"""]    )
						with open(self.vocab_file					, """w"""					, encoding="""utf-8"""    ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )

						A__							:					Any       =	[
						    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
						    for tokenizer_def in self.tokenizers_list
						]






					def 						__A    (				self					, A__    ):
						A__							:					int       =	"""UNwant\u00E9d,running"""
						A__							:					List[str]       =	"""unwanted, running"""
						return input_text, output_text






					def 						__A    (				self    ):
						A__							:					Tuple       =	self.tokenizer_class(self.vocab_file    )

						A__							:					str       =	tokenizer.tokenize("""UNwant\u00E9d,running"""    )
						self.assertListEqual(A__					, ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__    )					, [9, 6, 7, 12, 10, 11]    )






					def 						__A    (				self    ):
						if not self.test_rust_tokenizer:
							return

						A__							:					List[str]       =	self.get_tokenizer()
						A__							:					int       =	self.get_rust_tokenizer()

						A__							:					Optional[Any]       =	"""UNwant\u00E9d,running"""

						A__							:					Optional[Any]       =	tokenizer.tokenize(A__    )
						A__							:					str       =	rust_tokenizer.tokenize(A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					List[str]       =	tokenizer.encode(A__					, add_special_tokens=A__    )
						A__							:					Dict       =	rust_tokenizer.encode(A__					, add_special_tokens=A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					Optional[Any]       =	self.get_rust_tokenizer()
						A__							:					Optional[Any]       =	tokenizer.encode(A__    )
						A__							:					List[str]       =	rust_tokenizer.encode(A__    )
						self.assertListEqual(A__					, A__    )

						# With lower casing
						A__							:					List[str]       =	self.get_tokenizer(do_lower_case=A__    )
						A__							:					List[str]       =	self.get_rust_tokenizer(do_lower_case=A__    )

						A__							:					Dict       =	"""UNwant\u00E9d,running"""

						A__							:					str       =	tokenizer.tokenize(A__    )
						A__							:					int       =	rust_tokenizer.tokenize(A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					str       =	tokenizer.encode(A__					, add_special_tokens=A__    )
						A__							:					int       =	rust_tokenizer.encode(A__					, add_special_tokens=A__    )
						self.assertListEqual(A__					, A__    )

						A__							:					List[Any]       =	self.get_rust_tokenizer()
						A__							:					Optional[Any]       =	tokenizer.encode(A__    )
						A__							:					Optional[int]       =	rust_tokenizer.encode(A__    )
						self.assertListEqual(A__					, A__    )






					def 						__A    (				self    ):
						A__							:					List[str]       =	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )					, ["""ah""", """\u535A""", """\u63A8""", """zz"""]    )






					def 						__A    (				self    ):
						A__							:					str       =	BasicTokenizer(do_lower_case=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )					, ["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""hello"""]    )






					def 						__A    (				self    ):
						A__							:					Optional[int]       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""h\u00E9llo"""]    )






					def 						__A    (				self    ):
						A__							:					Dict       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""hello"""]    )






					def 						__A    (				self    ):
						A__							:					Tuple       =	BasicTokenizer(do_lower_case=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					, ["""hello"""]    )






					def 						__A    (				self    ):
						A__							:					Dict       =	BasicTokenizer(do_lower_case=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )					, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )






					def 						__A    (				self    ):
						A__							:					Tuple       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )






					def 						__A    (				self    ):
						A__							:					Optional[Any]       =	BasicTokenizer(do_lower_case=A__					, strip_accents=A__    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					, ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )






					def 						__A    (				self    ):
						A__							:					Union[str, Any]       =	BasicTokenizer(do_lower_case=A__					, never_split=["""[UNK]"""]    )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )					, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )






					def 						__A    (				self    ):
						A__							:					List[str]       =	["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						A__							:					Tuple       =	{}
						for i, token in enumerate(A__    ):
							A__							:					int       =	i
						A__							:					Dict       =	WordpieceTokenizer(vocab=A__					, unk_token="""[UNK]"""    )

						self.assertListEqual(tokenizer.tokenize(""""""    )					, []    )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )					, ["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )					, ["""[UNK]""", """runn""", """##ing"""]    )






					def 						__A    (				self    ):
						self.assertTrue(_is_whitespace(""" """    )    )
						self.assertTrue(_is_whitespace("""\t"""    )    )
						self.assertTrue(_is_whitespace("""\r"""    )    )
						self.assertTrue(_is_whitespace("""\n"""    )    )
						self.assertTrue(_is_whitespace("""\u00A0"""    )    )

						self.assertFalse(_is_whitespace("""A"""    )    )
						self.assertFalse(_is_whitespace("""-"""    )    )






					def 						__A    (				self    ):
						self.assertTrue(_is_control("""\u0005"""    )    )

						self.assertFalse(_is_control("""A"""    )    )
						self.assertFalse(_is_control(""" """    )    )
						self.assertFalse(_is_control("""\t"""    )    )
						self.assertFalse(_is_control("""\r"""    )    )






					def 						__A    (				self    ):
						self.assertTrue(_is_punctuation("""-"""    )    )
						self.assertTrue(_is_punctuation("""$"""    )    )
						self.assertTrue(_is_punctuation("""`"""    )    )
						self.assertTrue(_is_punctuation("""."""    )    )

						self.assertFalse(_is_punctuation("""A"""    )    )
						self.assertFalse(_is_punctuation(""" """    )    )






					def 						__A    (				self    ):
						A__							:					int       =	self.get_tokenizer()
						A__							:					int       =	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__    ) for t in ["""Test""", """\xad""", """test"""]]					, [["""[UNK]"""], [], ["""[UNK]"""]]    )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(A__    ) for t in ["""Test""", """\xad""", """test"""]]					, [["""[UNK]"""], [], ["""[UNK]"""]]    )






					@slow
					def 						__A    (				self    ):
						A__							:					int       =	self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""    )

						A__							:					List[Any]       =	tokenizer.encode("""sequence builders"""					, add_special_tokens=A__    )
						A__							:					Optional[Any]       =	tokenizer.encode("""multi-sequence build"""					, add_special_tokens=A__    )

						A__							:					List[Any]       =	tokenizer.build_inputs_with_special_tokens(A__    )
						A__							:					Dict       =	tokenizer.build_inputs_with_special_tokens(A__					, A__    )

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]






					def 						__A    (				self    ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
								A__							:					int       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					int       =	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
								A__							:					str       =	tokenizer_r.encode_plus(
								    A__					, return_attention_mask=A__					, return_token_type_ids=A__					, return_offsets_mapping=A__					, add_special_tokens=A__					, )

								A__							:					Optional[int]       =	tokenizer_r.do_lower_case if hasattr(A__					, """do_lower_case"""    ) else False
								A__							:					Optional[Any]       =	(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """A"""),
								        ((1, 2), ""","""),
								        ((3, 5), """na"""),
								        ((5, 6), """##ï"""),
								        ((6, 8), """##ve"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """Allen"""),
								        ((21, 23), """##NL"""),
								        ((23, 24), """##P"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """a"""),
								        ((1, 2), ""","""),
								        ((3, 8), """naive"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """allen"""),
								        ((21, 23), """##nl"""),
								        ((23, 24), """##p"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]					, tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
								self.assertEqual([e[0] for e in expected_results]					, tokens["""offset_mapping"""]    )






					def 						__A    (				self    ):
						A__							:					Any       =	["""的""", """人""", """有"""]
						A__							:					Union[str, Any]       =	"""""".join(A__    )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
								A__							:					Tuple       =	True
								A__							:					List[Any]       =	self.tokenizer_class.from_pretrained(A__					, **A__    )
								A__							:					Optional[int]       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					Dict       =	tokenizer_p.encode(A__					, add_special_tokens=A__    )
								A__							:					str       =	tokenizer_r.encode(A__					, add_special_tokens=A__    )

								A__							:					int       =	tokenizer_r.convert_ids_to_tokens(A__    )
								A__							:					int       =	tokenizer_p.convert_ids_to_tokens(A__    )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(A__					, A__    )
								self.assertListEqual(A__					, A__    )

								A__							:					str       =	False
								A__							:					List[Any]       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )
								A__							:					Optional[Any]       =	self.tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					Union[str, Any]       =	tokenizer_r.encode(A__					, add_special_tokens=A__    )
								A__							:					Union[str, Any]       =	tokenizer_p.encode(A__					, add_special_tokens=A__    )

								A__							:					Optional[Any]       =	tokenizer_r.convert_ids_to_tokens(A__    )
								A__							:					Any       =	tokenizer_p.convert_ids_to_tokens(A__    )

								# it is expected that only the first Chinese character is not preceded by "##".
								A__							:					Optional[int]       =	[
								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__    )
								]
								self.assertListEqual(A__					, A__    )
								self.assertListEqual(A__					, A__    )






import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


A_ :				Dict						 =    {
    'tiny.en': 'https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt',
    'tiny': 'https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt',
    'base.en': 'https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt',
    'base': 'https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt',
    'small.en': 'https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt',
    'small': 'https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt',
    'medium.en': 'https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt',
    'medium': 'https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt',
    'large': 'https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt',
    'large-v2': 'https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt',
}

def 		UpperCamelCase      (lowercase_:				Optional[Any]      )  ->   Optional[int]:
	A__							:					List[Any]       =	["""layers""", """blocks"""]
	for k in ignore_keys:
		state_dict.pop(lowercase_					,   lowercase_      )


A_ :				Any						 =    {
    'blocks': 'layers',
    'mlp.0': 'fc1',
    'mlp.2': 'fc2',
    'mlp_ln': 'final_layer_norm',
    '.attn.query': '.self_attn.q_proj',
    '.attn.key': '.self_attn.k_proj',
    '.attn.value': '.self_attn.v_proj',
    '.attn_ln': '.self_attn_layer_norm',
    '.attn.out': '.self_attn.out_proj',
    '.cross_attn.query': '.encoder_attn.q_proj',
    '.cross_attn.key': '.encoder_attn.k_proj',
    '.cross_attn.value': '.encoder_attn.v_proj',
    '.cross_attn_ln': '.encoder_attn_layer_norm',
    '.cross_attn.out': '.encoder_attn.out_proj',
    'decoder.ln.': 'decoder.layer_norm.',
    'encoder.ln.': 'encoder.layer_norm.',
    'token_embedding': 'embed_tokens',
    'encoder.positional_embedding': 'encoder.embed_positions.weight',
    'decoder.positional_embedding': 'decoder.embed_positions.weight',
    'ln_post': 'layer_norm',
}

def 		UpperCamelCase      (lowercase_:				str      )  ->   Any:
	A__							:					Dict       =	list(s_dict.keys()      )
	for key in keys:
		A__							:					List[str]       =	key
		for k, v in WHISPER_MAPPING.items():
			if k in key:
				A__							:					List[Any]       =	new_key.replace(lowercase_					,   lowercase_      )

		print(f"""{key} -> {new_key}"""      )

		A__							:					Tuple       =	s_dict.pop(lowercase_      )
	return s_dict

def 		UpperCamelCase      (lowercase_:				Tuple      )  ->   Optional[int]:
	A__     ,     A__							:					Any       =	emb.weight.shape
	A__							:					str       =	nn.Linear(lowercase_					,   lowercase_					,   bias=lowercase_      )
	A__							:					Union[str, Any]       =	emb.weight.data
	return lin_layer

def 		UpperCamelCase      (lowercase_:				str					,   lowercase_:				str      )  ->   bytes:
	os.makedirs(lowercase_					,   exist_ok=lowercase_      )
	A__							:					Tuple       =	os.path.basename(lowercase_      )

	A__							:					int       =	url.split("""/"""      )[-2]
	A__							:					Dict       =	os.path.join(lowercase_					,   lowercase_      )

	if os.path.exists(lowercase_      ) and not os.path.isfile(lowercase_      ):
		raise RuntimeError(f"""{download_target} exists and is not a regular file"""      )

	if os.path.isfile(lowercase_      ):
		A__							:					Optional[Any]       =	open(lowercase_					,   """rb"""      ).read()
		if hashlib.shaaaa(lowercase_      ).hexdigest() == expected_shaaaa:
			return model_bytes
		else:
			warnings.warn(f"""{download_target} exists, but the SHA256 checksum does not match; re-downloading the file"""      )

	with urllib.request.urlopen(lowercase_      ) as source, open(lowercase_					,   """wb"""      ) as output:
		with tqdm(
		    total=int(source.info().get("""Content-Length"""      )      )					,   ncols=80					,   unit="""iB"""					,   unit_scale=lowercase_					,   unit_divisor=1024      ) as loop:
			while True:
				A__							:					Any       =	source.read(8192      )
				if not buffer:
					break

				output.write(lowercase_      )
				loop.update(len(lowercase_      )      )

	A__							:					Dict       =	open(lowercase_					,   """rb"""      ).read()
	if hashlib.shaaaa(lowercase_      ).hexdigest() != expected_shaaaa:
		raise RuntimeError(
		    """Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model."""      )

	return model_bytes

def 		UpperCamelCase      (lowercase_:				List[Any]					,   lowercase_:				Tuple      )  ->   Optional[Any]:
	if ".pt" not in checkpoint_path:
		A__							:					Tuple       =	_download(_MODELS[checkpoint_path]      )
	else:
		A__							:					Optional[int]       =	torch.load(lowercase_					,   map_location="""cpu"""      )
	A__							:					str       =	original_checkpoint["""dims"""]
	A__							:					List[Any]       =	original_checkpoint["""model_state_dict"""]
	A__							:					Optional[Any]       =	state_dict["""decoder.token_embedding.weight"""]
	remove_ignore_keys_(lowercase_      )
	rename_keys(lowercase_      )
	A__							:					List[str]       =	True
	A__							:					Optional[Any]       =	state_dict["""decoder.layers.0.fc1.weight"""].shape[0]

	A__							:					List[Any]       =	WhisperConfig(
	    vocab_size=dimensions["""n_vocab"""]					,   encoder_ffn_dim=lowercase_					,   decoder_ffn_dim=lowercase_					,   num_mel_bins=dimensions["""n_mels"""]					,   d_model=dimensions["""n_audio_state"""]					,   max_target_positions=dimensions["""n_text_ctx"""]					,   encoder_layers=dimensions["""n_audio_layer"""]					,   encoder_attention_heads=dimensions["""n_audio_head"""]					,   decoder_layers=dimensions["""n_text_layer"""]					,   decoder_attention_heads=dimensions["""n_text_state"""]					,   max_source_positions=dimensions["""n_audio_ctx"""]					,   )

	A__							:					Optional[Any]       =	WhisperForConditionalGeneration(lowercase_      )
	A__     ,     A__							:					List[Any]       =	model.model.load_state_dict(lowercase_					,   strict=lowercase_      )
	if len(lowercase_      ) > 0 and not set(lowercase_      ) <= {
	    "encoder.embed_positions.weights",
	    "decoder.embed_positions.weights",
	}:
		raise ValueError(
		    """Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,"""
		    f""" but all the following weights are missing {missing}"""      )

	if tie_embeds:
		A__							:					Any       =	make_linear_from_emb(model.model.decoder.embed_tokens      )
	else:
		A__							:					str       =	proj_out_weights

	model.save_pretrained(lowercase_      )


if __name__ == "__main__":
	A_ :				Any						 =    argparse.ArgumentParser()
	# # Required parameters
	parser.add_argument('--checkpoint_path', type=str, help='Patht to the downloaded checkpoints')
	parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
	A_ :				Tuple						 =    parser.parse_args()

	convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)



