






"""simple docstring"""

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast

class 						A_							(      unittest.TestCase  ):







				def     _snake_case					(							self							:						Any	)				->					Union[str, Any]:
										__magic_name__            =     0







				@slow
				def     _snake_case					(							self							:						Any	)				->					Union[str, Any]:
										for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
																self.assertIsNotNone(__lowerCamelCase	)
																self.assertIsInstance(__lowerCamelCase						,    (BertTokenizer, BertTokenizerFast)	)
																self.assertGreater(len(__lowerCamelCase	)						,    0	)

										for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
																self.assertIsNotNone(__lowerCamelCase	)
																self.assertIsInstance(__lowerCamelCase						,    (GPTaTokenizer, GPTaTokenizerFast)	)
																self.assertGreater(len(__lowerCamelCase	)						,    0	)







				def     _snake_case					(							self							:						List[Any]	)				->					List[str]:
										__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
										self.assertIsInstance(__lowerCamelCase						,    (BertTokenizer, BertTokenizerFast)	)
										self.assertEqual(tokenizer.vocab_size						,    1_2	)







				def     _snake_case					(							self							:						Any	)				->					Optional[int]:
										__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
										self.assertIsInstance(__lowerCamelCase						,    (RobertaTokenizer, RobertaTokenizerFast)	)
										self.assertEqual(tokenizer.vocab_size						,    2_0	)







				def     _snake_case					(							self							:						int	)				->					List[str]:
										__magic_name__            =     AutoConfig.from_pretrained(__lowerCamelCase	)
										self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)
										# Check that tokenizer_type ≠ model_type
										__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    config=__lowerCamelCase	)
										self.assertIsInstance(__lowerCamelCase						,    (BertTokenizer, BertTokenizerFast)	)
										self.assertEqual(tokenizer.vocab_size						,    1_2	)







				def     _snake_case					(							self							:						Any	)				->					List[str]:
										with tempfile.TemporaryDirectory() as tmp_dir:
																shutil.copy("./tests/fixtures/vocab.txt"						,    os.path.join(__lowerCamelCase						,    "vocab.txt"	)	)

																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    tokenizer_type="bert"						,    use_fast=__lowerCamelCase	)
																self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)

										with tempfile.TemporaryDirectory() as tmp_dir:
																shutil.copy("./tests/fixtures/vocab.json"						,    os.path.join(__lowerCamelCase						,    "vocab.json"	)	)
																shutil.copy("./tests/fixtures/merges.txt"						,    os.path.join(__lowerCamelCase						,    "merges.txt"	)	)

																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    tokenizer_type="gpt2"						,    use_fast=__lowerCamelCase	)
																self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)







				@require_tokenizers
				def     _snake_case					(							self							:						List[Any]	)				->					Optional[int]:
										with tempfile.TemporaryDirectory() as tmp_dir:
																shutil.copy("./tests/fixtures/vocab.txt"						,    os.path.join(__lowerCamelCase						,    "vocab.txt"	)	)

																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    tokenizer_type="bert"	)
																self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)

										with tempfile.TemporaryDirectory() as tmp_dir:
																shutil.copy("./tests/fixtures/vocab.json"						,    os.path.join(__lowerCamelCase						,    "vocab.json"	)	)
																shutil.copy("./tests/fixtures/merges.txt"						,    os.path.join(__lowerCamelCase						,    "merges.txt"	)	)

																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    tokenizer_type="gpt2"	)
																self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)







				def     _snake_case					(							self							:						Tuple	)				->					Optional[int]:
										with pytest.raises(__lowerCamelCase	):
																AutoTokenizer.from_pretrained("./"						,    tokenizer_type="xxx"	)







				@require_tokenizers
				def     _snake_case					(							self							:						Any	)				->					Tuple:
										for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
																__magic_name__            =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"	)
																self.assertIsInstance(__lowerCamelCase						,    (BertTokenizer, BertTokenizerFast)	)

																if isinstance(__lowerCamelCase						,    __lowerCamelCase	):
																						self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,    __lowerCamelCase	)
																else:
																						self.assertEqual(tokenizer.do_lower_case						,    __lowerCamelCase	)

																self.assertEqual(tokenizer.model_max_length						,    5_1_2	)







				@require_tokenizers
				def     _snake_case					(							self							:						Optional[Any]	)				->					Union[str, Any]:
										for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
																with self.assertRaisesRegex(
																    __lowerCamelCase						,    "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"						,    ):
																						__magic_name__            =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"	)







				def     _snake_case					(							self							:						Optional[int]	)				->					List[Any]:
										# tests: https://github.com/huggingface/transformers/pull/13251
										# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
										# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
										__magic_name__            =     TOKENIZER_MAPPING.values()
										__magic_name__            =     []

										for slow_tok, fast_tok in tokenizers:
																if slow_tok is not None:
																						tokenizer_names.append(slow_tok.__name__	)

																if fast_tok is not None:
																						tokenizer_names.append(fast_tok.__name__	)

										for tokenizer_name in tokenizer_names:
																# must find the right class
																tokenizer_class_from_name(__lowerCamelCase	)







				@require_tokenizers
				def     _snake_case					(							self							:						Tuple	)				->					Dict:
										self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						,    use_fast=__lowerCamelCase	)						,    __lowerCamelCase	)
										self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"	)						,    __lowerCamelCase	)







				@require_tokenizers
				def     _snake_case					(							self							:						List[str]	)				->					Dict:
										__magic_name__            =     AutoTokenizer.from_pretrained("distilbert-base-uncased"						,    do_lower_case=__lowerCamelCase	)
										__magic_name__            =     "Hello, world. How are you?"
										__magic_name__            =     tokenizer.tokenize(__lowerCamelCase	)
										self.assertEqual("[UNK]"						,    tokens[0]	)

										__magic_name__            =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"						,    do_lower_case=__lowerCamelCase	)
										__magic_name__            =     tokenizer.tokenize(__lowerCamelCase	)
										self.assertEqual("[UNK]"						,    tokens[0]	)







				@require_tokenizers
				def     _snake_case					(							self							:						Dict	)				->					Union[str, Any]:
										__magic_name__            =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"	)
										self.assertEqual(type(__lowerCamelCase	)						,    __lowerCamelCase	)
										self.assertEqual(tokenizer.model_max_length						,    5_1_2	)
										self.assertEqual(tokenizer.vocab_size						,    3_0_0_0_0	)
										self.assertEqual(tokenizer.unk_token						,    "[UNK]"	)
										self.assertEqual(tokenizer.padding_side						,    "right"	)
										self.assertEqual(tokenizer.truncation_side						,    "right"	)







				def     _snake_case					(							self							:						int	)				->					Dict:
										__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
										self.assertIsInstance(__lowerCamelCase						,    (BertTokenizer, BertTokenizerFast)	)
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(__lowerCamelCase	)
																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)

										self.assertIsInstance(__lowerCamelCase						,    tokenizer.__class__	)
										self.assertEqual(tokenizera.vocab_size						,    1_2	)







				def     _snake_case					(							self							:						Union[str, Any]	)				->					Optional[Any]:
										__magic_name__            =     AutoTokenizer.from_pretrained("ctrl"	)
										# There is no fast CTRL so this always gives us a slow tokenizer.
										self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)







				def     _snake_case					(							self							:						Optional[int]	)				->					Union[str, Any]:
										# Check we can load the tokenizer config of an online model.
										__magic_name__            =     get_tokenizer_config("bert-base-cased"	)
										__magic_name__            =     config.pop("_commit_hash"						,    __lowerCamelCase	)
										# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
										self.assertEqual(__lowerCamelCase						,    {"do_lower_case": False}	)

										# This model does not have a tokenizer_config so we get back an empty dict.
										__magic_name__            =     get_tokenizer_config(__lowerCamelCase	)
										self.assertDictEqual(__lowerCamelCase						,    {}	)

										# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
										__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(__lowerCamelCase	)
																__magic_name__            =     get_tokenizer_config(__lowerCamelCase	)

										# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
										self.assertEqual(config["tokenizer_class"]						,    "BertTokenizer"	)







				def     _snake_case					(							self							:						str	)				->					List[str]:
										try:
																AutoConfig.register("custom"						,    __lowerCamelCase	)

																AutoTokenizer.register(__lowerCamelCase						,    slow_tokenizer_class=__lowerCamelCase	)
																# Trying to register something existing in the Transformers library will raise an error
																with self.assertRaises(__lowerCamelCase	):
																						AutoTokenizer.register(__lowerCamelCase						,    slow_tokenizer_class=__lowerCamelCase	)

																__magic_name__            =     CustomTokenizer.from_pretrained(__lowerCamelCase	)
																with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer.save_pretrained(__lowerCamelCase	)

																						__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
																						self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)

										finally:
																if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																if CustomConfig in TOKENIZER_MAPPING._extra_content:
																						del TOKENIZER_MAPPING._extra_content[CustomConfig]







				@require_tokenizers
				def     _snake_case					(							self							:						str	)				->					Tuple:
										try:
																AutoConfig.register("custom"						,    __lowerCamelCase	)

																# Can register in two steps
																AutoTokenizer.register(__lowerCamelCase						,    slow_tokenizer_class=__lowerCamelCase	)
																self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,    (CustomTokenizer, None)	)
																AutoTokenizer.register(__lowerCamelCase						,    fast_tokenizer_class=__lowerCamelCase	)
																self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,    (CustomTokenizer, CustomTokenizerFast)	)

																del TOKENIZER_MAPPING._extra_content[CustomConfig]
																# Can register in one step
																AutoTokenizer.register(
																    __lowerCamelCase						,    slow_tokenizer_class=__lowerCamelCase						,    fast_tokenizer_class=__lowerCamelCase	)
																self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,    (CustomTokenizer, CustomTokenizerFast)	)

																# Trying to register something existing in the Transformers library will raise an error
																with self.assertRaises(__lowerCamelCase	):
																						AutoTokenizer.register(__lowerCamelCase						,    fast_tokenizer_class=__lowerCamelCase	)

																# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
																# and that model does not have a tokenizer.json
																with tempfile.TemporaryDirectory() as tmp_dir:
																						__magic_name__            =     BertTokenizerFast.from_pretrained(__lowerCamelCase	)
																						bert_tokenizer.save_pretrained(__lowerCamelCase	)
																						__magic_name__            =     CustomTokenizerFast.from_pretrained(__lowerCamelCase	)

																with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer.save_pretrained(__lowerCamelCase	)

																						__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase	)
																						self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)

																						__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    use_fast=__lowerCamelCase	)
																						self.assertIsInstance(__lowerCamelCase						,    __lowerCamelCase	)

										finally:
																if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																if CustomConfig in TOKENIZER_MAPPING._extra_content:
																						del TOKENIZER_MAPPING._extra_content[CustomConfig]







				def     _snake_case					(							self							:						List[Any]	)				->					List[str]:
										# If remote code is not set, we will time out when asking whether to load the model.
										with self.assertRaises(__lowerCamelCase	):
																__magic_name__            =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"	)
										# If remote code is disabled, we can't load this config.
										with self.assertRaises(__lowerCamelCase	):
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase	)

										__magic_name__            =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase	)
										self.assertTrue(tokenizer.special_attribute_present	)
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(__lowerCamelCase	)
																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    trust_remote_code=__lowerCamelCase	)
										self.assertTrue(reloaded_tokenizer.special_attribute_present	)

										if is_tokenizers_available():
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizerFast"	)
																self.assertEqual(reloaded_tokenizer.__class__.__name__						,    "NewTokenizerFast"	)

																# Test we can also load the slow version
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase						,    use_fast=__lowerCamelCase	)
																self.assertTrue(tokenizer.special_attribute_present	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)
																# Test tokenizer can be reloaded.
																with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer.save_pretrained(__lowerCamelCase	)
																						__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    trust_remote_code=__lowerCamelCase						,    use_fast=__lowerCamelCase	)
																self.assertEqual(reloaded_tokenizer.__class__.__name__						,    "NewTokenizer"	)
																self.assertTrue(reloaded_tokenizer.special_attribute_present	)
										else:
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)
																self.assertEqual(reloaded_tokenizer.__class__.__name__						,    "NewTokenizer"	)







				@require_tokenizers
				def     _snake_case					(							self							:						Tuple	)				->					Optional[int]:

										class 						A_							(      snake_case_  ):
														UpperCAmelCase__        =				False





										class 						A_							(      snake_case_  ):
														UpperCAmelCase__        =				NewTokenizer
														UpperCAmelCase__        =				False

										try:
																AutoConfig.register("custom"						,    __lowerCamelCase	)
																AutoTokenizer.register(__lowerCamelCase						,    slow_tokenizer_class=__lowerCamelCase	)
																AutoTokenizer.register(__lowerCamelCase						,    fast_tokenizer_class=__lowerCamelCase	)
																# If remote code is not set, the default is to use local
																__magic_name__            =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizerFast"	)
																self.assertFalse(tokenizer.special_attribute_present	)
																__magic_name__            =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						,    use_fast=__lowerCamelCase	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)
																self.assertFalse(tokenizer.special_attribute_present	)

																# If remote code is disabled, we load the local one.
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizerFast"	)
																self.assertFalse(tokenizer.special_attribute_present	)
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase						,    use_fast=__lowerCamelCase	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)
																self.assertFalse(tokenizer.special_attribute_present	)

																# If remote is enabled, we load from the Hub
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizerFast"	)
																self.assertTrue(tokenizer.special_attribute_present	)
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer"						,    trust_remote_code=__lowerCamelCase						,    use_fast=__lowerCamelCase	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)
																self.assertTrue(tokenizer.special_attribute_present	)

										finally:
																if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																if CustomConfig in TOKENIZER_MAPPING._extra_content:
																						del TOKENIZER_MAPPING._extra_content[CustomConfig]







				def     _snake_case					(							self							:						Dict	)				->					Any:
										__magic_name__            =     AutoTokenizer.from_pretrained(
										    "hf-internal-testing/test_dynamic_tokenizer_legacy"						,    trust_remote_code=__lowerCamelCase	)
										self.assertTrue(tokenizer.special_attribute_present	)
										if is_tokenizers_available():
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizerFast"	)

																# Test we can also load the slow version
																__magic_name__            =     AutoTokenizer.from_pretrained(
																    "hf-internal-testing/test_dynamic_tokenizer_legacy"						,    trust_remote_code=__lowerCamelCase						,    use_fast=__lowerCamelCase	)
																self.assertTrue(tokenizer.special_attribute_present	)
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)
										else:
																self.assertEqual(tokenizer.__class__.__name__						,    "NewTokenizer"	)







				def     _snake_case					(							self							:						List[str]	)				->					Union[str, Any]:
										with self.assertRaisesRegex(
										    __lowerCamelCase						,    "bert-base is not a local folder and is not a valid model identifier"	):
																__magic_name__            =     AutoTokenizer.from_pretrained("bert-base"	)







				def     _snake_case					(							self							:						str	)				->					List[str]:
										with self.assertRaisesRegex(
										    __lowerCamelCase						,    R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"	):
																__magic_name__            =     AutoTokenizer.from_pretrained(__lowerCamelCase						,    revision="aaaaaa"	)







				def     _snake_case					(							self							:						Optional[Any]	)				->					str:
										# Make sure we have cached the tokenizer.
										__magic_name__            =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"	)
										with RequestCounter() as counter:
																__magic_name__            =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"	)
																self.assertEqual(counter.get_request_count						,    0	)
																self.assertEqual(counter.head_request_count						,    1	)
																self.assertEqual(counter.other_request_count						,    0	)











"""simple docstring"""

import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse('''0.8.3'''):
			raise Exception('''requires gluonnlp == 0.8.3''')

if version.parse(mx.__version__) != version.parse('''1.5.0'''):
			raise Exception('''requires mxnet == 1.5.0''')

logging.set_verbosity_info()
lowercase    			=       logging.get_logger(__name__)

lowercase    			=       '''The Nymphenburg Palace is a beautiful palace in Munich!'''






def _lowerCAmelCase	(  __lowerCamelCase:str ,						__lowerCamelCase:str   ):

						'''simple docstring'''






						__magic_name__            =     {
						    "attention_cell": "multi_head",
						    "num_layers": 4,
						    "units": 1_0_2_4,
						    "hidden_size": 7_6_8,
						    "max_length": 5_1_2,
						    "num_heads": 8,
						    "scaled": True,
						    "dropout": 0.1,
						    "use_residual": True,
						    "embed_size": 1_0_2_4,
						    "embed_dropout": 0.1,
						    "word_embed": None,
						    "layer_norm_eps": 1E-5,
						    "token_type_vocab_size": 2,
						}

						__magic_name__            =     bort_4_8_768_1024_hparams

						# Let's construct the original Bort model here
						# Taken from official BERT implementation, see:
						# https://github.com/alexa/bort/blob/master/bort/bort.py
						__magic_name__            =     BERTEncoder(
						    attention_cell=predefined_args["attention_cell"] ,						num_layers=predefined_args["num_layers"] ,						units=predefined_args["units"] ,						hidden_size=predefined_args["hidden_size"] ,						max_length=predefined_args["max_length"] ,						num_heads=predefined_args["num_heads"] ,						scaled=predefined_args["scaled"] ,						dropout=predefined_args["dropout"] ,						output_attention=__lowerCamelCase ,						output_all_encodings=__lowerCamelCase ,						use_residual=predefined_args["use_residual"] ,						activation=predefined_args.get("activation" ,						"gelu"   ) ,						layer_norm_eps=predefined_args.get("layer_norm_eps" ,						__lowerCamelCase   ) ,						)

						# Vocab information needs to be fetched first
						# It's the same as RoBERTa, so RobertaTokenizer can be used later
						__magic_name__            =     "openwebtext_ccnews_stories_books_cased"

						# Specify download folder to Gluonnlp's vocab
						__magic_name__            =     os.path.join(get_home_dir() ,						"models"   )
						__magic_name__            =     _load_vocab(__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase ,						cls=__lowerCamelCase   )

						__magic_name__            =     nlp.model.BERTModel(
						    __lowerCamelCase ,						len(__lowerCamelCase   ) ,						units=predefined_args["units"] ,						embed_size=predefined_args["embed_size"] ,						embed_dropout=predefined_args["embed_dropout"] ,						word_embed=predefined_args["word_embed"] ,						use_pooler=__lowerCamelCase ,						use_token_type_embed=__lowerCamelCase ,						token_type_vocab_size=predefined_args["token_type_vocab_size"] ,						use_classifier=__lowerCamelCase ,						use_decoder=__lowerCamelCase ,						)

						original_bort.load_parameters(__lowerCamelCase ,						cast_dtype=__lowerCamelCase ,						ignore_extra=__lowerCamelCase   )
						__magic_name__            =     original_bort._collect_params_with_prefix()

						# Build our config 🤗
						__magic_name__            =     {
						    "architectures": ["BertForMaskedLM"],
						    "attention_probs_dropout_prob": predefined_args["dropout"],
						    "hidden_act": "gelu",
						    "hidden_dropout_prob": predefined_args["dropout"],
						    "hidden_size": predefined_args["embed_size"],
						    "initializer_range": 0.0_2,
						    "intermediate_size": predefined_args["hidden_size"],
						    "layer_norm_eps": predefined_args["layer_norm_eps"],
						    "max_position_embeddings": predefined_args["max_length"],
						    "model_type": "bort",
						    "num_attention_heads": predefined_args["num_heads"],
						    "num_hidden_layers": predefined_args["num_layers"],
						    "pad_token_id": 1,  # 2 = BERT, 1 = RoBERTa
						    "type_vocab_size": 1,  # 2 = BERT, 1 = RoBERTa
						    "vocab_size": len(__lowerCamelCase   ),
						}

						__magic_name__            =     BertConfig.from_dict(__lowerCamelCase   )
						__magic_name__            =     BertForMaskedLM(__lowerCamelCase   )
						hf_bort_model.eval()

						# Parameter mapping table (Gluonnlp to Transformers)
						# * denotes layer index
						#
						# | Gluon Parameter                                                | Transformers Parameter
						# | -------------------------------------------------------------- | ----------------------
						# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
						# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
						# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
						# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
						# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
						# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
						# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
						# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
						# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
						# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
						# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
						# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
						# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
						# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

						# Helper function to convert MXNET Arrays to PyTorch
						def to_torch(__lowerCamelCase:Tuple   ) -> nn.Parameter:
												return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()   )   )

						# Check param shapes and map new HF param back
						def check_and_map_params(__lowerCamelCase:str ,						__lowerCamelCase:Union[str, Any]   ):
												__magic_name__            =     hf_param.shape

												__magic_name__            =     to_torch(params[gluon_param]   )
												__magic_name__            =     gluon_param.shape

												assert (
												    shape_hf == shape_gluon
												), f'''The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers'''

												return gluon_param

						__magic_name__            =     check_and_map_params(
						    hf_bort_model.bert.embeddings.word_embeddings.weight ,						"word_embed.0.weight"   )
						__magic_name__            =     check_and_map_params(
						    hf_bort_model.bert.embeddings.position_embeddings.weight ,						"encoder.position_weight"   )
						__magic_name__            =     check_and_map_params(
						    hf_bort_model.bert.embeddings.LayerNorm.bias ,						"encoder.layer_norm.beta"   )
						__magic_name__            =     check_and_map_params(
						    hf_bort_model.bert.embeddings.LayerNorm.weight ,						"encoder.layer_norm.gamma"   )

						# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
						__magic_name__            =     torch.zeros_like(
						    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data   )

						for i in range(hf_bort_config.num_hidden_layers   ):
												__magic_name__            =     hf_bort_model.bert.encoder.layer[i]

												# self attention
												__magic_name__            =     layer.attention.self

												__magic_name__            =     check_and_map_params(
												    self_attn.key.bias.data ,						f'''encoder.transformer_cells.{i}.attention_cell.proj_key.bias'''   )

												__magic_name__            =     check_and_map_params(
												    self_attn.key.weight.data ,						f'''encoder.transformer_cells.{i}.attention_cell.proj_key.weight'''   )
												__magic_name__            =     check_and_map_params(
												    self_attn.query.bias.data ,						f'''encoder.transformer_cells.{i}.attention_cell.proj_query.bias'''   )
												__magic_name__            =     check_and_map_params(
												    self_attn.query.weight.data ,						f'''encoder.transformer_cells.{i}.attention_cell.proj_query.weight'''   )
												__magic_name__            =     check_and_map_params(
												    self_attn.value.bias.data ,						f'''encoder.transformer_cells.{i}.attention_cell.proj_value.bias'''   )
												__magic_name__            =     check_and_map_params(
												    self_attn.value.weight.data ,						f'''encoder.transformer_cells.{i}.attention_cell.proj_value.weight'''   )

												# self attention output
												__magic_name__            =     layer.attention.output

												__magic_name__            =     check_and_map_params(
												    self_output.dense.bias ,						f'''encoder.transformer_cells.{i}.proj.bias'''   )
												__magic_name__            =     check_and_map_params(
												    self_output.dense.weight ,						f'''encoder.transformer_cells.{i}.proj.weight'''   )
												__magic_name__            =     check_and_map_params(
												    self_output.LayerNorm.bias ,						f'''encoder.transformer_cells.{i}.layer_norm.beta'''   )
												__magic_name__            =     check_and_map_params(
												    self_output.LayerNorm.weight ,						f'''encoder.transformer_cells.{i}.layer_norm.gamma'''   )

												# intermediate
												__magic_name__            =     layer.intermediate

												__magic_name__            =     check_and_map_params(
												    intermediate.dense.bias ,						f'''encoder.transformer_cells.{i}.ffn.ffn_1.bias'''   )
												__magic_name__            =     check_and_map_params(
												    intermediate.dense.weight ,						f'''encoder.transformer_cells.{i}.ffn.ffn_1.weight'''   )

												# output
												__magic_name__            =     layer.output

												__magic_name__            =     check_and_map_params(
												    bert_output.dense.bias ,						f'''encoder.transformer_cells.{i}.ffn.ffn_2.bias'''   )
												__magic_name__            =     check_and_map_params(
												    bert_output.dense.weight ,						f'''encoder.transformer_cells.{i}.ffn.ffn_2.weight'''   )
												__magic_name__            =     check_and_map_params(
												    bert_output.LayerNorm.bias ,						f'''encoder.transformer_cells.{i}.ffn.layer_norm.beta'''   )
												__magic_name__            =     check_and_map_params(
												    bert_output.LayerNorm.weight ,						f'''encoder.transformer_cells.{i}.ffn.layer_norm.gamma'''   )

						# Save space and energy 🎄
						hf_bort_model.half()

						# Compare output of both models
						__magic_name__            =     RobertaTokenizer.from_pretrained("roberta-base"   )

						__magic_name__            =     tokenizer.encode_plus(__lowerCamelCase   )["input_ids"]

						# Get gluon output
						__magic_name__            =     mx.nd.array([input_ids]   )
						__magic_name__            =     original_bort(inputs=__lowerCamelCase ,						token_types=[]   )

						# Get Transformer output (save and reload model again)
						hf_bort_model.save_pretrained(__lowerCamelCase   )
						__magic_name__            =     BertModel.from_pretrained(__lowerCamelCase   )
						hf_bort_model.eval()

						__magic_name__            =     tokenizer.encode_plus(__lowerCamelCase ,						return_tensors="pt"   )
						__magic_name__            =     hf_bort_model(**__lowerCamelCase   )[0]

						__magic_name__            =     output_gluon[0].asnumpy()
						__magic_name__            =     output_hf[0].detach().numpy()

						__magic_name__            =     np.max(np.abs(hf_layer - gluon_layer   )   ).item()
						__magic_name__            =     np.allclose(__lowerCamelCase ,						__lowerCamelCase ,						atol=1E-3   )

						if success:
												print("✔️ Both model do output the same tensors"   )
						else:
												print("❌ Both model do **NOT** output the same tensors"   )
												print("Absolute difference is:" ,						__lowerCamelCase   )


if __name__ == "__main__":
			lowercase    			=       argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '''--bort_checkpoint_path''', default=None, type=str, required=True, help='''Path the official Bort params file.'''
			)
			parser.add_argument(
			    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
			)
			lowercase    			=       parser.parse_args()
			convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)



