





import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


_lowerCAmelCase 				=     {
    """return_dict""": False,
    """output_hidden_states""": True,
    """output_attentions""": True,
    """torchscript""": True,
    """torch_dtype""": """float16""",
    """use_bfloat16""": True,
    """tf_legacy_loss""": True,
    """pruned_heads""": {"""a""": 1},
    """tie_word_embeddings""": False,
    """is_decoder""": True,
    """cross_attention_hidden_size""": 128,
    """add_cross_attention""": True,
    """tie_encoder_decoder""": True,
    """max_length""": 50,
    """min_length""": 3,
    """do_sample""": True,
    """early_stopping""": True,
    """num_beams""": 3,
    """num_beam_groups""": 3,
    """diversity_penalty""": 0.5,
    """temperature""": 2.0,
    """top_k""": 10,
    """top_p""": 0.7,
    """typical_p""": 0.2,
    """repetition_penalty""": 0.8,
    """length_penalty""": 0.8,
    """no_repeat_ngram_size""": 5,
    """encoder_no_repeat_ngram_size""": 5,
    """bad_words_ids""": [1, 2, 3],
    """num_return_sequences""": 3,
    """chunk_size_feed_forward""": 5,
    """output_scores""": True,
    """return_dict_in_generate""": True,
    """forced_bos_token_id""": 2,
    """forced_eos_token_id""": 3,
    """remove_invalid_values""": True,
    """architectures""": ["""BertModel"""],
    """finetuning_task""": """translation""",
    """id2label""": {0: """label"""},
    """label2id""": {"""label""": """0"""},
    """tokenizer_class""": """BertTokenizerFast""",
    """prefix""": """prefix""",
    """bos_token_id""": 6,
    """pad_token_id""": 7,
    """eos_token_id""": 8,
    """sep_token_id""": 9,
    """decoder_start_token_id""": 10,
    """exponential_decay_length_penalty""": (5, 1.01),
    """suppress_tokens""": [0, 1],
    """begin_suppress_tokens""": 2,
    """task_specific_params""": {"""translation""": """some_params"""},
    """problem_type""": """regression""",
}

@is_staging_test
class    _UpperCAmelCase							(		unittest.TestCase     ):
      @classmethod
      def 		_lowerCamelCase      (		cls       ):
         A_     :				Any			      =	TOKEN
         HfFolder.save_token(a__       )
      @classmethod
      def 		_lowerCamelCase      (		cls       ):
         try:
            delete_repo(token=cls._token      ,       repo_id="""test-config"""       )
         except HTTPError:
            pass

         try:
            delete_repo(token=cls._token      ,       repo_id="""valid_org/test-config-org"""       )
         except HTTPError:
            pass

         try:
            delete_repo(token=cls._token      ,       repo_id="""test-dynamic-config"""       )
         except HTTPError:
            pass
      def 		_lowerCamelCase      (		self       ):
         A_     :				List[str]			      =	BertConfig(
             vocab_size=99      ,       hidden_size=32      ,       num_hidden_layers=5      ,       num_attention_heads=4      ,       intermediate_size=37       )
         config.push_to_hub("""test-config"""      ,       use_auth_token=self._token       )

         A_     :				Optional[Any]			      =	BertConfig.from_pretrained(F"""{USER}/test-config"""       )
         for k, v in config.to_dict().items():
            if k != "transformers_version":
               self.assertEqual(a__      ,       getattr(a__      ,       a__       )       )

        # Reset repo
         delete_repo(token=self._token      ,       repo_id="""test-config"""       )

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
            config.save_pretrained(a__      ,       repo_id="""test-config"""      ,       push_to_hub=a__      ,       use_auth_token=self._token       )

         A_     :				Tuple			      =	BertConfig.from_pretrained(F"""{USER}/test-config"""       )
         for k, v in config.to_dict().items():
            if k != "transformers_version":
               self.assertEqual(a__      ,       getattr(a__      ,       a__       )       )
      def 		_lowerCamelCase      (		self       ):
         A_     :				int			      =	BertConfig(
             vocab_size=99      ,       hidden_size=32      ,       num_hidden_layers=5      ,       num_attention_heads=4      ,       intermediate_size=37       )
         config.push_to_hub("""valid_org/test-config-org"""      ,       use_auth_token=self._token       )

         A_     :				Optional[int]			      =	BertConfig.from_pretrained("""valid_org/test-config-org"""       )
         for k, v in config.to_dict().items():
            if k != "transformers_version":
               self.assertEqual(a__      ,       getattr(a__      ,       a__       )       )

        # Reset repo
         delete_repo(token=self._token      ,       repo_id="""valid_org/test-config-org"""       )

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
            config.save_pretrained(
                a__      ,       repo_id="""valid_org/test-config-org"""      ,       push_to_hub=a__      ,       use_auth_token=self._token       )

         A_     :				str			      =	BertConfig.from_pretrained("""valid_org/test-config-org"""       )
         for k, v in config.to_dict().items():
            if k != "transformers_version":
               self.assertEqual(a__      ,       getattr(a__      ,       a__       )       )






      def 		_lowerCamelCase      (		self       ):
         CustomConfig.register_for_auto_class()
         A_     :				Any			      =	CustomConfig(attribute=42       )

         config.push_to_hub("""test-dynamic-config"""      ,       use_auth_token=self._token       )

         # This has added the proper auto_map field to the config
         self.assertDictEqual(config.auto_map      ,       {"""AutoConfig""": """custom_configuration.CustomConfig"""}       )

         A_     :				Union[str, Any]			      =	AutoConfig.from_pretrained(F"""{USER}/test-dynamic-config"""      ,       trust_remote_code=a__       )
         # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
         self.assertEqual(new_config.__class__.__name__      ,       """CustomConfig"""       )
         self.assertEqual(new_config.attribute      ,       42       )






class    _UpperCAmelCase							(		unittest.TestCase     ):
      def 		_lowerCamelCase      (		self       ):
         A_     :				str			      =	GPTaConfig()

         # attempt to modify each of int/float/bool/str config records and verify they were updated
         A_     :				str			      =	c.n_embd + 1  # int
         A_     :				List[str]			      =	c.resid_pdrop + 1.0  # float
         A_     :				Optional[Any]			      =	not c.scale_attn_weights  # bool
         A_     :				List[Any]			      =	c.summary_type + """foo"""  # str
         c.update_from_string(
             F"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""       )
         self.assertEqual(a__      ,       c.n_embd      ,       """mismatch for key: n_embd"""       )
         self.assertEqual(a__      ,       c.resid_pdrop      ,       """mismatch for key: resid_pdrop"""       )
         self.assertEqual(a__      ,       c.scale_attn_weights      ,       """mismatch for key: scale_attn_weights"""       )
         self.assertEqual(a__      ,       c.summary_type      ,       """mismatch for key: summary_type"""       )
      def 		_lowerCamelCase      (		self       ):
         A_     :				Dict			      =	PretrainedConfig()
         A_     :				Optional[Any]			      =	[key for key in base_config.__dict__ if key not in config_common_kwargs]
         # If this part of the test fails, you have arguments to addin config_common_kwargs above.
         self.assertListEqual(
             a__      ,       ["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]       )
         A_     :				Optional[Any]			      =	[key for key, value in config_common_kwargs.items() if value == getattr(a__      ,       a__       )]
         if len(a__       ) > 0:
            raise ValueError(
                """The following keys are set with the default values in"""
                """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
                F""" {', '.join(a__       )}."""       )
      def 		_lowerCamelCase      (		self       ):
         with self.assertRaises(a__       ):
            # config is in subfolder, the following should not work without specifying the subfolder
            A_     :				Any			      =	BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""       )

         A_     :				int			      =	BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""      ,       subfolder="""bert"""       )

         self.assertIsNotNone(a__       )
      def 		_lowerCamelCase      (		self       ):
         # A mock response for an HTTP head request to emulate server down
         A_     :				Optional[int]			      =	mock.Mock()
         A_     :				Tuple			      =	500
         A_     :				Optional[Any]			      =	{}
         A_     :				int			      =	HTTPError
         A_     :				Tuple			      =	{}

         # Download this model to make sure it's in the cache.
         A_     :				str			      =	BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )

         # Under the mock environment we get a 500 error when trying to reach the model.
         with mock.patch("""requests.Session.request"""      ,       return_value=a__       ) as mock_head:
            A_     :				Union[str, Any]			      =	BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
            # This check we did call the fake head request
            mock_head.assert_called()
      def 		_lowerCamelCase      (		self       ):
         # This test is for deprecated behavior and can be removed in v5
         A_     :				Union[str, Any]			      =	BertConfig.from_pretrained(
             """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""       )
      def 		_lowerCamelCase      (		self       ):
         A_     :				Optional[Any]			      =	AutoConfig.from_pretrained("""bert-base-cased"""       )
         A_     :				Union[str, Any]			      =	["""config.4.0.0.json"""]

         with tempfile.TemporaryDirectory() as tmp_dir:
            configuration.save_pretrained(a__       )
            A_     :				str			      =	2
            json.dump(configuration.to_dict()      ,       open(os.path.join(a__      ,       """config.4.0.0.json"""       )      ,       """w"""       )       )

            # This should pick the new configuration file as the version of Transformers is > 4.0.0
            A_     :				Optional[Any]			      =	AutoConfig.from_pretrained(a__       )
            self.assertEqual(new_configuration.hidden_size      ,       2       )

            # Will need to be adjusted if we reach v42 and this test is still here.
            # Should pick the old configuration file as the version of Transformers is < 4.42.0
            A_     :				List[Any]			      =	["""config.42.0.0.json"""]
            A_     :				Any			      =	768
            configuration.save_pretrained(a__       )
            shutil.move(os.path.join(a__      ,       """config.4.0.0.json"""       )      ,       os.path.join(a__      ,       """config.42.0.0.json"""       )       )
            A_     :				str			      =	AutoConfig.from_pretrained(a__       )
            self.assertEqual(new_configuration.hidden_size      ,       768       )






      def 		_lowerCamelCase      (		self       ):
         # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
         A_     :				Any			      =	"""hf-internal-testing/test-two-configs"""

         import transformers as new_transformers

         A_     :				Union[str, Any]			      =	"""v4.0.0"""
         A_      ,				A_     :				int			      =	new_transformers.models.auto.AutoConfig.from_pretrained(
             a__      ,       return_unused_kwargs=a__       )
         self.assertEqual(new_configuration.hidden_size      ,       2       )
         # This checks `_configuration_file` ia not kept in the kwargs by mistake.
         self.assertDictEqual(a__      ,       {}       )

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         A_     :				List[Any]			      =	"""v3.0.0"""
         A_     :				Optional[int]			      =	old_transformers.models.auto.AutoConfig.from_pretrained(a__       )
         self.assertEqual(old_configuration.hidden_size      ,       768       )













import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    UniSpeechConfig,
    UniSpeechForCTC,
    UniSpeechForPreTraining,
    WavaVecaFeatureExtractor,
    WavaVecaPhonemeCTCTokenizer,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
_lowerCAmelCase 				=     logging.get_logger(__name__)

_lowerCAmelCase 				=     {
    """post_extract_proj""": """feature_projection.projection""",
    """encoder.pos_conv.0""": """encoder.pos_conv_embed.conv""",
    """self_attn.k_proj""": """encoder.layers.*.attention.k_proj""",
    """self_attn.v_proj""": """encoder.layers.*.attention.v_proj""",
    """self_attn.q_proj""": """encoder.layers.*.attention.q_proj""",
    """self_attn.out_proj""": """encoder.layers.*.attention.out_proj""",
    """self_attn_layer_norm""": """encoder.layers.*.layer_norm""",
    """fc1""": """encoder.layers.*.feed_forward.intermediate_dense""",
    """fc2""": """encoder.layers.*.feed_forward.output_dense""",
    """final_layer_norm""": """encoder.layers.*.final_layer_norm""",
    """encoder.layer_norm""": """encoder.layer_norm""",
    """w2v_model.layer_norm""": """feature_projection.layer_norm""",
    """quantizer.weight_proj""": """quantizer.weight_proj""",
    """quantizer.vars""": """quantizer.codevectors""",
    """project_q""": """project_q""",
    """final_proj""": """project_hid""",
    """w2v_encoder.proj""": """ctc_proj""",
    """mask_emb""": """masked_spec_embed""",
}
_lowerCAmelCase 				=     [
    """ctc_proj""",
    """quantizer.weight_proj""",
    """quantizer.codevectors""",
    """project_q""",
    """project_hid""",
]



def     _lowerCAmelCase		(  _lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase     ):
   '''simple docstring'''


   for attribute in key.split("""."""     ):
      if is_finetuned:
         if attribute in ["quantizer", "project_q", "project_hid"]:
            # those layers are only relevant for pretraining and should be dropped
            return

         if attribute == "ctc_proj":
            # we should rename `ctc_proj` to `lm_head` for fine-tuned phoneme models
            A_     :				Tuple			      =	"""lm_head"""

      A_     :				int			      =	getattr(_lowerCAmelCase  ,_lowerCAmelCase     )

   if weight_type is not None:
      A_     :				List[str]			      =	getattr(_lowerCAmelCase  ,_lowerCAmelCase     ).shape
   else:
      A_     :				Optional[Any]			      =	hf_pointer.shape

   assert hf_shape == value.shape, (
       f"""Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"""
       f""" {value.shape} for {full_name}"""
   )

   if weight_type == "weight":
      A_     :				Union[str, Any]			      =	value
   elif weight_type == "weight_g":
      A_     :				Any			      =	value
   elif weight_type == "weight_v":
      A_     :				Tuple			      =	value
   elif weight_type == "bias":
      A_     :				List[Any]			      =	value
   else:
      A_     :				int			      =	value

   logger.info(f"""{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}."""     )



def     _lowerCAmelCase		(  _lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase     ):
   '''simple docstring'''


   A_     :				Dict			      =	[]
   A_     :				List[Any]			      =	fairseq_model.state_dict()

   A_     :				Any			      =	hf_model.unispeech.feature_extractor

   for name, value in fairseq_dict.items():
      A_     :				Any			      =	False
      if "conv_layers" in name:
         load_conv_layer(
             _lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,hf_model.config.feat_extract_norm == """group"""  ,)
         A_     :				int			      =	True
      else:
         for key, mapped_key in MAPPING.items():
            A_     :				Any			      =	"""unispeech.""" + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
            if key in name or key.split("""w2v_model."""     )[-1] == name.split("""."""     )[0]:
               A_     :				Dict			      =	True
               if "*" in mapped_key:
                  A_     :				Any			      =	name.split(_lowerCAmelCase     )[0].split("""."""     )[-2]
                  A_     :				Optional[int]			      =	mapped_key.replace("""*"""  ,_lowerCAmelCase     )
               if "weight_g" in name:
                  A_     :				Tuple			      =	"""weight_g"""
               elif "weight_v" in name:
                  A_     :				Optional[int]			      =	"""weight_v"""
               elif "bias" in name:
                  A_     :				List[Any]			      =	"""bias"""
               elif "weight" in name:
                  # TODO: don't match quantizer.weight_proj
                  A_     :				Optional[int]			      =	"""weight"""
               else:
                  A_     :				Dict			      =	None
               set_recursively(_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase     )
            continue
      if not is_used:
         unused_weights.append(_lowerCAmelCase     )

   logger.warning(f"""Unused weights: {unused_weights}"""     )



def     _lowerCAmelCase		(  _lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase     ):
   '''simple docstring'''


   A_     :				Union[str, Any]			      =	full_name.split("""conv_layers."""     )[-1]
   A_     :				Union[str, Any]			      =	name.split("""."""     )
   A_     :				List[str]			      =	int(items[0]     )
   A_     :				Optional[Any]			      =	int(items[1]     )

   if type_id == 0:
      if "bias" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
             f"""{full_name} has size {value.shape}, but"""
             f""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""
         )
         A_     :				List[Any]			      =	value
         logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}."""     )
      elif "weight" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
             f"""{full_name} has size {value.shape}, but"""
             f""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""
         )
         A_     :				int			      =	value
         logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}."""     )
   elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
      if "bias" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
             f"""{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was"""
             " found."
         )
         A_     :				List[str]			      =	value
         logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""     )
      elif "weight" in name:
         assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
             f"""{full_name} has size {value.shape}, but"""
             f""" {feature_extractor[layer_id].layer_norm.weight.data.shape} was found."""
         )
         A_     :				Tuple			      =	value
         logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""     )
   else:
      unused_weights.append(_lowerCAmelCase     )



@torch.no_grad()
def     _lowerCAmelCase		(  _lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase=None  ,_lowerCAmelCase=None  ,_lowerCAmelCase=True     ):
   '''simple docstring'''


   if config_path is not None:
      A_     :				Optional[int]			      =	UniSpeechConfig.from_pretrained(_lowerCAmelCase     )
   else:
      A_     :				List[str]			      =	UniSpeechConfig()

   if is_finetuned:
      if dict_path:
         A_     :				str			      =	Dictionary.load_from_json(_lowerCAmelCase     )

         # important change bos & pad token id since CTC symbol is <pad> and
         # not <s> as in fairseq
         A_     :				List[Any]			      =	target_dict.pad_index
         A_     :				str			      =	target_dict.bos_index
         A_     :				Optional[Any]			      =	target_dict.eos_index
         A_     :				Optional[int]			      =	len(target_dict.symbols     )
         A_     :				str			      =	os.path.join(_lowerCAmelCase  ,"""vocab.json"""     )
         if not os.path.isdir(_lowerCAmelCase     ):
            logger.error("""--pytorch_dump_folder_path ({}) should be a directory""".format(_lowerCAmelCase     )     )
            return
         os.makedirs(_lowerCAmelCase  ,exist_ok=_lowerCAmelCase     )
         A_     :				Optional[int]			      =	target_dict.indices

         # fairseq has the <pad> and <s> switched
         A_     :				Dict			      =	4_2
         A_     :				Optional[Any]			      =	4_3
         with open(_lowerCAmelCase  ,"""w"""  ,encoding="""utf-8"""     ) as vocab_handle:
            json.dump(_lowerCAmelCase  ,_lowerCAmelCase     )
         A_     :				Tuple			      =	WavaVecaPhonemeCTCTokenizer(
             _lowerCAmelCase  ,unk_token=target_dict.unk_word  ,pad_token=target_dict.pad_word  ,bos_token=target_dict.bos_word  ,eos_token=target_dict.eos_word  ,word_delimiter_token="""|"""  ,do_lower_case=_lowerCAmelCase  ,)
         A_     :				Any			      =	True if config.feat_extract_norm == """layer""" else False
         A_     :				List[str]			      =	WavaVecaFeatureExtractor(
             feature_size=1  ,sampling_rate=1_6_0_0_0  ,padding_value=0  ,do_normalize=_lowerCAmelCase  ,return_attention_mask=_lowerCAmelCase  ,)
         A_     :				Optional[int]			      =	WavaVecaProcessor(feature_extractor=_lowerCAmelCase  ,tokenizer=_lowerCAmelCase     )
         processor.save_pretrained(_lowerCAmelCase     )

      A_     :				int			      =	UniSpeechForCTC(_lowerCAmelCase     )
   else:
      A_     :				str			      =	UniSpeechForPreTraining(_lowerCAmelCase     )

   if is_finetuned:
      A_      ,				A_      ,				A_     :				Union[str, Any]			      =	fairseq.checkpoint_utils.load_model_ensemble_and_task(
          [checkpoint_path]  ,arg_overrides={"""data""": """/""".join(dict_path.split("""/"""     )[:-1]     ), """w2v_path""": checkpoint_path}     )
   else:
      A_      ,				A_      ,				A_     :				Any			      =	fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]     )

   A_     :				List[str]			      =	model[0].eval()

   recursively_load_weights(_lowerCAmelCase  ,_lowerCAmelCase  ,_lowerCAmelCase     )

   hf_unispeech.save_pretrained(_lowerCAmelCase     )


if __name__ == "__main__":
      _lowerCAmelCase 				=     argparse.ArgumentParser()
      parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
      parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
      parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
      parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
      parser.add_argument(
          """--not_finetuned""", action="""store_true""", help="""Whether the model to convert is a fine-tuned model or not"""
      )
      _lowerCAmelCase 				=     parser.parse_args()
      convert_unispeech_checkpoint(
          args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
      )






