"""simple docstring"""







import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__SCREAMING_SNAKE_CASE								=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE								={
    "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/config.json",
}







class  UpperCamelCase	(    lowercase_						):
   lowercase							=						'mvp'
   lowercase							=						['past_key_values']
   lowercase							=						{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}
   def __init__(    self  ,__UpperCamelCase=5_0267  ,__UpperCamelCase=1024  ,__UpperCamelCase=12  ,__UpperCamelCase=4096  ,__UpperCamelCase=16  ,__UpperCamelCase=12  ,__UpperCamelCase=4096  ,__UpperCamelCase=16  ,__UpperCamelCase=0.0  ,__UpperCamelCase=0.0  ,__UpperCamelCase="gelu"  ,__UpperCamelCase=1024  ,__UpperCamelCase=0.1  ,__UpperCamelCase=0.0  ,__UpperCamelCase=0.0  ,__UpperCamelCase=0.02  ,__UpperCamelCase=0.0  ,__UpperCamelCase=False  ,__UpperCamelCase=True  ,__UpperCamelCase=1  ,__UpperCamelCase=0  ,__UpperCamelCase=2  ,__UpperCamelCase=True  ,__UpperCamelCase=2  ,__UpperCamelCase=2  ,__UpperCamelCase=False  ,__UpperCamelCase=100  ,__UpperCamelCase=800  ,**__UpperCamelCase  ,)			->   Optional[int]:


        '''simple docstring'''


        lowercase_			:       List[Any]									=			vocab_size
        lowercase_			:       Union[str, Any]									=			max_position_embeddings
        lowercase_			:       List[Any]									=			d_model
        lowercase_			:       List[Any]									=			encoder_ffn_dim
        lowercase_			:       str									=			encoder_layers
        lowercase_			:       str									=			encoder_attention_heads
        lowercase_			:       Any									=			decoder_ffn_dim
        lowercase_			:       List[Any]									=			decoder_layers
        lowercase_			:       List[str]									=			decoder_attention_heads
        lowercase_			:       Union[str, Any]									=			dropout
        lowercase_			:       int									=			attention_dropout
        lowercase_			:       List[str]									=			activation_dropout
        lowercase_			:       List[Any]									=			activation_function
        lowercase_			:       Any									=			init_std
        lowercase_			:       Optional[Any]									=			encoder_layerdrop
        lowercase_			:       Any									=			decoder_layerdrop
        lowercase_			:       Optional[int]									=			classifier_dropout
        lowercase_			:       int									=			use_cache
        lowercase_			:       Union[str, Any]									=			encoder_layers
        lowercase_			:       Optional[int]									=			scale_embedding  # scale factor will be sqrt(d_model) if True
        lowercase_			:       Union[str, Any]									=			use_prompt
        lowercase_			:       Tuple									=			prompt_length
        lowercase_			:       Dict									=			prompt_mid_dim

        super().__init__(
            pad_token_id=__UpperCamelCase  ,bos_token_id=__UpperCamelCase  ,eos_token_id=__UpperCamelCase  ,is_encoder_decoder=__UpperCamelCase  ,decoder_start_token_id=__UpperCamelCase  ,forced_eos_token_id=__UpperCamelCase  ,**__UpperCamelCase  ,)

        if self.forced_bos_token_id is None and kwargs.get('force_bos_token_to_be_generated'  ,__UpperCamelCase    ):
             lowercase_			:       Tuple									=			self.bos_token_id
             warnings.warn(
                 f'''Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. '''
                 'The config can simply be saved and uploaded again to be fixed.'    )


"""simple docstring"""







import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)







class  UpperCamelCase	(    nn.Module						):
   def __init__(    self    )			->   Any:


        '''simple docstring'''


        super().__init__()
        lowercase_			:       Dict									=			nn.Linear(3  ,4    )
        lowercase_			:       Union[str, Any]									=			nn.BatchNormad(4    )
        lowercase_			:       Union[str, Any]									=			nn.Linear(4  ,5    )







   def     _UpperCAmelCase     (    self  ,__UpperCamelCase    )			->   int:


        '''simple docstring'''


        return self.lineara(self.batchnorm(self.lineara(__UpperCamelCase    )    )    )







class  UpperCamelCase	(    unittest.TestCase						):
   def     _UpperCAmelCase     (    self    )			->   Dict:


        '''simple docstring'''


        lowercase_			:       int									=			ModelForTest()
        with TemporaryDirectory() as tmp_dir:
             offload_state_dict(__UpperCamelCase  ,model.state_dict()    )
             lowercase_			:       str									=			os.path.join(__UpperCamelCase  ,'index.json'    )
             self.assertTrue(os.path.isfile(__UpperCamelCase    )    )
             # TODO: add tests on what is inside the index

             for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
                  lowercase_			:       int									=			os.path.join(__UpperCamelCase  ,f'''{key}.dat'''    )
                  self.assertTrue(os.path.isfile(__UpperCamelCase    )    )
                  # TODO: add tests on the fact weights are properly loaded
   def     _UpperCAmelCase     (    self    )			->   Tuple:


        '''simple docstring'''


        lowercase_			:       List[str]									=			[torch.floataa, torch.floataa, torch.bfloataa]

        for dtype in dtypes:
             lowercase_			:       Tuple									=			torch.randn(2  ,3  ,dtype=__UpperCamelCase    )
             with TemporaryDirectory() as tmp_dir:
                  lowercase_			:       List[str]									=			offload_weight(__UpperCamelCase  ,'weight'  ,__UpperCamelCase  ,{}    )
                  lowercase_			:       str									=			os.path.join(__UpperCamelCase  ,'weight.dat'    )
                  self.assertTrue(os.path.isfile(__UpperCamelCase    )    )
                  self.assertDictEqual(__UpperCamelCase  ,{'weight': {'shape': [2, 3], 'dtype': str(__UpperCamelCase    ).split('.'    )[1]}}    )

                  lowercase_			:       Optional[Any]									=			load_offloaded_weight(__UpperCamelCase  ,index['weight']    )
                  self.assertTrue(torch.equal(__UpperCamelCase  ,__UpperCamelCase    )    )
   def     _UpperCAmelCase     (    self    )			->   List[Any]:


        '''simple docstring'''


        lowercase_			:       List[str]									=			ModelForTest()
        lowercase_			:       Optional[int]									=			model.state_dict()
        lowercase_			:       int									=			{k: v for k, v in state_dict.items() if 'linear2' not in k}
        lowercase_			:       Optional[Any]									=			{k: v for k, v in state_dict.items() if 'linear2' in k}

        with TemporaryDirectory() as tmp_dir:
             offload_state_dict(__UpperCamelCase  ,__UpperCamelCase    )
             lowercase_			:       Tuple									=			OffloadedWeightsLoader(state_dict=__UpperCamelCase  ,save_folder=__UpperCamelCase    )

             # Every key is there with the right value
             self.assertEqual(sorted(__UpperCamelCase    )  ,sorted(state_dict.keys()    )    )
             for key, param in state_dict.items():
                  self.assertTrue(torch.allclose(__UpperCamelCase  ,weight_map[key]    )    )

        lowercase_			:       int									=			{k: v for k, v in state_dict.items() if 'weight' in k}
        lowercase_			:       int									=			{k: v for k, v in state_dict.items() if 'weight' not in k}

        with TemporaryDirectory() as tmp_dir:
             offload_state_dict(__UpperCamelCase  ,__UpperCamelCase    )
             lowercase_			:       int									=			OffloadedWeightsLoader(state_dict=__UpperCamelCase  ,save_folder=__UpperCamelCase    )

             # Every key is there with the right value
             self.assertEqual(sorted(__UpperCamelCase    )  ,sorted(state_dict.keys()    )    )
             for key, param in state_dict.items():
                  self.assertTrue(torch.allclose(__UpperCamelCase  ,weight_map[key]    )    )

        with TemporaryDirectory() as tmp_dir:
             offload_state_dict(__UpperCamelCase  ,__UpperCamelCase    )
             # Duplicates are removed
             lowercase_			:       Tuple									=			OffloadedWeightsLoader(state_dict=__UpperCamelCase  ,save_folder=__UpperCamelCase    )

             # Every key is there with the right value
             self.assertEqual(sorted(__UpperCamelCase    )  ,sorted(state_dict.keys()    )    )
             for key, param in state_dict.items():
                  self.assertTrue(torch.allclose(__UpperCamelCase  ,weight_map[key]    )    )







   def     _UpperCAmelCase     (    self    )			->   Union[str, Any]:


        '''simple docstring'''


        lowercase_			:       Optional[int]									=			{'a.1': 0, 'a.10': 1, 'a.2': 2}
        lowercase_			:       int									=			extract_submodules_state_dict(__UpperCamelCase  ,['a.1', 'a.2']    )
        self.assertDictEqual(__UpperCamelCase  ,{'a.1': 0, 'a.2': 2}    )

        lowercase_			:       Union[str, Any]									=			{'a.1.a': 0, 'a.10.a': 1, 'a.2.a': 2}
        lowercase_			:       Optional[int]									=			extract_submodules_state_dict(__UpperCamelCase  ,['a.1', 'a.2']    )
        self.assertDictEqual(__UpperCamelCase  ,{'a.1.a': 0, 'a.2.a': 2}    )

