





"""simple docstring"""






import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)



class 	lowerCAmelCase					(      nn.Module							):






       def __init__(   self     ):
              super().__init__()
              _UpperCAmelCase     =     nn.Linear(3    ,  4     )
              _UpperCAmelCase     =     nn.BatchNormad(4     )
              _UpperCAmelCase     =     nn.Linear(4    ,  5     )






       def 							__A (   self    ,  a__     ):
              return self.lineara(self.batchnorm(self.lineara(a__     )     )     )



class 	lowerCAmelCase					(      unittest.TestCase							):






       def 							__A (   self     ):
              _UpperCAmelCase     =     ModelForTest()
              with TemporaryDirectory() as tmp_dir:
                     offload_state_dict(a__    ,  model.state_dict()     )
                     _UpperCAmelCase     =     os.path.join(a__    ,  'index.json'     )
                     self.assertTrue(os.path.isfile(a__     )     )
                     # TODO: add tests on what is inside the index

                     for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
                            _UpperCAmelCase     =     os.path.join(a__    ,  f"""{key}.dat"""     )
                            self.assertTrue(os.path.isfile(a__     )     )
                            # TODO: add tests on the fact weights are properly loaded






       def 							__A (   self     ):
              _UpperCAmelCase     =     [torch.floataa, torch.floataa, torch.bfloataa]

              for dtype in dtypes:
                     _UpperCAmelCase     =     torch.randn(2    ,  3    ,  dtype=a__     )
                     with TemporaryDirectory() as tmp_dir:
                            _UpperCAmelCase     =     offload_weight(a__    ,  'weight'    ,  a__    ,  {}     )
                            _UpperCAmelCase     =     os.path.join(a__    ,  'weight.dat'     )
                            self.assertTrue(os.path.isfile(a__     )     )
                            self.assertDictEqual(a__    ,  {'weight': {'shape': [2, 3], 'dtype': str(a__     ).split('.'     )[1]}}     )

                            _UpperCAmelCase     =     load_offloaded_weight(a__    ,  index['weight']     )
                            self.assertTrue(torch.equal(a__    ,  a__     )     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     ModelForTest()
              _UpperCAmelCase     =     model.state_dict()
              _UpperCAmelCase     =     {k: v for k, v in state_dict.items() if 'linear2' not in k}
              _UpperCAmelCase     =     {k: v for k, v in state_dict.items() if 'linear2' in k}

              with TemporaryDirectory() as tmp_dir:
                     offload_state_dict(a__    ,  a__     )
                     _UpperCAmelCase     =     OffloadedWeightsLoader(state_dict=a__    ,  save_folder=a__     )

                     # Every key is there with the right value
                     self.assertEqual(sorted(a__     )    ,  sorted(state_dict.keys()     )     )
                     for key, param in state_dict.items():
                            self.assertTrue(torch.allclose(a__    ,  weight_map[key]     )     )

              _UpperCAmelCase     =     {k: v for k, v in state_dict.items() if 'weight' in k}
              _UpperCAmelCase     =     {k: v for k, v in state_dict.items() if 'weight' not in k}

              with TemporaryDirectory() as tmp_dir:
                     offload_state_dict(a__    ,  a__     )
                     _UpperCAmelCase     =     OffloadedWeightsLoader(state_dict=a__    ,  save_folder=a__     )

                     # Every key is there with the right value
                     self.assertEqual(sorted(a__     )    ,  sorted(state_dict.keys()     )     )
                     for key, param in state_dict.items():
                            self.assertTrue(torch.allclose(a__    ,  weight_map[key]     )     )

              with TemporaryDirectory() as tmp_dir:
                     offload_state_dict(a__    ,  a__     )
                     # Duplicates are removed
                     _UpperCAmelCase     =     OffloadedWeightsLoader(state_dict=a__    ,  save_folder=a__     )

                     # Every key is there with the right value
                     self.assertEqual(sorted(a__     )    ,  sorted(state_dict.keys()     )     )
                     for key, param in state_dict.items():
                            self.assertTrue(torch.allclose(a__    ,  weight_map[key]     )     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     {'a.1': 0, 'a.10': 1, 'a.2': 2}
              _UpperCAmelCase     =     extract_submodules_state_dict(a__    ,  ['a.1', 'a.2']     )
              self.assertDictEqual(a__    ,  {'a.1': 0, 'a.2': 2}     )

              _UpperCAmelCase     =     {'a.1.a': 0, 'a.10.a': 1, 'a.2.a': 2}
              _UpperCAmelCase     =     extract_submodules_state_dict(a__    ,  ['a.1', 'a.2']     )
              self.assertDictEqual(a__    ,  {'a.1.a': 0, 'a.2.a': 2}     )













"""simple docstring"""






import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
    import optax
    from flax.training.common_utils import onehot

    from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
    from transformers.models.ta.modeling_flax_ta import shift_tokens_right



@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 	lowerCAmelCase					(      unittest.TestCase							):






       @slow
       def 							__A (   self     ):
              _UpperCAmelCase     =     FlaxMTaForConditionalGeneration.from_pretrained('google/mt5-small'     )
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('google/mt5-small'     )

              _UpperCAmelCase     =     tokenizer('Hello there'    ,  return_tensors='np'     ).input_ids
              _UpperCAmelCase     =     tokenizer('Hi I am'    ,  return_tensors='np'     ).input_ids

              _UpperCAmelCase     =     shift_tokens_right(a__    ,  model.config.pad_token_id    ,  model.config.decoder_start_token_id     )

              _UpperCAmelCase     =     model(a__    ,  decoder_input_ids=a__     ).logits
              _UpperCAmelCase     =     optax.softmax_cross_entropy(a__    ,  onehot(a__    ,  logits.shape[-1]     )     ).mean()

              _UpperCAmelCase     =     -(labels.shape[-1] * loss.item())

              _UpperCAmelCase     =     -84.9_127
              self.assertTrue(abs(mtf_score - EXPECTED_SCORE     ) < 1E-4     )






