'''simple docstring'''




import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     if model.config.model_type == "gpt2":
          return model.transformer.h[0].mlp.c_fc
     return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
 import torch
 import torch.nn as nn


 class       UpperCAmelCase__  (       nn.Module			):






    """simple docstring"""
    def __init__(self       ,				_a       ,				_a				)    ->     Union[str, Any]:
         super().__init__()
         lowercase_					:       Dict     				=					module
         lowercase_					:       Dict     				=					nn.Sequential(
             nn.Linear(module.in_features       ,				_a       ,				bias=_a				)       ,				nn.Linear(_a       ,				module.out_features       ,				bias=_a				)       ,				)
         lowercase_					:       Dict     				=					(2.0 / (5 * min(module.in_features       ,				module.out_features				))) ** 0.5
         nn.init.normal_(self.adapter[0].weight       ,				std=_a				)
         nn.init.zeros_(self.adapter[1].weight				)
         self.adapter.to(module.weight.device				)





    def 		_lowerCamelCase (self       ,				_a       ,				*_a       ,				**_a				)    ->     Tuple:
         return self.module(_a       ,				*_a       ,				**_a				) + self.adapter(_a				)


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class       UpperCAmelCase__  (       unittest.TestCase			):






   """simple docstring"""
   # We keep the constants inside the init function and model loading inside setUp function

   # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
   # Therefore here we use only bloom-1b3 to test our module
   A							:       Union[str, Any]     =    '''bigscience/bloom-1b7'''

   # Constant values
   A							:       int     =    2.109_659_552_692_574

   A							:       Dict     =    '''Hello my name is'''
   A							:       Optional[int]     =    set()
   EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I'''			)
   EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n'''			)
   EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University'''			)
   A							:       Tuple     =    10
   def 		_lowerCamelCase (self				)    ->     str:
        # Models and tokenizer
        lowercase_					:       int     				=					AutoTokenizer.from_pretrained(self.model_name				)



class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   def 		_lowerCamelCase (self				)    ->     Tuple:
        super().setUp()

        # Models and tokenizer
        lowercase_					:       int     				=					AutoModelForCausalLM.from_pretrained(
            self.model_name       ,				torch_dtype=torch.floataa       ,				device_map='auto'				)
        lowercase_					:       Union[str, Any]     				=					AutoModelForCausalLM.from_pretrained(self.model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
   def 		_lowerCamelCase (self				)    ->     Tuple:
        del self.model_fpaa
        del self.model_abit

        gc.collect()
        torch.cuda.empty_cache()
   def 		_lowerCamelCase (self				)    ->     str:
        lowercase_					:       Optional[Any]     				=					self.model_abit.config

        self.assertTrue(hasattr(_a       ,				'quantization_config'				)				)

        lowercase_					:       List[str]     				=					config.to_dict()
        lowercase_					:       List[Any]     				=					config.to_diff_dict()

        lowercase_					:       Any     				=					config.to_json_string()
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        from bitsandbytes.nn import Paramsabit

        lowercase_					:       Union[str, Any]     				=					self.model_fpaa.get_memory_footprint()
        lowercase_					:       List[str]     				=					self.model_abit.get_memory_footprint()

        self.assertAlmostEqual(mem_fpaa / mem_abit       ,				self.EXPECTED_RELATIVE_DIFFERENCE				)
        lowercase_					:       int     				=					get_some_linear_layer(self.model_abit				)
        self.assertTrue(linear.weight.__class__ == Paramsabit				)
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        from transformers import TaPreTrainedModel

        self.model_fpaa.get_memory_footprint()
        self.model_abit.get_memory_footprint()

        for name, module in self.model_abit.named_modules():
             if isinstance(_a       ,				torch.nn.Linear				):
                  if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                       # 4-bit parameters are packed in uint8 variables
                       self.assertTrue(module.weight.dtype == torch.uinta				)
   def 		_lowerCamelCase (self				)    ->     List[str]:
        lowercase_					:       str     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				)
        lowercase_					:       Tuple     				=					self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0				)       ,				max_new_tokens=10				)

        self.assertIn(self.tokenizer.decode(output_sequences[0]       ,				skip_special_tokens=_a				)       ,				self.EXPECTED_OUTPUTS				)
   def 		_lowerCamelCase (self				)    ->     Dict:
        lowercase_					:       int     				=					BitsAndBytesConfig()
        lowercase_					:       str     				=					True

        lowercase_					:       Tuple     				=					AutoModelForCausalLM.from_pretrained(
            self.model_name       ,				quantization_config=_a       ,				device_map='auto'				)

        lowercase_					:       Tuple     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				)
        lowercase_					:       List[Any]     				=					model_abit_from_config.generate(
            input_ids=encoded_input['input_ids'].to(0				)       ,				max_new_tokens=10				)

        self.assertIn(self.tokenizer.decode(output_sequences[0]       ,				skip_special_tokens=_a				)       ,				self.EXPECTED_OUTPUTS				)
   def 		_lowerCamelCase (self				)    ->     Any:
        with self.assertRaises(_a				), tempfile.TemporaryDirectory() as tmpdirname:
             self.model_abit.save_pretrained(_a				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        lowercase_					:       Union[str, Any]     				=					BitsAndBytesConfig()

        with self.assertRaises(_a				):
             lowercase_					:       Union[str, Any]     				=					AutoModelForCausalLM.from_pretrained(
                 self.model_name       ,				quantization_config=_a       ,				load_in_abit=_a       ,				device_map='auto'       ,				bnb_abit_quant_type='nf4'       ,				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        with self.assertRaises(_a				):
             # Tries with `str`
             self.model_abit.to('cpu'				)

        with self.assertRaises(_a				):
             # Tries with a `dtype``
             self.model_abit.to(torch.floataa				)

        with self.assertRaises(_a				):
             # Tries with a `device`
             self.model_abit.to(torch.device('cuda:0'				)				)

        with self.assertRaises(_a				):
             # Tries with a `device`
             self.model_abit.float()

        with self.assertRaises(_a				):
             # Tries with a `device`
             self.model_abit.half()

        # Test if we did not break anything
        lowercase_					:       int     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				)

        lowercase_					:       Any     				=					self.model_fpaa.to(torch.floataa				)
        lowercase_					:       Any     				=					self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0				)       ,				max_new_tokens=10				)

        # Check this does not throw an error
        lowercase_					:       List[Any]     				=					self.model_fpaa.to('cpu'				)

        # Check this does not throw an error
        lowercase_					:       Optional[Any]     				=					self.model_fpaa.half()

        # Check this does not throw an error
        lowercase_					:       int     				=					self.model_fpaa.float()





   def 		_lowerCamelCase (self				)    ->     Any:
        lowercase_					:       int     				=					AutoModelForSeqaSeqLM.from_pretrained('t5-small'       ,				load_in_abit=_a       ,				device_map='auto'				)
        self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa				)



@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class       UpperCAmelCase__  (       unittest.TestCase			):






   """simple docstring"""
   @classmethod
   def 		_lowerCamelCase (cls				)    ->     Union[str, Any]:
        lowercase_					:       Optional[Any]     				=					't5-small'
        lowercase_					:       Dict     				=					'google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
        lowercase_					:       Union[str, Any]     				=					AutoTokenizer.from_pretrained(cls.model_name				)
        lowercase_					:       List[Any]     				=					'Translate in German: Hello, my dog is cute'
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        gc.collect()
        torch.cuda.empty_cache()
   def 		_lowerCamelCase (self				)    ->     Any:
        from transformers import TaForConditionalGeneration

        lowercase_					:       Union[str, Any]     				=					TaForConditionalGeneration._keep_in_fpaa_modules
        lowercase_					:       Union[str, Any]     				=					None

        # test with `t5-small`
        lowercase_					:       List[Any]     				=					TaForConditionalGeneration.from_pretrained(self.model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
        lowercase_					:       Union[str, Any]     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				).to(0				)
        lowercase_					:       Tuple     				=					model.generate(**_a				)

        # test with `flan-t5-small`
        lowercase_					:       Tuple     				=					TaForConditionalGeneration.from_pretrained(
            self.dense_act_model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
        lowercase_					:       List[str]     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				).to(0				)
        lowercase_					:       List[Any]     				=					model.generate(**_a				)
        lowercase_					:       List[Any]     				=					modules





   def 		_lowerCamelCase (self				)    ->     Dict:
        import bitsandbytes as bnb

        from transformers import TaForConditionalGeneration

        # test with `t5-small`
        lowercase_					:       Any     				=					TaForConditionalGeneration.from_pretrained(self.model_name       ,				load_in_abit=_a       ,				device_map='auto'				)

        # there was a bug with decoders - this test checks that it is fixed
        self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q       ,				bnb.nn.Linearabit				)				)

        lowercase_					:       Union[str, Any]     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				).to(0				)
        lowercase_					:       int     				=					model.generate(**_a				)

        # test with `flan-t5-small`
        lowercase_					:       List[str]     				=					TaForConditionalGeneration.from_pretrained(
            self.dense_act_model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
        lowercase_					:       Optional[int]     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				).to(0				)
        lowercase_					:       List[Any]     				=					model.generate(**_a				)



class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        super().setUp()
        # model_name
        lowercase_					:       Optional[int]     				=					'bigscience/bloom-560m'
        lowercase_					:       Any     				=					't5-small'

        # Different types of model

        lowercase_					:       Any     				=					AutoModel.from_pretrained(self.model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
        # Sequence classification model
        lowercase_					:       Dict     				=					AutoModelForSequenceClassification.from_pretrained(
            self.model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
        # CausalLM model
        lowercase_					:       Tuple     				=					AutoModelForCausalLM.from_pretrained(self.model_name       ,				load_in_abit=_a       ,				device_map='auto'				)
        # Seq2seq model
        lowercase_					:       int     				=					AutoModelForSeqaSeqLM.from_pretrained(
            self.seq_to_seq_name       ,				load_in_abit=_a       ,				device_map='auto'				)
   def 		_lowerCamelCase (self				)    ->     Tuple:
        del self.base_model
        del self.sequence_model
        del self.model_abit
        del self.seq_to_seq_model

        gc.collect()
        torch.cuda.empty_cache()





   def 		_lowerCamelCase (self				)    ->     Tuple:
        from bitsandbytes.nn import Paramsabit

        self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit				)

        # Other heads should be nn.Parameter
        self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter				)
        self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter				)
        self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter				)



class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   def 		_lowerCamelCase (self				)    ->     Optional[int]:
        super().setUp()
   def 		_lowerCamelCase (self				)    ->     str:
        del self.pipe

        gc.collect()
        torch.cuda.empty_cache()





   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       List[str]     				=					pipeline(
            'text-generation'       ,				model=self.model_name       ,				model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa}       ,				max_new_tokens=self.MAX_NEW_TOKENS       ,				)

        # Real second forward pass
        lowercase_					:       Tuple     				=					self.pipe(self.input_text				)
        self.assertIn(pipeline_output[0]['generated_text']       ,				self.EXPECTED_OUTPUTS				)



@require_torch_multi_gpu
class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   def 		_lowerCamelCase (self				)    ->     int:
        super().setUp()





   def 		_lowerCamelCase (self				)    ->     str:
        lowercase_					:       Tuple     				=					AutoModelForCausalLM.from_pretrained(
            self.model_name       ,				load_in_abit=_a       ,				device_map='balanced'				)

        # Check correct device map
        self.assertEqual(set(model_parallel.hf_device_map.values()				)       ,				{0, 1}				)

        # Check that inference pass works on the model
        lowercase_					:       str     				=					self.tokenizer(self.input_text       ,				return_tensors='pt'				)

        # Second real batch
        lowercase_					:       int     				=					model_parallel.generate(input_ids=encoded_input['input_ids'].to(0				)       ,				max_new_tokens=10				)
        self.assertIn(self.tokenizer.decode(output_parallel[0]       ,				skip_special_tokens=_a				)       ,				self.EXPECTED_OUTPUTS				)



class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   def 		_lowerCamelCase (self				)    ->     Any:
        lowercase_					:       Dict     				=					'facebook/opt-350m'
        super().setUp()





   def 		_lowerCamelCase (self				)    ->     str:
        if version.parse(importlib.metadata.version('bitsandbytes'				)				) < version.parse('0.37.0'				):
             return

        # Step 1: freeze all parameters
        lowercase_					:       Optional[Any]     				=					AutoModelForCausalLM.from_pretrained(self.model_name       ,				load_in_abit=_a				)

        self.assertEqual(set(model.hf_device_map.values()				)       ,				{torch.cuda.current_device()}				)

        for param in model.parameters():
             lowercase_					:       str     				=					False  # freeze the model - train adapters later
             if param.ndim == 1:
                  # cast the small parameters (e.g. layernorm) to fp32 for stability
                  lowercase_					:       Union[str, Any]     				=					param.data.to(torch.floataa				)

        # Step 2: add adapters
        for _, module in model.named_modules():
             if "OPTAttention" in repr(type(_a				)				):
                  lowercase_					:       int     				=					LoRALayer(module.q_proj       ,				rank=16				)
                  lowercase_					:       Tuple     				=					LoRALayer(module.k_proj       ,				rank=16				)
                  lowercase_					:       List[Any]     				=					LoRALayer(module.v_proj       ,				rank=16				)

        # Step 3: dummy batch
        lowercase_					:       Dict     				=					self.tokenizer('Test batch '       ,				return_tensors='pt'				).to(0				)

        # Step 4: Check if the gradient is not None
        with torch.cuda.amp.autocast():
             lowercase_					:       Tuple     				=					model.forward(**_a				)
             out.logits.norm().backward()

        for module in model.modules():
             if isinstance(_a       ,				_a				):
                  self.assertTrue(module.adapter[1].weight.grad is not None				)
                  self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0				)
             elif isinstance(_a       ,				nn.Embedding				):
                  self.assertTrue(module.weight.grad is None				)



class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   A							:       Optional[Any]     =    '''gpt2-xl'''
   A							:       List[Any]     =    3.3_191_854_854_152_187





'''simple docstring'''




from __future__ import annotations

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):

     # preprocessing the first row
     for i in range(1  ,       len(matrix[0]							)							):
          matrix[0][i] += matrix[0][i - 1]

     # preprocessing the first column
     for i in range(1  ,       len(SCREAMING_SNAKE_CASE_							)							):
          matrix[i][0] += matrix[i - 1][0]

     # updating the path cost for current position
     for i in range(1  ,       len(SCREAMING_SNAKE_CASE_							)							):
          for j in range(1  ,       len(matrix[0]							)							):
               matrix[i][j] += min(matrix[i - 1][j]  ,       matrix[i][j - 1]							)

     return matrix[-1][-1]


if __name__ == "__main__":
 import doctest

 doctest.testmod()




