

import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)
def      _lowerCAmelCase(	__A			):
  if model.config.model_type == "gpt2":
    return model.transformer.h[0].mlp.c_fc
  return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
      import torch
      import torch.nn as nn






      class 						__magic_name__    (       nn.Module    ):


       def __init__(						self		: Optional[int]			,						lowerCAmelCase__		: nn.Module			,						lowerCAmelCase__		: int    )    ->      Tuple:
         super().__init__()
         UpperCAmelCase					      =  module
         UpperCAmelCase					      =  nn.Sequential(
             nn.Linear(module.in_features			,						lowerCAmelCase__			,						bias=lowerCAmelCase__    )			,						nn.Linear(lowerCAmelCase__			,						module.out_features			,						bias=lowerCAmelCase__    )			,						)
         UpperCAmelCase					      =  (2.0 / (5 * min(module.in_features			,						module.out_features    ))) ** 0.5
         nn.init.normal_(self.adapter[0].weight			,						std=lowerCAmelCase__    )
         nn.init.zeros_(self.adapter[1].weight    )
         self.adapter.to(module.weight.device    )


       def _UpperCamelCase	(						self		: Dict			,						lowerCAmelCase__		: Tuple			,						*lowerCAmelCase__		: List[str]			,						**lowerCAmelCase__		: List[str]    )    ->      Optional[Any]:
         return self.module(lowerCAmelCase__			,						*lowerCAmelCase__			,						**lowerCAmelCase__    ) + self.adapter(lowerCAmelCase__    )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 						__magic_name__    (       unittest.TestCase    ):
 # We keep the constants inside the init function and model loading inside setUp function

 # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
 # Therefore here we use only bloom-1b3 to test our module
 UpperCAmelCase							    =						"""bigscience/bloom-1b7"""

 # Constant values
 UpperCAmelCase							    =						2.109_6595_5269_2574

 UpperCAmelCase							    =						"""Hello my name is"""
 UpperCAmelCase							    =						set()
 EXPECTED_OUTPUTS.add("""Hello my name is John and I am a professional photographer. I"""    )
 EXPECTED_OUTPUTS.add("""Hello my name is John.\nI am a friend of your father.\n"""    )
 EXPECTED_OUTPUTS.add("""Hello my name is John Doe, I am a student at the University"""    )
 UpperCAmelCase							    =						10


 def _UpperCamelCase	(						self		: List[Any]    )    ->      List[Any]:
   # Models and tokenizer
   UpperCAmelCase					      =  AutoTokenizer.from_pretrained(self.model_name    )






class 						__magic_name__    (       _snake_case    ):


 def _UpperCamelCase	(						self		: Union[str, Any]    )    ->      Dict:
   super().setUp()

   # Models and tokenizer
   UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(
       self.model_name			,						torch_dtype=torch.floataa			,						device_map="auto"    )
   UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      List[str]:
   del self.model_fpaa
   del self.model_abit

   gc.collect()
   torch.cuda.empty_cache()


 def _UpperCamelCase	(						self		: Any    )    ->      Union[str, Any]:
   UpperCAmelCase					      =  self.model_abit.config

   self.assertTrue(hasattr(lowerCAmelCase__			,						"quantization_config"    )    )

   UpperCAmelCase					      =  config.to_dict()
   UpperCAmelCase					      =  config.to_diff_dict()

   UpperCAmelCase					      =  config.to_json_string()


 def _UpperCamelCase	(						self		: Union[str, Any]    )    ->      Optional[int]:
   from bitsandbytes.nn import Paramsabit

   UpperCAmelCase					      =  self.model_fpaa.get_memory_footprint()
   UpperCAmelCase					      =  self.model_abit.get_memory_footprint()

   self.assertAlmostEqual(mem_fpaa / mem_abit			,						self.EXPECTED_RELATIVE_DIFFERENCE    )
   UpperCAmelCase					      =  get_some_linear_layer(self.model_abit    )
   self.assertTrue(linear.weight.__class__ == Paramsabit    )


 def _UpperCamelCase	(						self		: Tuple    )    ->      Optional[Any]:
   from transformers import TaPreTrainedModel

   self.model_fpaa.get_memory_footprint()
   self.model_abit.get_memory_footprint()

   for name, module in self.model_abit.named_modules():
     if isinstance(lowerCAmelCase__			,						torch.nn.Linear    ):
       if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
         # 4-bit parameters are packed in uint8 variables
         self.assertTrue(module.weight.dtype == torch.uinta    )


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      int:
   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    )
   UpperCAmelCase					      =  self.model_abit.generate(input_ids=encoded_input["input_ids"].to(0    )			,						max_new_tokens=1_0    )

   self.assertIn(self.tokenizer.decode(output_sequences[0]			,						skip_special_tokens=lowerCAmelCase__    )			,						self.EXPECTED_OUTPUTS    )


 def _UpperCamelCase	(						self		: Dict    )    ->      List[str]:
   UpperCAmelCase					      =  BitsAndBytesConfig()
   UpperCAmelCase					      =  True

   UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(
       self.model_name			,						quantization_config=lowerCAmelCase__			,						device_map="auto"    )

   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    )
   UpperCAmelCase					      =  model_abit_from_config.generate(
       input_ids=encoded_input["input_ids"].to(0    )			,						max_new_tokens=1_0    )

   self.assertIn(self.tokenizer.decode(output_sequences[0]			,						skip_special_tokens=lowerCAmelCase__    )			,						self.EXPECTED_OUTPUTS    )


 def _UpperCamelCase	(						self		: str    )    ->      Tuple:
   with self.assertRaises(lowerCAmelCase__    ), tempfile.TemporaryDirectory() as tmpdirname:
     self.model_abit.save_pretrained(lowerCAmelCase__    )


 def _UpperCamelCase	(						self		: int    )    ->      Tuple:
   UpperCAmelCase					      =  BitsAndBytesConfig()

   with self.assertRaises(lowerCAmelCase__    ):
     UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(
         self.model_name			,						quantization_config=lowerCAmelCase__			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"			,						bnb_abit_quant_type="nf4"			,						)


 def _UpperCamelCase	(						self		: Any    )    ->      Union[str, Any]:
   with self.assertRaises(lowerCAmelCase__    ):
     # Tries with `str`
     self.model_abit.to("cpu"    )

   with self.assertRaises(lowerCAmelCase__    ):
     # Tries with a `dtype``
     self.model_abit.to(torch.floataa    )

   with self.assertRaises(lowerCAmelCase__    ):
     # Tries with a `device`
     self.model_abit.to(torch.device("cuda:0"    )    )

   with self.assertRaises(lowerCAmelCase__    ):
     # Tries with a `device`
     self.model_abit.float()

   with self.assertRaises(lowerCAmelCase__    ):
     # Tries with a `device`
     self.model_abit.half()

   # Test if we did not break anything
   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    )

   UpperCAmelCase					      =  self.model_fpaa.to(torch.floataa    )
   UpperCAmelCase					      =  self.model_fpaa.generate(input_ids=encoded_input["input_ids"].to(0    )			,						max_new_tokens=1_0    )

   # Check this does not throw an error
   UpperCAmelCase					      =  self.model_fpaa.to("cpu"    )

   # Check this does not throw an error
   UpperCAmelCase					      =  self.model_fpaa.half()

   # Check this does not throw an error
   UpperCAmelCase					      =  self.model_fpaa.float()


 def _UpperCamelCase	(						self		: Any    )    ->      Dict:
   UpperCAmelCase					      =  AutoModelForSeqaSeqLM.from_pretrained("t5-small"			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa    )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 						__magic_name__    (       unittest.TestCase    ):


 @classmethod
 def _UpperCamelCase	(						cls		: str    )    ->      Any:
   UpperCAmelCase					      =  "t5-small"
   UpperCAmelCase					      =  "google/flan-t5-small"  # flan-t5 uses dense-act instead of dense-relu-dense
   UpperCAmelCase					      =  AutoTokenizer.from_pretrained(cls.model_name    )
   UpperCAmelCase					      =  "Translate in German: Hello, my dog is cute"


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      List[str]:
   gc.collect()
   torch.cuda.empty_cache()


 def _UpperCamelCase	(						self		: List[str]    )    ->      List[str]:
   from transformers import TaForConditionalGeneration

   UpperCAmelCase					      =  TaForConditionalGeneration._keep_in_fpaa_modules
   UpperCAmelCase					      =  None

   # test with `t5-small`
   UpperCAmelCase					      =  TaForConditionalGeneration.from_pretrained(self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    ).to(0    )
   UpperCAmelCase					      =  model.generate(**lowerCAmelCase__    )

   # test with `flan-t5-small`
   UpperCAmelCase					      =  TaForConditionalGeneration.from_pretrained(
       self.dense_act_model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    ).to(0    )
   UpperCAmelCase					      =  model.generate(**lowerCAmelCase__    )
   UpperCAmelCase					      =  modules


 def _UpperCamelCase	(						self		: Any    )    ->      int:
   import bitsandbytes as bnb

   from transformers import TaForConditionalGeneration

   # test with `t5-small`
   UpperCAmelCase					      =  TaForConditionalGeneration.from_pretrained(self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )

   # there was a bug with decoders - this test checks that it is fixed
   self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q			,						bnb.nn.Linearabit    )    )

   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    ).to(0    )
   UpperCAmelCase					      =  model.generate(**lowerCAmelCase__    )

   # test with `flan-t5-small`
   UpperCAmelCase					      =  TaForConditionalGeneration.from_pretrained(
       self.dense_act_model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    ).to(0    )
   UpperCAmelCase					      =  model.generate(**lowerCAmelCase__    )






class 						__magic_name__    (       _snake_case    ):


 def _UpperCamelCase	(						self		: str    )    ->      str:
   super().setUp()
   # model_name
   UpperCAmelCase					      =  "bigscience/bloom-560m"
   UpperCAmelCase					      =  "t5-small"

   # Different types of model

   UpperCAmelCase					      =  AutoModel.from_pretrained(self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   # Sequence classification model
   UpperCAmelCase					      =  AutoModelForSequenceClassification.from_pretrained(
       self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   # CausalLM model
   UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )
   # Seq2seq model
   UpperCAmelCase					      =  AutoModelForSeqaSeqLM.from_pretrained(
       self.seq_to_seq_name			,						load_in_abit=lowerCAmelCase__			,						device_map="auto"    )


 def _UpperCamelCase	(						self		: Optional[int]    )    ->      Optional[int]:
   del self.base_model
   del self.sequence_model
   del self.model_abit
   del self.seq_to_seq_model

   gc.collect()
   torch.cuda.empty_cache()


 def _UpperCamelCase	(						self		: Any    )    ->      Dict:
   from bitsandbytes.nn import Paramsabit

   self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit    )

   # Other heads should be nn.Parameter
   self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter    )
   self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter    )
   self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter    )






class 						__magic_name__    (       _snake_case    ):


 def _UpperCamelCase	(						self		: Dict    )    ->      Optional[int]:
   super().setUp()


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      Optional[Any]:
   del self.pipe

   gc.collect()
   torch.cuda.empty_cache()


 def _UpperCamelCase	(						self		: Optional[Any]    )    ->      Any:
   UpperCAmelCase					      =  pipeline(
       "text-generation"			,						model=self.model_name			,						model_kwargs={"device_map": "auto", "load_in_4bit": True, "torch_dtype": torch.floataa}			,						max_new_tokens=self.MAX_NEW_TOKENS			,						)

   # Real second forward pass
   UpperCAmelCase					      =  self.pipe(self.input_text    )
   self.assertIn(pipeline_output[0]["generated_text"]			,						self.EXPECTED_OUTPUTS    )






@require_torch_multi_gpu
class 						__magic_name__    (       _snake_case    ):


 def _UpperCamelCase	(						self		: List[str]    )    ->      List[str]:
   super().setUp()


 def _UpperCamelCase	(						self		: List[Any]    )    ->      Tuple:
   UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(
       self.model_name			,						load_in_abit=lowerCAmelCase__			,						device_map="balanced"    )

   # Check correct device map
   self.assertEqual(set(model_parallel.hf_device_map.values()    )			,						{0, 1}    )

   # Check that inference pass works on the model
   UpperCAmelCase					      =  self.tokenizer(self.input_text			,						return_tensors="pt"    )

   # Second real batch
   UpperCAmelCase					      =  model_parallel.generate(input_ids=encoded_input["input_ids"].to(0    )			,						max_new_tokens=1_0    )
   self.assertIn(self.tokenizer.decode(output_parallel[0]			,						skip_special_tokens=lowerCAmelCase__    )			,						self.EXPECTED_OUTPUTS    )






class 						__magic_name__    (       _snake_case    ):


 def _UpperCamelCase	(						self		: Dict    )    ->      Dict:
   UpperCAmelCase					      =  "facebook/opt-350m"
   super().setUp()


 def _UpperCamelCase	(						self		: Tuple    )    ->      int:
   if version.parse(importlib.metadata.version("bitsandbytes"    )    ) < version.parse("0.37.0"    ):
     return

   # Step 1: freeze all parameters
   UpperCAmelCase					      =  AutoModelForCausalLM.from_pretrained(self.model_name			,						load_in_abit=lowerCAmelCase__    )

   self.assertEqual(set(model.hf_device_map.values()    )			,						{torch.cuda.current_device()}    )

   for param in model.parameters():
     UpperCAmelCase					      =  False  # freeze the model - train adapters later
     if param.ndim == 1:
       # cast the small parameters (e.g. layernorm) to fp32 for stability
       UpperCAmelCase					      =  param.data.to(torch.floataa    )

        # Step 2: add adapters
   for _, module in model.named_modules():
     if "OPTAttention" in repr(type(lowerCAmelCase__    )    ):
       UpperCAmelCase					      =  LoRALayer(module.q_proj			,						rank=1_6    )
       UpperCAmelCase					      =  LoRALayer(module.k_proj			,						rank=1_6    )
       UpperCAmelCase					      =  LoRALayer(module.v_proj			,						rank=1_6    )

        # Step 3: dummy batch
   UpperCAmelCase					      =  self.tokenizer("Test batch "			,						return_tensors="pt"    ).to(0    )

   # Step 4: Check if the gradient is not None
   with torch.cuda.amp.autocast():
     UpperCAmelCase					      =  model.forward(**lowerCAmelCase__    )
     out.logits.norm().backward()

   for module in model.modules():
     if isinstance(lowerCAmelCase__			,						lowerCAmelCase__    ):
       self.assertTrue(module.adapter[1].weight.grad is not None    )
       self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0    )
     elif isinstance(lowerCAmelCase__			,						nn.Embedding    ):
       self.assertTrue(module.weight.grad is None    )






class 						__magic_name__    (       _snake_case    ):
 UpperCAmelCase							    =						"""gpt2-xl"""
 UpperCAmelCase							    =						3.3191_8548_5415_2187






def      _lowerCAmelCase(	__A   ,       __A   ,       __A			):
  if n == 0:
    return 1

  elif n % 2 == 1:
    return (binary_exponentiation(__A   ,       n - 1   ,       __A			) * a) % mod

  else:
    UpperCAmelCase					      =  binary_exponentiation(__A   ,       n / 2   ,       __A			)
    return (b * b) % mod


# a prime number
lowerCAmelCase__       = 701

lowerCAmelCase__       = 1000000000
lowerCAmelCase__       = 10

# using binary exponentiation function, O(log(p)):
print((a / b) % p == (a * binary_exponentiation(b, p - 2, p)) % p)





print((a / b) % p == (a * b ** (p - 2)) % p)



