





'''simple docstring'''



import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)




def    _lowerCAmelCase						(				lowerCamelCase_			:  int				):
			if model.config.model_type == "gpt2":
						return model.transformer.h[0].mlp.c_fc
			return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
	import torch
	import torch.nn as nn






	class 							__lowercase       (      nn.Module						):







		'''simple docstring'''

		def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase )							->							int:







					'''simple docstring'''





					super().__init__()
					__lowercase										=							module
					__lowercase										=							nn.Sequential(
					    nn.Linear(module.in_features     ,_lowerCamelCase     ,bias=_lowerCamelCase )     ,nn.Linear(_lowerCamelCase     ,module.out_features     ,bias=_lowerCamelCase )     ,)
					__lowercase										=							(2.0 / (5 * min(module.in_features     ,module.out_features ))) ** 0.5
					nn.init.normal_(self.adapter[0].weight     ,std=_lowerCamelCase )
					nn.init.zeros_(self.adapter[1].weight )
					self.adapter.to(module.weight.device )


		def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,*_lowerCamelCase     ,**_lowerCamelCase )							->							Dict:







					'''simple docstring'''





					return self.module(_lowerCamelCase     ,*_lowerCamelCase     ,**_lowerCamelCase ) + self.adapter(_lowerCamelCase )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	a    :				Any											=       "bigscience/bloom-1b7"

	# Constant values
	a    :				str											=       2.1_09_65_95_52_69_25_74

	a    :				Tuple											=       "Hello my name is"
	a    :				Any											=       set()
	EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I"						)
	EXPECTED_OUTPUTS.add("Hello my name is John.\nI am a friend of your father.\n"						)
	EXPECTED_OUTPUTS.add("Hello my name is John Doe, I am a student at the University"						)
	a    :				List[str]											=       10


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							AutoTokenizer.from_pretrained(self.model_name )






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				super().setUp()

				# Models and tokenizer
				__lowercase										=							AutoModelForCausalLM.from_pretrained(
				    self.model_name     ,torch_dtype=torch.floataa     ,device_map='''auto''' )
				__lowercase										=							AutoModelForCausalLM.from_pretrained(self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )


	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				del self.model_fpaa
				del self.model_abit

				gc.collect()
				torch.cuda.empty_cache()


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				__lowercase										=							self.model_abit.config

				self.assertTrue(hasattr(_lowerCamelCase     ,'''quantization_config''' ) )

				__lowercase										=							config.to_dict()
				__lowercase										=							config.to_diff_dict()

				__lowercase										=							config.to_json_string()


	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				from bitsandbytes.nn import Paramsabit

				__lowercase										=							self.model_fpaa.get_memory_footprint()
				__lowercase										=							self.model_abit.get_memory_footprint()

				self.assertAlmostEqual(mem_fpaa / mem_abit     ,self.EXPECTED_RELATIVE_DIFFERENCE )
				__lowercase										=							get_some_linear_layer(self.model_abit )
				self.assertTrue(linear.weight.__class__ == Paramsabit )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				from transformers import TaPreTrainedModel

				self.model_fpaa.get_memory_footprint()
				self.model_abit.get_memory_footprint()

				for name, module in self.model_abit.named_modules():
							if isinstance(_lowerCamelCase     ,torch.nn.Linear ):
										if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
													# 4-bit parameters are packed in uint8 variables
													self.assertTrue(module.weight.dtype == torch.uinta )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' )
				__lowercase										=							self.model_abit.generate(input_ids=encoded_input['''input_ids'''].to(0 )     ,max_new_tokens=10 )

				self.assertIn(self.tokenizer.decode(output_sequences[0]     ,skip_special_tokens=_lowerCamelCase )     ,self.EXPECTED_OUTPUTS )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							BitsAndBytesConfig()
				__lowercase										=							True

				__lowercase										=							AutoModelForCausalLM.from_pretrained(
				    self.model_name     ,quantization_config=_lowerCamelCase     ,device_map='''auto''' )

				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' )
				__lowercase										=							model_abit_from_config.generate(
				    input_ids=encoded_input['''input_ids'''].to(0 )     ,max_new_tokens=10 )

				self.assertIn(self.tokenizer.decode(output_sequences[0]     ,skip_special_tokens=_lowerCamelCase )     ,self.EXPECTED_OUTPUTS )


	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				with self.assertRaises(_lowerCamelCase ), tempfile.TemporaryDirectory() as tmpdirname:
							self.model_abit.save_pretrained(_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							BitsAndBytesConfig()

				with self.assertRaises(_lowerCamelCase ):
							__lowercase										=							AutoModelForCausalLM.from_pretrained(
							    self.model_name     ,quantization_config=_lowerCamelCase     ,load_in_abit=_lowerCamelCase     ,device_map='''auto'''     ,bnb_abit_quant_type='''nf4'''     ,)


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				with self.assertRaises(_lowerCamelCase ):
							# Tries with `str`
							self.model_abit.to('''cpu''' )

				with self.assertRaises(_lowerCamelCase ):
							# Tries with a `dtype``
							self.model_abit.to(torch.floataa )

				with self.assertRaises(_lowerCamelCase ):
							# Tries with a `device`
							self.model_abit.to(torch.device('''cuda:0''' ) )

				with self.assertRaises(_lowerCamelCase ):
							# Tries with a `device`
							self.model_abit.float()

				with self.assertRaises(_lowerCamelCase ):
							# Tries with a `device`
							self.model_abit.half()

				# Test if we did not break anything
				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' )

				__lowercase										=							self.model_fpaa.to(torch.floataa )
				__lowercase										=							self.model_fpaa.generate(input_ids=encoded_input['''input_ids'''].to(0 )     ,max_new_tokens=10 )

				# Check this does not throw an error
				__lowercase										=							self.model_fpaa.to('''cpu''' )

				# Check this does not throw an error
				__lowercase										=							self.model_fpaa.half()

				# Check this does not throw an error
				__lowercase										=							self.model_fpaa.float()


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							AutoModelForSeqaSeqLM.from_pretrained('''t5-small'''     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	@classmethod
	def 						_UpperCAmelCase     (cls )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							'''t5-small'''
				__lowercase										=							'''google/flan-t5-small'''  # flan-t5 uses dense-act instead of dense-relu-dense
				__lowercase										=							AutoTokenizer.from_pretrained(cls.model_name )
				__lowercase										=							'''Translate in German: Hello, my dog is cute'''


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				gc.collect()
				torch.cuda.empty_cache()


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				from transformers import TaForConditionalGeneration

				__lowercase										=							TaForConditionalGeneration._keep_in_fpaa_modules
				__lowercase										=							None

				# test with `t5-small`
				__lowercase										=							TaForConditionalGeneration.from_pretrained(self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' ).to(0 )
				__lowercase										=							model.generate(**_lowerCamelCase )

				# test with `flan-t5-small`
				__lowercase										=							TaForConditionalGeneration.from_pretrained(
				    self.dense_act_model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' ).to(0 )
				__lowercase										=							model.generate(**_lowerCamelCase )
				__lowercase										=							modules


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				import bitsandbytes as bnb

				from transformers import TaForConditionalGeneration

				# test with `t5-small`
				__lowercase										=							TaForConditionalGeneration.from_pretrained(self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )

				# there was a bug with decoders - this test checks that it is fixed
				self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q     ,bnb.nn.Linearabit ) )

				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' ).to(0 )
				__lowercase										=							model.generate(**_lowerCamelCase )

				# test with `flan-t5-small`
				__lowercase										=							TaForConditionalGeneration.from_pretrained(
				    self.dense_act_model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' ).to(0 )
				__lowercase										=							model.generate(**_lowerCamelCase )






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				super().setUp()
				# model_name
				__lowercase										=							'''bigscience/bloom-560m'''
				__lowercase										=							'''t5-small'''

				# Different types of model

				__lowercase										=							AutoModel.from_pretrained(self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				# Sequence classification model
				__lowercase										=							AutoModelForSequenceClassification.from_pretrained(
				    self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				# CausalLM model
				__lowercase										=							AutoModelForCausalLM.from_pretrained(self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )
				# Seq2seq model
				__lowercase										=							AutoModelForSeqaSeqLM.from_pretrained(
				    self.seq_to_seq_name     ,load_in_abit=_lowerCamelCase     ,device_map='''auto''' )


	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				del self.base_model
				del self.sequence_model
				del self.model_abit
				del self.seq_to_seq_model

				gc.collect()
				torch.cuda.empty_cache()


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				from bitsandbytes.nn import Paramsabit

				self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit )

				# Other heads should be nn.Parameter
				self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter )
				self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter )
				self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter )






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				super().setUp()


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				del self.pipe

				gc.collect()
				torch.cuda.empty_cache()


	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							pipeline(
				    '''text-generation'''     ,model=self.model_name     ,model_kwargs={'''device_map''': '''auto''', '''load_in_4bit''': True, '''torch_dtype''': torch.floataa}     ,max_new_tokens=self.MAX_NEW_TOKENS     ,)

				# Real second forward pass
				__lowercase										=							self.pipe(self.input_text )
				self.assertIn(pipeline_output[0]['''generated_text''']     ,self.EXPECTED_OUTPUTS )






@require_torch_multi_gpu
class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				super().setUp()


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							AutoModelForCausalLM.from_pretrained(
				    self.model_name     ,load_in_abit=_lowerCamelCase     ,device_map='''balanced''' )

				# Check correct device map
				self.assertEqual(set(model_parallel.hf_device_map.values() )     ,{0, 1} )

				# Check that inference pass works on the model
				__lowercase										=							self.tokenizer(self.input_text     ,return_tensors='''pt''' )

				# Second real batch
				__lowercase										=							model_parallel.generate(input_ids=encoded_input['''input_ids'''].to(0 )     ,max_new_tokens=10 )
				self.assertIn(self.tokenizer.decode(output_parallel[0]     ,skip_special_tokens=_lowerCamelCase )     ,self.EXPECTED_OUTPUTS )






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				__lowercase										=							'''facebook/opt-350m'''
				super().setUp()


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				if version.parse(importlib.metadata.version('''bitsandbytes''' ) ) < version.parse('''0.37.0''' ):
							return

				# Step 1: freeze all parameters
				__lowercase										=							AutoModelForCausalLM.from_pretrained(self.model_name     ,load_in_abit=_lowerCamelCase )

				self.assertEqual(set(model.hf_device_map.values() )     ,{torch.cuda.current_device()} )

				for param in model.parameters():
							__lowercase										=							False  # freeze the model - train adapters later
							if param.ndim == 1:
										# cast the small parameters (e.g. layernorm) to fp32 for stability
										__lowercase										=							param.data.to(torch.floataa )

        # Step 2: add adapters
				for _, module in model.named_modules():
							if "OPTAttention" in repr(type(_lowerCamelCase ) ):
										__lowercase										=							LoRALayer(module.q_proj     ,rank=16 )
										__lowercase										=							LoRALayer(module.k_proj     ,rank=16 )
										__lowercase										=							LoRALayer(module.v_proj     ,rank=16 )

        # Step 3: dummy batch
				__lowercase										=							self.tokenizer('''Test batch '''     ,return_tensors='''pt''' ).to(0 )

				# Step 4: Check if the gradient is not None
				with torch.cuda.amp.autocast():
							__lowercase										=							model.forward(**_lowerCamelCase )
							out.logits.norm().backward()

				for module in model.modules():
							if isinstance(_lowerCamelCase     ,_lowerCamelCase ):
										self.assertTrue(module.adapter[1].weight.grad is not None )
										self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0 )
							elif isinstance(_lowerCamelCase     ,nn.Embedding ):
										self.assertTrue(module.weight.grad is None )






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	a    :				Dict											=       "gpt2-xl"
	a    :				Union[str, Any]											=       3.31_91_85_48_54_15_21_87












'''simple docstring'''



from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


_SCREAMING_SNAKE_CASE					=    '''\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
'''

_SCREAMING_SNAKE_CASE					=    '''\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
'''

_SCREAMING_SNAKE_CASE					=    '''
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    "accuracy": Accuracy
    "f1": F1 score
    "pearson": Pearson Correlation
    "spearmanr": Spearman Correlation
    "matthews_correlation": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric(\'glue\', \'sst2\')  # \'sst2\' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> glue_metric = datasets.load_metric(\'glue\', \'mrpc\')  # \'mrpc\' or \'qqp\'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> glue_metric = datasets.load_metric(\'glue\', \'stsb\')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)})
    {\'pearson\': 1.0, \'spearmanr\': 1.0}

    >>> glue_metric = datasets.load_metric(\'glue\', \'cola\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''




def    _lowerCAmelCase						(				lowerCamelCase_			:  Any		,       lowerCamelCase_			:  int				):
			return float((preds == labels).mean()				)




def    _lowerCAmelCase						(				lowerCamelCase_			:  List[str]		,       lowerCamelCase_			:  str				):
			__lowercase										=							simple_accuracy(lowerCamelCase_		,       lowerCamelCase_				)
			__lowercase										=							float(fa_score(y_true=lowerCamelCase_		,       y_pred=lowerCamelCase_				)				)
			return {
			    "accuracy": acc,
			    "f1": fa,
			}




def    _lowerCAmelCase						(				lowerCamelCase_			:  List[Any]		,       lowerCamelCase_			:  Any				):
			__lowercase										=							float(pearsonr(lowerCamelCase_		,       lowerCamelCase_				)[0]				)
			__lowercase										=							float(spearmanr(lowerCamelCase_		,       lowerCamelCase_				)[0]				)
			return {
			    "pearson": pearson_corr,
			    "spearmanr": spearman_corr,
			}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,      _KWARGS_DESCRIPTION						)
class 							__lowercase       (      datasets.Metric						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				if self.config_name not in [
				    "sst2",
				    "mnli",
				    "mnli_mismatched",
				    "mnli_matched",
				    "cola",
				    "stsb",
				    "mrpc",
				    "qqp",
				    "qnli",
				    "rte",
				    "wnli",
				    "hans",
				]:
							raise KeyError(
							    '''You should supply a configuration name selected in '''
							    '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
							    '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]''' )
				return datasets.MetricInfo(
				    description=_DESCRIPTION     ,citation=_CITATION     ,inputs_description=_KWARGS_DESCRIPTION     ,features=datasets.Features(
				        {
				            '''predictions''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32''' ),
				            '''references''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32''' ),
				        } )     ,codebase_urls=[]     ,reference_urls=[]     ,format='''numpy'''     ,)


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							Dict:







				'''simple docstring'''





				if self.config_name == "cola":
							return {"matthews_correlation": matthews_corrcoef(_lowerCamelCase     ,_lowerCamelCase )}
				elif self.config_name == "stsb":
							return pearson_and_spearman(_lowerCamelCase     ,_lowerCamelCase )
				elif self.config_name in ["mrpc", "qqp"]:
							return acc_and_fa(_lowerCamelCase     ,_lowerCamelCase )
				elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
							return {"accuracy": simple_accuracy(_lowerCamelCase     ,_lowerCamelCase )}
				else:
							raise KeyError(
							    '''You should supply a configuration name selected in '''
							    '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
							    '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]''' )





