'''simple docstring'''

import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
       import bitsandbytes as bnb
       import torch
       import torch.nn as nn

       from ..pytorch_utils import ConvaD

if is_accelerate_available():
       from accelerate import init_empty_weights
       from accelerate.utils import find_tied_parameters

lowerCAmelCase_ : Optional[Any]									=      logging.get_logger(__name__)


def __a (	__lowerCamelCase			:						Any ,		__lowerCamelCase			:						Any ,		__lowerCamelCase			:						Any ,		__lowerCamelCase			:						Optional[int]=None ,		__lowerCamelCase			:						int=None				)	->							Any:
      '''simple docstring'''


      if "." in tensor_name:
            lowercase_     =      tensor_name.split("."				)
            for split in splits[:-1]:
                  lowercase_     =      getattr(__lowerCamelCase ,		__lowerCamelCase				)
                  if new_module is None:
                        raise ValueError(f'{module} has no attribute {split}.'				)
                  lowercase_     =      new_module
            lowercase_     =      splits[-1]

      if tensor_name not in module._parameters and tensor_name not in module._buffers:
            raise ValueError(f'{module} does not have a parameter or a buffer named {tensor_name}.'				)
      lowercase_     =      tensor_name in module._buffers
      lowercase_     =      getattr(__lowerCamelCase ,		__lowerCamelCase				)

      if old_value.device == torch.device("meta"				) and device not in ["meta", torch.device("meta"				)] and value is None:
            raise ValueError(f'{tensor_name} is on the meta device, we need a `value` to put in on {device}.'				)

      lowercase_     =      False
      lowercase_     =      False
      if is_buffer or not is_bitsandbytes_available():
            lowercase_     =      False
            lowercase_     =      False
      else:
            lowercase_     =      hasattr(bnb.nn ,		"Params4bit"				) and isinstance(module._parameters[tensor_name] ,		bnb.nn.Paramsabit				)
            lowercase_     =      isinstance(module._parameters[tensor_name] ,		bnb.nn.IntaParams				)

      if is_abit or is_abit:
            lowercase_     =      module._parameters[tensor_name]
            if param.device.type != "cuda":
                  if value is None:
                        lowercase_     =      old_value.to(__lowerCamelCase				)
                  elif isinstance(__lowerCamelCase ,		torch.Tensor				):
                        lowercase_     =      value.to("cpu"				)
                        if value.dtype == torch.inta:
                              lowercase_     =      version.parse(importlib.metadata.version("bitsandbytes"				)				) > version.parse(
                                  "0.37.2"				)
                              if not is_abit_serializable:
                                    raise ValueError(
                                        "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
                                        "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."				)
                  else:
                        lowercase_     =      torch.tensor(__lowerCamelCase ,		device="cpu"				)

                  # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
                  # Since weights are saved in the correct "orientation", we skip transposing when loading.
                  if issubclass(module.source_cls ,		__lowerCamelCase				) and fpaa_statistics is None:
                        lowercase_     =      new_value.T

                  lowercase_     =      old_value.__dict__
                  if is_abit:
                        lowercase_     =      bnb.nn.IntaParams(__lowerCamelCase ,		requires_grad=__lowerCamelCase ,		**__lowerCamelCase				).to(__lowerCamelCase				)
                  elif is_abit:
                        lowercase_     =      bnb.nn.Paramsabit(__lowerCamelCase ,		requires_grad=__lowerCamelCase ,		**__lowerCamelCase				).to(__lowerCamelCase				)

                  lowercase_     =      new_value
                  if fpaa_statistics is not None:
                        setattr(module.weight ,		"SCB" ,		fpaa_statistics.to(__lowerCamelCase				)				)

      else:
            if value is None:
                  lowercase_     =      old_value.to(__lowerCamelCase				)
            elif isinstance(__lowerCamelCase ,		torch.Tensor				):
                  lowercase_     =      value.to(__lowerCamelCase				)
            else:
                  lowercase_     =      torch.tensor(__lowerCamelCase ,		device=__lowerCamelCase				)

            if is_buffer:
                  lowercase_     =      new_value
            else:
                  lowercase_     =      nn.Parameter(__lowerCamelCase ,		requires_grad=old_value.requires_grad				)
                  lowercase_     =      new_value


def __a (	__lowerCamelCase			:						Tuple ,		__lowerCamelCase			:						List[str]=None ,		__lowerCamelCase			:						Optional[Any]=None ,		__lowerCamelCase			:						Tuple=None ,		__lowerCamelCase			:						Optional[int]=False				)	->							str:
      '''simple docstring'''


      for name, module in model.named_children():
            if current_key_name is None:
                  lowercase_     =      []
            current_key_name.append(__lowerCamelCase				)

            if (isinstance(__lowerCamelCase ,		nn.Linear				) or isinstance(__lowerCamelCase ,		__lowerCamelCase				)) and name not in modules_to_not_convert:
                  # Check if the current key is not in the `modules_to_not_convert`
                  if not any(key in ".".join(__lowerCamelCase				) for key in modules_to_not_convert				):
                        with init_empty_weights():
                              if isinstance(__lowerCamelCase ,		__lowerCamelCase				):
                                    lowercase_ ,							lowercase_     =      module.weight.shape
                              else:
                                    lowercase_     =      module.in_features
                                    lowercase_     =      module.out_features

                              if quantization_config.quantization_method() == "llm_int8":
                                    lowercase_     =      bnb.nn.LinearabitLt(
                                        __lowerCamelCase ,		__lowerCamelCase ,		module.bias is not None ,		has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight ,		threshold=quantization_config.llm_inta_threshold ,		)
                                    lowercase_     =      True
                              else:
                                    if (
                                        quantization_config.llm_inta_skip_modules is not None
                                        and name in quantization_config.llm_inta_skip_modules
                                    ):
                                          pass
                                    else:
                                          lowercase_     =      bnb.nn.Linearabit(
                                              __lowerCamelCase ,		__lowerCamelCase ,		module.bias is not None ,		quantization_config.bnb_abit_compute_dtype ,		compress_statistics=quantization_config.bnb_abit_use_double_quant ,		quant_type=quantization_config.bnb_abit_quant_type ,		)
                                          lowercase_     =      True
                    # Store the module class in case we need to transpose the weight later
                              lowercase_     =      type(__lowerCamelCase				)
                              # Force requires grad to False to avoid unexpected errors
                              model._modules[name].requires_grad_(__lowerCamelCase				)
            if len(list(module.children()				)				) > 0:
                  lowercase_ ,							lowercase_     =      _replace_with_bnb_linear(
                      __lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		has_been_replaced=__lowerCamelCase ,		)
            # Remove the last key for recursion
            current_key_name.pop(-1				)
      return model, has_been_replaced


def __a (	__lowerCamelCase			:						Dict ,		__lowerCamelCase			:						str=None ,		__lowerCamelCase			:						List[Any]=None ,		__lowerCamelCase			:						List[str]=None				)	->							Tuple:
      '''simple docstring'''


      lowercase_     =      ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert
      lowercase_ ,							lowercase_     =      _replace_with_bnb_linear(
          __lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase				)

      if not has_been_replaced:
            logger.warning(
                "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
                " Please double check your model architecture, or submit an issue on github if you think this is"
                " a bug."				)

      return model


def __a (	*__lowerCamelCase			:						Optional[int] ,		**__lowerCamelCase			:						Optional[int]				)	->							Optional[Any]:
      '''simple docstring'''


      warnings.warn(
          "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead" ,		__lowerCamelCase ,		)
      return replace_with_bnb_linear(*__lowerCamelCase ,		**__lowerCamelCase				)


def __a (	*__lowerCamelCase			:						List[Any] ,		**__lowerCamelCase			:						Tuple				)	->							List[str]:
      '''simple docstring'''


      warnings.warn(
          "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead" ,		__lowerCamelCase ,		)
      return set_module_quantized_tensor_to_device(*__lowerCamelCase ,		**__lowerCamelCase				)






def __a (	__lowerCamelCase			:						Union[str, Any]				)	->							Tuple:
      '''simple docstring'''


      lowercase_     =      deepcopy(__lowerCamelCase				)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
      tied_model.tie_weights()

      lowercase_     =      find_tied_parameters(__lowerCamelCase				)
      # For compatibility with Accelerate < 0.18
      if isinstance(__lowerCamelCase ,		__lowerCamelCase				):
            lowercase_     =      sum(list(tied_params.values()				) ,		[]				) + list(tied_params.keys()				)
      else:
            lowercase_     =      sum(__lowerCamelCase ,		[]				)
      lowercase_     =      len(__lowerCamelCase				) > 0

      # Check if it is a base model
      lowercase_     =      not hasattr(__lowerCamelCase ,		model.base_model_prefix				)

      # Ignore this for base models (BertModel, GPT2Model, etc.)
      if (not has_tied_params) and is_base_model:
            return []

      # otherwise they have an attached head
      lowercase_     =      list(model.named_children()				)
      lowercase_     =      [list_modules[-1][0]]

      # add last module together with tied weights
      lowercase_     =      set(__lowerCamelCase				) - set(__lowerCamelCase				)
      lowercase_     =      list(set(__lowerCamelCase				)				) + list(__lowerCamelCase				)

      # remove ".weight" from the keys
      lowercase_     =      [".weight", ".bias"]
      lowercase_     =      []
      for name in list_untouched:
            for name_to_remove in names_to_remove:
                  if name_to_remove in name:
                        lowercase_     =      name.replace(__lowerCamelCase ,		""				)
            filtered_module_names.append(__lowerCamelCase				)

      return filtered_module_names







'''simple docstring'''

import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
           CTRLForSequenceClassification,
           CTRLLMHeadModel,
           CTRLModel,
       )





class 							lowercase						:


   def __init__(   self				:	List[Any]   ,		__lowerCAmelCase				:	Optional[Any]   ,		__lowerCAmelCase				:	Any=14   ,		__lowerCAmelCase				:	Optional[Any]=7   ,		__lowerCAmelCase				:	List[str]=True   ,		__lowerCAmelCase				:	Optional[int]=True   ,		__lowerCAmelCase				:	Optional[Any]=True   ,		__lowerCAmelCase				:	Optional[Any]=True   ,		__lowerCAmelCase				:	Optional[Any]=True   ,		__lowerCAmelCase				:	Any=99   ,		__lowerCAmelCase				:	Tuple=32   ,		__lowerCAmelCase				:	Union[str, Any]=5   ,		__lowerCAmelCase				:	str=4   ,		__lowerCAmelCase				:	Dict=37   ,		__lowerCAmelCase				:	Optional[int]="gelu"   ,		__lowerCAmelCase				:	int=0.1   ,		__lowerCAmelCase				:	Tuple=0.1   ,		__lowerCAmelCase				:	str=512   ,		__lowerCAmelCase				:	Optional[Any]=16   ,		__lowerCAmelCase				:	Optional[int]=2   ,		__lowerCAmelCase				:	str=0.02   ,		__lowerCAmelCase				:	str=3   ,		__lowerCAmelCase				:	Tuple=4   ,		__lowerCAmelCase				:	Union[str, Any]=None   ,		)      ->					List[Any]:
         lowercase_     =      parent
         lowercase_     =      batch_size
         lowercase_     =      seq_length
         lowercase_     =      is_training
         lowercase_     =      use_token_type_ids
         lowercase_     =      use_input_mask
         lowercase_     =      use_labels
         lowercase_     =      use_mc_token_ids
         lowercase_     =      vocab_size
         lowercase_     =      hidden_size
         lowercase_     =      num_hidden_layers
         lowercase_     =      num_attention_heads
         lowercase_     =      intermediate_size
         lowercase_     =      hidden_act
         lowercase_     =      hidden_dropout_prob
         lowercase_     =      attention_probs_dropout_prob
         lowercase_     =      max_position_embeddings
         lowercase_     =      type_vocab_size
         lowercase_     =      type_sequence_label_size
         lowercase_     =      initializer_range
         lowercase_     =      num_labels
         lowercase_     =      num_choices
         lowercase_     =      scope
         lowercase_     =      self.vocab_size - 1


   def __UpperCAmelCase       (   self				:	Tuple)      ->					int:
         lowercase_     =      ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size)

         lowercase_     =      None
         if self.use_input_mask:
               lowercase_     =      random_attention_mask([self.batch_size, self.seq_length])

         lowercase_     =      None
         if self.use_token_type_ids:
               lowercase_     =      ids_tensor([self.batch_size, self.seq_length]   ,		self.type_vocab_size)

         lowercase_     =      None
         if self.use_mc_token_ids:
               lowercase_     =      ids_tensor([self.batch_size, self.num_choices]   ,		self.seq_length)

         lowercase_     =      None
         lowercase_     =      None
         lowercase_     =      None
         if self.use_labels:
               lowercase_     =      ids_tensor([self.batch_size]   ,		self.type_sequence_label_size)
               lowercase_     =      ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels)
               lowercase_     =      ids_tensor([self.batch_size]   ,		self.num_choices)

         lowercase_     =      self.get_config()

         lowercase_     =      ids_tensor([self.num_hidden_layers, self.num_attention_heads]   ,		2)

         return (
             config,
             input_ids,
             input_mask,
             head_mask,
             token_type_ids,
             mc_token_ids,
             sequence_labels,
             token_labels,
             choice_labels,
         )


   def __UpperCAmelCase       (   self				:	Union[str, Any])      ->					int:
         return CTRLConfig(
             vocab_size=self.vocab_size   ,		n_embd=self.hidden_size   ,		n_layer=self.num_hidden_layers   ,		n_head=self.num_attention_heads   ,		n_positions=self.max_position_embeddings   ,		pad_token_id=self.pad_token_id   ,		)


   def __UpperCAmelCase       (   self				:	Union[str, Any]   ,		__lowerCAmelCase				:	Optional[Any]   ,		__lowerCAmelCase				:	Union[str, Any]   ,		__lowerCAmelCase				:	Union[str, Any]   ,		__lowerCAmelCase				:	Tuple   ,		__lowerCAmelCase				:	Union[str, Any]   ,		*__lowerCAmelCase				:	Tuple)      ->					List[str]:
         lowercase_     =      CTRLModel(config=__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()

         model(__lowerCAmelCase   ,		token_type_ids=__lowerCAmelCase   ,		head_mask=__lowerCAmelCase)
         model(__lowerCAmelCase   ,		token_type_ids=__lowerCAmelCase)
         lowercase_     =      model(__lowerCAmelCase)
         self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size))
         self.parent.assertEqual(len(result.past_key_values)   ,		config.n_layer)


   def __UpperCAmelCase       (   self				:	List[Any]   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Union[str, Any]   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	int   ,		__lowerCAmelCase				:	Dict   ,		*__lowerCAmelCase				:	int)      ->					int:
         lowercase_     =      CTRLLMHeadModel(__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()

         lowercase_     =      model(__lowerCAmelCase   ,		token_type_ids=__lowerCAmelCase   ,		labels=__lowerCAmelCase)
         self.parent.assertEqual(result.loss.shape   ,		())
         self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size))


   def __UpperCAmelCase       (   self				:	Dict)      ->					int:
         lowercase_     =      self.prepare_config_and_inputs()

         (
             (
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							(
             lowercase_
         ) ,							
         )     =      config_and_inputs

         lowercase_     =      {"input_ids": input_ids, "token_type_ids": token_type_ids, "head_mask": head_mask}

         return config, inputs_dict


   def __UpperCAmelCase       (   self				:	Any   ,		__lowerCAmelCase				:	Optional[int]   ,		__lowerCAmelCase				:	Optional[Any]   ,		__lowerCAmelCase				:	int   ,		__lowerCAmelCase				:	List[str]   ,		*__lowerCAmelCase				:	List[Any])      ->					int:
         lowercase_     =      self.num_labels
         lowercase_     =      CTRLForSequenceClassification(__lowerCAmelCase)
         model.to(__lowerCAmelCase)
         model.eval()
         lowercase_     =      ids_tensor([self.batch_size]   ,		self.type_sequence_label_size)
         lowercase_     =      model(__lowerCAmelCase   ,		token_type_ids=__lowerCAmelCase   ,		labels=__lowerCAmelCase)
         self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_labels))





@require_torch
class 							lowercase						(						__lowerCamelCase ,				__lowerCamelCase ,				__lowerCamelCase ,				unittest.TestCase ):
   lowerCamelCase_    					=(CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
   lowerCamelCase_    					=(CTRLLMHeadModel,) if is_torch_available() else ()
   lowerCamelCase_    					=(
       {
           'feature-extraction': CTRLModel,
           'text-classification': CTRLForSequenceClassification,
           'text-generation': CTRLLMHeadModel,
           'zero-shot': CTRLForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   lowerCamelCase_    					=True
   lowerCamelCase_    					=False
   lowerCamelCase_    					=False


   def __UpperCAmelCase       (   self				:	Tuple   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	Dict   ,		__lowerCAmelCase				:	str   ,		__lowerCAmelCase				:	List[Any]   ,		__lowerCAmelCase				:	int)      ->					Optional[int]:
         if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
               # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
               # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
               # config could not be created.
               return True

         return False


   def __UpperCAmelCase       (   self				:	str)      ->					Tuple:
         lowercase_     =      CTRLModelTester(self)
         lowercase_     =      ConfigTester(self   ,		config_class=__lowerCAmelCase   ,		n_embd=37)


   def __UpperCAmelCase       (   self				:	Dict)      ->					Optional[Any]:
         super().tearDown()
         # clean-up as much as possible GPU memory occupied by PyTorch
         gc.collect()
         torch.cuda.empty_cache()


   def __UpperCAmelCase       (   self				:	str)      ->					List[Any]:
         self.config_tester.run_common_tests()


   def __UpperCAmelCase       (   self				:	Any)      ->					List[Any]:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_ctrl_model(*__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Tuple)      ->					Dict:
         lowercase_     =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head_model(*__lowerCAmelCase)


   @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests.")
   def __UpperCAmelCase       (   self				:	Union[str, Any])      ->					Optional[Any]:
         pass


   @slow
   def __UpperCAmelCase       (   self				:	Dict)      ->					List[str]:
         for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               lowercase_     =      CTRLModel.from_pretrained(__lowerCAmelCase)
               self.assertIsNotNone(__lowerCAmelCase)


   @unittest.skip("The model doesn't support left padding")  # and it's not used enough to be worth fixing :)
   def __UpperCAmelCase       (   self				:	Optional[Any])      ->					Dict:
         pass





@require_torch
class 							lowercase						(						unittest.TestCase ):


   def __UpperCAmelCase       (   self				:	Optional[int])      ->					Any:
         super().tearDown()
         # clean-up as much as possible GPU memory occupied by PyTorch
         gc.collect()
         torch.cuda.empty_cache()


   @slow
   def __UpperCAmelCase       (   self				:	int)      ->					Any:
         lowercase_     =      CTRLLMHeadModel.from_pretrained("ctrl")
         model.to(__lowerCAmelCase)
         lowercase_     =      torch.tensor(
             [[1_1859, 0, 1611, 8]]   ,		dtype=torch.long   ,		device=__lowerCAmelCase)  # Legal the president is
         lowercase_     =      [
             1_1859,
             0,
             1611,
             8,
             5,
             150,
             2_6449,
             2,
             19,
             348,
             469,
             3,
             2595,
             48,
             2_0740,
             24_6533,
             24_6533,
             19,
             30,
             5,
         ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

         lowercase_     =      model.generate(__lowerCAmelCase   ,		do_sample=__lowerCAmelCase)
         self.assertListEqual(output_ids[0].tolist()   ,		__lowerCAmelCase)






