import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
       import bitsandbytes as bnb
       import torch
       import torch.nn as nn

       from ..pytorch_utils import ConvaD

if is_accelerate_available():
       from accelerate import init_empty_weights
       from accelerate.utils import find_tied_parameters

lowerCAmelCase_ : List[Any]						  =      logging.get_logger(__name__)


def __a (	__lowerCamelCase			:						Dict ,		__lowerCamelCase			:						Dict ,		__lowerCamelCase			:						str ,		__lowerCamelCase			:						Tuple=None ,		__lowerCamelCase			:						List[str]=None				)	->							Union[str, Any]:
      '''simple docstring'''


      if "." in tensor_name:
            lowercase_     =      tensor_name.split("."				)
            for split in splits[:-1]:
                  lowercase_     =      getattr(__lowerCamelCase ,		__lowerCamelCase				)
                  if new_module is None:
                        raise ValueError(f'{module} has no attribute {split}.'				)
                  lowercase_     =      new_module
            lowercase_     =      splits[-1]

      if tensor_name not in module._parameters and tensor_name not in module._buffers:
            raise ValueError(f'{module} does not have a parameter or a buffer named {tensor_name}.'				)
      lowercase_     =      tensor_name in module._buffers
      lowercase_     =      getattr(__lowerCamelCase ,		__lowerCamelCase				)

      if old_value.device == torch.device("meta"				) and device not in ["meta", torch.device("meta"				)] and value is None:
            raise ValueError(f'{tensor_name} is on the meta device, we need a `value` to put in on {device}.'				)

      lowercase_     =      False
      lowercase_     =      False
      if is_buffer or not is_bitsandbytes_available():
            lowercase_     =      False
            lowercase_     =      False
      else:
            lowercase_     =      hasattr(bnb.nn ,		"Params4bit"				) and isinstance(module._parameters[tensor_name] ,		bnb.nn.Paramsabit				)
            lowercase_     =      isinstance(module._parameters[tensor_name] ,		bnb.nn.IntaParams				)

      if is_abit or is_abit:
            lowercase_     =      module._parameters[tensor_name]
            if param.device.type != "cuda":
                  if value is None:
                        lowercase_     =      old_value.to(__lowerCamelCase				)
                  elif isinstance(__lowerCamelCase ,		torch.Tensor				):
                        lowercase_     =      value.to("cpu"				)
                        if value.dtype == torch.inta:
                              lowercase_     =      version.parse(importlib.metadata.version("bitsandbytes"				)				) > version.parse(
                                  "0.37.2"				)
                              if not is_abit_serializable:
                                    raise ValueError(
                                        "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
                                        "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."				)
                  else:
                        lowercase_     =      torch.tensor(__lowerCamelCase ,		device="cpu"				)

                  # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
                  # Since weights are saved in the correct "orientation", we skip transposing when loading.
                  if issubclass(module.source_cls ,		__lowerCamelCase				) and fpaa_statistics is None:
                        lowercase_     =      new_value.T

                  lowercase_     =      old_value.__dict__
                  if is_abit:
                        lowercase_     =      bnb.nn.IntaParams(__lowerCamelCase ,		requires_grad=__lowerCamelCase ,		**__lowerCamelCase				).to(__lowerCamelCase				)
                  elif is_abit:
                        lowercase_     =      bnb.nn.Paramsabit(__lowerCamelCase ,		requires_grad=__lowerCamelCase ,		**__lowerCamelCase				).to(__lowerCamelCase				)

                  lowercase_     =      new_value
                  if fpaa_statistics is not None:
                        setattr(module.weight ,		"SCB" ,		fpaa_statistics.to(__lowerCamelCase				)				)

      else:
            if value is None:
                  lowercase_     =      old_value.to(__lowerCamelCase				)
            elif isinstance(__lowerCamelCase ,		torch.Tensor				):
                  lowercase_     =      value.to(__lowerCamelCase				)
            else:
                  lowercase_     =      torch.tensor(__lowerCamelCase ,		device=__lowerCamelCase				)

            if is_buffer:
                  lowercase_     =      new_value
            else:
                  lowercase_     =      nn.Parameter(__lowerCamelCase ,		requires_grad=old_value.requires_grad				)
                  lowercase_     =      new_value


def __a (	__lowerCamelCase			:						Tuple ,		__lowerCamelCase			:						Optional[Any]=None ,		__lowerCamelCase			:						int=None ,		__lowerCamelCase			:						int=None ,		__lowerCamelCase			:						List[Any]=False				)	->							Any:
      '''simple docstring'''


      for name, module in model.named_children():
            if current_key_name is None:
                  lowercase_     =      []
            current_key_name.append(__lowerCamelCase				)

            if (isinstance(__lowerCamelCase ,		nn.Linear				) or isinstance(__lowerCamelCase ,		__lowerCamelCase				)) and name not in modules_to_not_convert:
                  # Check if the current key is not in the `modules_to_not_convert`
                  if not any(key in ".".join(__lowerCamelCase				) for key in modules_to_not_convert				):
                        with init_empty_weights():
                              if isinstance(__lowerCamelCase ,		__lowerCamelCase				):
                                    lowercase_ ,							lowercase_     =      module.weight.shape
                              else:
                                    lowercase_     =      module.in_features
                                    lowercase_     =      module.out_features

                              if quantization_config.quantization_method() == "llm_int8":
                                    lowercase_     =      bnb.nn.LinearabitLt(
                                        __lowerCamelCase ,		__lowerCamelCase ,		module.bias is not None ,		has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight ,		threshold=quantization_config.llm_inta_threshold ,		)
                                    lowercase_     =      True
                              else:
                                    if (
                                        quantization_config.llm_inta_skip_modules is not None
                                        and name in quantization_config.llm_inta_skip_modules
                                    ):
                                          pass
                                    else:
                                          lowercase_     =      bnb.nn.Linearabit(
                                              __lowerCamelCase ,		__lowerCamelCase ,		module.bias is not None ,		quantization_config.bnb_abit_compute_dtype ,		compress_statistics=quantization_config.bnb_abit_use_double_quant ,		quant_type=quantization_config.bnb_abit_quant_type ,		)
                                          lowercase_     =      True
                    # Store the module class in case we need to transpose the weight later
                              lowercase_     =      type(__lowerCamelCase				)
                              # Force requires grad to False to avoid unexpected errors
                              model._modules[name].requires_grad_(__lowerCamelCase				)
            if len(list(module.children()				)				) > 0:
                  lowercase_ ,							lowercase_     =      _replace_with_bnb_linear(
                      __lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		has_been_replaced=__lowerCamelCase ,		)
            # Remove the last key for recursion
            current_key_name.pop(-1				)
      return model, has_been_replaced


def __a (	__lowerCamelCase			:						Union[str, Any] ,		__lowerCamelCase			:						List[str]=None ,		__lowerCamelCase			:						Any=None ,		__lowerCamelCase			:						Optional[int]=None				)	->							Optional[int]:
      '''simple docstring'''


      lowercase_     =      ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert
      lowercase_ ,							lowercase_     =      _replace_with_bnb_linear(
          __lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase				)

      if not has_been_replaced:
            logger.warning(
                "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
                " Please double check your model architecture, or submit an issue on github if you think this is"
                " a bug."				)

      return model


def __a (	*__lowerCamelCase			:						Dict ,		**__lowerCamelCase			:						int				)	->							Dict:
      '''simple docstring'''


      warnings.warn(
          "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead" ,		__lowerCamelCase ,		)
      return replace_with_bnb_linear(*__lowerCamelCase ,		**__lowerCamelCase				)


def __a (	*__lowerCamelCase			:						Union[str, Any] ,		**__lowerCamelCase			:						List[Any]				)	->							Optional[int]:
      '''simple docstring'''


      warnings.warn(
          "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead" ,		__lowerCamelCase ,		)
      return set_module_quantized_tensor_to_device(*__lowerCamelCase ,		**__lowerCamelCase				)






def __a (	__lowerCamelCase			:						List[Any]				)	->							Any:
      '''simple docstring'''


      lowercase_     =      deepcopy(__lowerCamelCase				)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
      tied_model.tie_weights()

      lowercase_     =      find_tied_parameters(__lowerCamelCase				)
      # For compatibility with Accelerate < 0.18
      if isinstance(__lowerCamelCase ,		__lowerCamelCase				):
            lowercase_     =      sum(list(tied_params.values()				) ,		[]				) + list(tied_params.keys()				)
      else:
            lowercase_     =      sum(__lowerCamelCase ,		[]				)
      lowercase_     =      len(__lowerCamelCase				) > 0

      # Check if it is a base model
      lowercase_     =      not hasattr(__lowerCamelCase ,		model.base_model_prefix				)

      # Ignore this for base models (BertModel, GPT2Model, etc.)
      if (not has_tied_params) and is_base_model:
            return []

      # otherwise they have an attached head
      lowercase_     =      list(model.named_children()				)
      lowercase_     =      [list_modules[-1][0]]

      # add last module together with tied weights
      lowercase_     =      set(__lowerCamelCase				) - set(__lowerCamelCase				)
      lowercase_     =      list(set(__lowerCamelCase				)				) + list(__lowerCamelCase				)

      # remove ".weight" from the keys
      lowercase_     =      [".weight", ".bias"]
      lowercase_     =      []
      for name in list_untouched:
            for name_to_remove in names_to_remove:
                  if name_to_remove in name:
                        lowercase_     =      name.replace(__lowerCamelCase ,		""				)
            filtered_module_names.append(__lowerCamelCase				)

      return filtered_module_names







'''simple docstring'''

import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_ : List[Any]						  =      get_tests_dir("fixtures/test_sentencepiece.model")





@require_sentencepiece
@require_tokenizers
class 							lowercase						(						__lowerCamelCase ,				unittest.TestCase ):
   lowerCamelCase_    					=ReformerTokenizer
   lowerCamelCase_    					=ReformerTokenizerFast
   lowerCamelCase_    					=True
   lowerCamelCase_    					=False
   lowerCamelCase_    					=True


   def __UpperCAmelCase       (   self				:	Optional[Any])      ->					Optional[Any]:
         super().setUp()

         lowercase_     =      ReformerTokenizer(__lowerCAmelCase   ,		keep_accents=__lowerCAmelCase)
         tokenizer.save_pretrained(self.tmpdirname)


   def __UpperCAmelCase       (   self				:	int)      ->					int:
         lowercase_     =      "<s>"
         lowercase_     =      1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase)   ,		__lowerCAmelCase)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase)   ,		__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Tuple)      ->					Optional[int]:
         lowercase_     =      list(self.get_tokenizer().get_vocab().keys())

         self.assertEqual(vocab_keys[0]   ,		"<unk>")
         self.assertEqual(vocab_keys[1]   ,		"<s>")
         self.assertEqual(vocab_keys[-1]   ,		"j")
         self.assertEqual(len(__lowerCAmelCase)   ,		1000)


   def __UpperCAmelCase       (   self				:	Union[str, Any])      ->					Optional[Any]:
         self.assertEqual(self.get_tokenizer().vocab_size   ,		1000)


   def __UpperCAmelCase       (   self				:	List[Any])      ->					List[Any]:
         if not self.test_rust_tokenizer:
               return

         lowercase_     =      self.get_tokenizer()
         lowercase_     =      self.get_rust_tokenizer()

         lowercase_     =      "I was born in 92000, and this is falsé."

         lowercase_     =      tokenizer.tokenize(__lowerCAmelCase)
         lowercase_     =      rust_tokenizer.tokenize(__lowerCAmelCase)
         self.assertListEqual(__lowerCAmelCase   ,		__lowerCAmelCase)

         lowercase_     =      tokenizer.encode(__lowerCAmelCase   ,		add_special_tokens=__lowerCAmelCase)
         lowercase_     =      rust_tokenizer.encode(__lowerCAmelCase   ,		add_special_tokens=__lowerCAmelCase)
         self.assertListEqual(__lowerCAmelCase   ,		__lowerCAmelCase)

         lowercase_     =      self.get_rust_tokenizer()
         lowercase_     =      tokenizer.encode(__lowerCAmelCase)
         lowercase_     =      rust_tokenizer.encode(__lowerCAmelCase)
         self.assertListEqual(__lowerCAmelCase   ,		__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	Tuple   ,		__lowerCAmelCase				:	List[str]=15)      ->					Tuple:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                     lowercase_     =      self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase   ,		**__lowerCAmelCase)

                     # Simple input
                     lowercase_     =      "This is a simple input"
                     lowercase_     =      ["This is a simple input 1", "This is a simple input 2"]
                     lowercase_     =      ("This is a simple input", "This is a pair")
                     lowercase_     =      [
                         ("This is a simple input 1", "This is a simple input 2"),
                         ("This is a simple pair 1", "This is a simple pair 2"),
                     ]

                     # Simple input tests
                     self.assertRaises(__lowerCAmelCase   ,		tokenizer_r.encode   ,		__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		padding="max_length")

                     # Simple input
                     self.assertRaises(__lowerCAmelCase   ,		tokenizer_r.encode_plus   ,		__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		padding="max_length")

                     # Simple input
                     self.assertRaises(
                         __lowerCAmelCase   ,		tokenizer_r.batch_encode_plus   ,		__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		padding="max_length"   ,		)

                     # Pair input
                     self.assertRaises(__lowerCAmelCase   ,		tokenizer_r.encode   ,		__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		padding="max_length")

                     # Pair input
                     self.assertRaises(__lowerCAmelCase   ,		tokenizer_r.encode_plus   ,		__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		padding="max_length")

                     # Pair input
                     self.assertRaises(
                         __lowerCAmelCase   ,		tokenizer_r.batch_encode_plus   ,		__lowerCAmelCase   ,		max_length=__lowerCAmelCase   ,		padding="max_length"   ,		)


   def __UpperCAmelCase       (   self				:	Tuple)      ->					Tuple:
         pass


   def __UpperCAmelCase       (   self				:	Any)      ->					Optional[Any]:
         lowercase_     =      ReformerTokenizer(__lowerCAmelCase   ,		keep_accents=__lowerCAmelCase)

         lowercase_     =      tokenizer.tokenize("This is a test")
         self.assertListEqual(__lowerCAmelCase   ,		["▁This", "▁is", "▁a", "▁t", "est"])

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__lowerCAmelCase)   ,		[285, 46, 10, 170, 382]   ,		)

         lowercase_     =      tokenizer.tokenize("I was born in 92000, and this is falsé.")
         self.assertListEqual(
             __lowerCAmelCase   ,		[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]   ,		)
         lowercase_     =      tokenizer.convert_tokens_to_ids(__lowerCAmelCase)
         self.assertListEqual(
             __lowerCAmelCase   ,		[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]   ,		)

         lowercase_     =      tokenizer.convert_ids_to_tokens(__lowerCAmelCase)
         self.assertListEqual(
             __lowerCAmelCase   ,		[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]   ,		)


   @cached_property
   def __UpperCAmelCase       (   self				:	List[str])      ->					Any:
         return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment")


   @slow
   def __UpperCAmelCase       (   self				:	Tuple)      ->					str:
         lowercase_     =      "Hello World!"
         lowercase_     =      [126, 32, 262, 152, 38, 72, 287]

         self.assertListEqual(__lowerCAmelCase   ,		self.big_tokenizer.encode(__lowerCAmelCase))


   @slow
   def __UpperCAmelCase       (   self				:	Any)      ->					Dict:
         lowercase_     =      (
             "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
             " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
         )
         lowercase_     =      [
             108,
             265,
             24,
             111,
             4,
             258,
             156,
             35,
             28,
             275,
             3,
             259,
             297,
             260,
             84,
             4,
             35,
             110,
             44,
             8,
             259,
             91,
             268,
             21,
             11,
             209,
             274,
             109,
             266,
             277,
             117,
             86,
             93,
             315,
             258,
             278,
             258,
             277,
             258,
             0,
             258,
             288,
             258,
             319,
             258,
             0,
             258,
             0,
             258,
             0,
             258,
             0,
             258,
             287,
             258,
             315,
             258,
             289,
             258,
             278,
             99,
             269,
             266,
             262,
             8,
             259,
             241,
             4,
             217,
             230,
             268,
             266,
             55,
             168,
             106,
             75,
             193,
             266,
             223,
             27,
             49,
             26,
             282,
             25,
             264,
             299,
             19,
             26,
             0,
             258,
             277,
             117,
             86,
             93,
             176,
             183,
             270,
             11,
             262,
             42,
             61,
             265,
         ]

         self.assertListEqual(__lowerCAmelCase   ,		self.big_tokenizer.encode(__lowerCAmelCase))


   @require_torch
   @slow
   def __UpperCAmelCase       (   self				:	Any)      ->					Tuple:
         import torch

         from transformers import ReformerConfig, ReformerModel

         # Build sequence
         lowercase_     =      list(self.big_tokenizer.get_vocab().keys())[:10]
         lowercase_     =      " ".join(__lowerCAmelCase)
         lowercase_     =      self.big_tokenizer.encode_plus(__lowerCAmelCase   ,		return_tensors="pt")
         lowercase_     =      self.big_tokenizer.batch_encode_plus([sequence, sequence]   ,		return_tensors="pt")

         lowercase_     =      ReformerConfig()
         # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
         lowercase_     =      encoded_sequence["input_ids"].shape
         lowercase_     =      ReformerModel(__lowerCAmelCase)

         # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
               model(**__lowerCAmelCase)
               model(**__lowerCAmelCase)


   @slow
   def __UpperCAmelCase       (   self				:	List[Any])      ->					Tuple:
         # fmt: off
         lowercase_     =      {"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # This tokenizer does not know some characters like ")".
         # That is the reason why we use very simple texts here.
         # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
         lowercase_     =      [
             "This is a very simple sentence.",
             "The quick brown fox jumps over the lazy dog.",
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=__lowerCAmelCase   ,		model_name="google/reformer-crime-and-punishment"   ,		revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"   ,		padding=__lowerCAmelCase   ,		sequences=__lowerCAmelCase   ,		)






