





from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFTransfoXLForSequenceClassification,
          TFTransfoXLLMHeadModel,
          TFTransfoXLModel,
      )

class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       ):
         A_     :				str			      =	parent
         A_     :				Any			      =	13
         A_     :				List[Any]			      =	7
         A_     :				Optional[Any]			      =	30
         A_     :				Optional[Any]			      =	self.seq_length + self.mem_len
         A_     :				Dict			      =	15
         A_     :				Union[str, Any]			      =	True
         A_     :				int			      =	True
         A_     :				List[str]			      =	99
         A_     :				Dict			      =	[10, 50, 80]
         A_     :				Dict			      =	32
         A_     :				Optional[Any]			      =	32
         A_     :				List[Any]			      =	4
         A_     :				Optional[Any]			      =	8
         A_     :				Any			      =	128
         A_     :				Optional[int]			      =	2
         A_     :				Union[str, Any]			      =	2
         A_     :				Union[str, Any]			      =	None
         A_     :				Union[str, Any]			      =	1
         A_     :				Optional[Any]			      =	0
         A_     :				Dict			      =	3
         A_     :				List[Any]			      =	self.vocab_size - 1
         A_     :				int			      =	0.01
      def 		_lowerCamelCase      (		self       ):
         A_     :				Tuple			      =	ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size       )
         A_     :				Any			      =	ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size       )

         A_     :				List[str]			      =	None
         if self.use_labels:
            A_     :				Optional[Any]			      =	ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size       )

         A_     :				Optional[int]			      =	TransfoXLConfig(
             vocab_size=self.vocab_size      ,       mem_len=self.mem_len      ,       clamp_len=self.clamp_len      ,       cutoffs=self.cutoffs      ,       d_model=self.hidden_size      ,       d_embed=self.d_embed      ,       n_head=self.num_attention_heads      ,       d_head=self.d_head      ,       d_inner=self.d_inner      ,       div_val=self.div_val      ,       n_layer=self.num_hidden_layers      ,       eos_token_id=self.eos_token_id      ,       pad_token_id=self.vocab_size - 1      ,       init_range=self.init_range      ,       num_labels=self.num_labels      ,       )

         return (config, input_ids_a, input_ids_a, lm_labels)
      def 		_lowerCamelCase      (		self       ):
         random.seed(self.seed       )
         tf.random.set_seed(self.seed       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       a__       ):
         A_     :				Any			      =	TFTransfoXLModel(a__       )

         A_     :				str			      =	model(a__       ).to_tuple()

         A_     :				Union[str, Any]			      =	{"""input_ids""": input_ids_a, """mems""": mems_a}

         A_     :				Optional[int]			      =	model(a__       ).to_tuple()

         self.parent.assertEqual(hidden_states_a.shape      ,       (self.batch_size, self.seq_length, self.hidden_size)       )
         self.parent.assertEqual(hidden_states_a.shape      ,       (self.batch_size, self.seq_length, self.hidden_size)       )
         self.parent.assertListEqual(
             [mem.shape for mem in mems_a]      ,       [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers      ,       )
         self.parent.assertListEqual(
             [mem.shape for mem in mems_a]      ,       [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers      ,       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       a__       ):
         A_     :				Dict			      =	TFTransfoXLLMHeadModel(a__       )

         A_     :				List[Any]			      =	model(a__       ).to_tuple()

         A_     :				int			      =	{"""input_ids""": input_ids_a, """labels""": lm_labels}
         A_     :				Any			      =	model(a__       ).to_tuple()

         A_     :				List[str]			      =	model([input_ids_a, mems_a]       ).to_tuple()

         A_     :				List[Any]			      =	{"""input_ids""": input_ids_a, """mems""": mems_a, """labels""": lm_labels}

         A_     :				str			      =	model(a__       ).to_tuple()

         self.parent.assertEqual(lm_logits_a.shape      ,       (self.batch_size, self.seq_length, self.vocab_size)       )
         self.parent.assertListEqual(
             [mem.shape for mem in mems_a]      ,       [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers      ,       )

         self.parent.assertEqual(lm_logits_a.shape      ,       (self.batch_size, self.seq_length, self.vocab_size)       )
         self.parent.assertListEqual(
             [mem.shape for mem in mems_a]      ,       [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers      ,       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       a__       ):
         A_     :				List[Any]			      =	TFTransfoXLForSequenceClassification(a__       )
         A_     :				Optional[Any]			      =	model(a__       )
         self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_labels)       )






      def 		_lowerCamelCase      (		self       ):
         A_     :				Tuple			      =	self.prepare_config_and_inputs()
         (A_)     :				Tuple			      =	config_and_inputs
         A_     :				Optional[int]			      =	{"""input_ids""": input_ids_a}
         return config, inputs_dict






@require_tf
class    _UpperCAmelCase							(		_lowerCamelCase			,       _lowerCamelCase			,       unittest.TestCase     ):
      a						       =							(
          (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
      )
      a						       =							() if is_tf_available() else ()
      a						       =							(
          {
              '''feature-extraction''': TFTransfoXLModel,
              '''text-classification''': TFTransfoXLForSequenceClassification,
              '''text-generation''': TFTransfoXLLMHeadModel,
              '''zero-shot''': TFTransfoXLForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      # TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
      a						       =							False
      a						       =							False
      a						       =							False
      a						       =							False
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       a__      ,       a__       ):
         if pipeline_test_casse_name == "TextGenerationPipelineTests":
            # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
            # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
            # tokenizer.
            return True

         return False
      def 		_lowerCamelCase      (		self       ):
         A_     :				Optional[int]			      =	TFTransfoXLModelTester(self       )
         A_     :				List[Any]			      =	ConfigTester(self      ,       config_class=a__      ,       d_embed=37       )
      def 		_lowerCamelCase      (		self       ):
         self.config_tester.run_common_tests()
      def 		_lowerCamelCase      (		self       ):
         self.model_tester.set_seed()
         A_     :				Dict			      =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_transfo_xl_model(*a__       )
      def 		_lowerCamelCase      (		self       ):
         self.model_tester.set_seed()
         A_     :				Optional[Any]			      =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_transfo_xl_lm_head(*a__       )
      def 		_lowerCamelCase      (		self       ):
         A_     :				Union[str, Any]			      =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*a__       )
      def 		_lowerCamelCase      (		self       ):
         A_     :				Any			      =	self.model_tester.prepare_config_and_inputs_for_common()
         A_     :				Dict			      =	[TFTransfoXLForSequenceClassification]

         for model_class in self.all_model_classes:
            A_     :				List[Any]			      =	model_class(a__       )
            assert isinstance(model.get_input_embeddings()      ,       tf.keras.layers.Layer       )
            if model_class in list_other_models_with_output_ebd:
               A_     :				Optional[int]			      =	model.get_output_embeddings()
               assert isinstance(a__      ,       tf.keras.layers.Layer       )
               A_     :				Union[str, Any]			      =	model.get_bias()
               assert name is None
            else:
               A_     :				Union[str, Any]			      =	model.get_output_embeddings()
               assert x is None
               A_     :				List[str]			      =	model.get_bias()
               assert name is None
      def 		_lowerCamelCase      (		self       ):
         # TODO JP: Make TransfoXL XLA compliant
         pass
      @slow
      def 		_lowerCamelCase      (		self       ):
         for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            A_     :				Tuple			      =	TFTransfoXLModel.from_pretrained(a__       )
            self.assertIsNotNone(a__       )






      @unittest.skip(reason="""This model doesn't play well with fit() due to not returning a single loss."""       )
      def 		_lowerCamelCase      (		self       ):
         pass






@require_tf
class    _UpperCAmelCase							(		unittest.TestCase     ):
      @unittest.skip("""Skip test until #12651 is resolved."""       )
      @slow
      def 		_lowerCamelCase      (		self       ):
         A_     :				List[Any]			      =	TFTransfoXLLMHeadModel.from_pretrained("""transfo-xl-wt103"""       )
         # fmt: off
         A_     :				Optional[int]			      =	tf.convert_to_tensor([[33,1297,2,1,1009,4,1109,11739,4762,358,5,25,245,22,1706,17,20098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,6224,831,16002,2,8,603,78967,29546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,29546,54,8,3609,5,57211,49,4,1,277,18,8,1755,15691,3,341,25,416,693,42573,71,17,401,94,31,17919,2,29546,7873,18,1,435,23,11011,755,5,5167,3,7983,98,84,2,29546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,29546,824,1400,1868,2,19,160,2,311,8,5496,2,20920,17,25,15097,3,24,24,0]]      ,       dtype=tf.intaa       )  # noqa: E231
         # fmt: on
         #  In 1991 , the remains of Russian Tsar Nicholas II and his family
         #  ( except for Alexei and Maria ) are discovered .
         #  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
         #  remainder of the story . 1883 Western Siberia ,
         #  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
         #  Rasputin has a vision and denounces one of the men as a horse thief . Although his
         #  father initially slaps him for making such an accusation , Rasputin watches as the
         #  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
         #  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
         #  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

         # fmt: off
         A_     :				Any			      =	[33,1297,2,1,1009,4,1109,11739,4762,358,5,25,245,22,1706,17,20098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,6224,831,16002,2,8,603,78967,29546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,29546,54,8,3609,5,57211,49,4,1,277,18,8,1755,15691,3,341,25,416,693,42573,71,17,401,94,31,17919,2,29546,7873,18,1,435,23,11011,755,5,5167,3,7983,98,84,2,29546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,29546,824,1400,1868,2,19,160,2,311,8,5496,2,20920,17,25,15097,3,24,24,0,33,1,1857,2,1,1009,4,1109,11739,4762,358,5,25,245,28,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,0]  # noqa: E231
         # fmt: on
         #  In 1991, the remains of Russian Tsar Nicholas II and his family (
         #  except for Alexei and Maria ) are discovered. The voice of young son,
         #  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
         #  1883 Western Siberia, a young Grigori Rasputin is asked by his father
         #  and a group of men to perform magic. Rasputin has a vision and
         #  denounces one of the men as a horse thief. Although his father initially
         #  slaps him for making such an accusation, Rasputin watches as the man
         #  is chased outside and beaten. Twenty years later, Rasputin sees a vision
         #  of the Virgin Mary, prompting him to become a priest.
         #  Rasputin quickly becomes famous, with people, even a bishop, begging for
         #  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
         # Nicholas II and his family were discovered. The voice of <unk> young son,
         # Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

         A_     :				List[Any]			      =	model.generate(a__      ,       max_length=200      ,       do_sample=a__       )
         self.assertListEqual(output_ids[0].numpy().tolist()      ,       a__       )









import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler

class    _UpperCAmelCase							:
      def __init__(		self      ,       a__       ):
         if isinstance(a__      ,       a__       ):
            # Don't modify user's data should they want to reuse it (e.g. in tests), because once we
            # modified it, it will not be accepted here again, since `auto` values would have been overridden
            A_     :				Optional[Any]			      =	deepcopy(a__       )
         elif os.path.exists(a__       ):
            with io.open(a__      ,       """r"""      ,       encoding="""utf-8"""       ) as f:
               A_     :				str			      =	json.load(a__       )
         else:
            try:
               A_     :				Dict			      =	baseaa.urlsafe_baadecode(a__       ).decode("""utf-8"""       )
               A_     :				List[Any]			      =	json.loads(a__       )
            except (UnicodeDecodeError, AttributeError, ValueError):
               raise ValueError(
                   F"""Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}"""       )

         A_     :				Any			      =	config

         self.set_stage_and_offload()
      def 		_lowerCamelCase      (		self       ):
         # zero stage - this is done as early as possible, before model is created, to allow
         # ``is_deepspeed_zero3_enabled`` query and getting to the early deepspeed config object
         # during ``zero.Init()`` which needs to know the dtype, and some other hparams.
         A_     :				List[str]			      =	self.get_value("""zero_optimization.stage"""      ,       -1       )

         # offload
         A_     :				Any			      =	False
         if self.is_zeroa() or self.is_zeroa():
            A_     :				Optional[int]			      =	set(["""cpu""", """nvme"""]       )
            A_     :				Dict			      =	set(
                [
                    self.get_value("""zero_optimization.offload_optimizer.device"""       ),
                    self.get_value("""zero_optimization.offload_param.device"""       ),
                ]       )
            if len(offload_devices & offload_devices_valid       ) > 0:
               A_     :				Tuple			      =	True
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				List[Any]			      =	self.config

         # find the config node of interest if it exists
         A_     :				Optional[Any]			      =	ds_key_long.split("""."""       )
         A_     :				Union[str, Any]			      =	nodes.pop()
         for node in nodes:
            A_     :				List[str]			      =	config.get(a__       )
            if config is None:
               return None, ds_key

         return config, ds_key
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=None       ):
         A_      ,				A_     :				Union[str, Any]			      =	self.find_config_node(a__       )
         if config is None:
            return default
         return config.get(a__      ,       a__       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=False       ):
         A_     :				Union[str, Any]			      =	self.config

         # find the config node of interest if it exists
         A_     :				str			      =	ds_key_long.split("""."""       )
         for node in nodes:
            A_     :				int			      =	config
            A_     :				int			      =	config.get(a__       )
            if config is None:
               if must_exist:
                  raise ValueError(F"""Can't find {ds_key_long} entry in the config: {self.config}"""       )
               else:
                  return

        # if found remove it
         if parent_config is not None:
            parent_config.pop(a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Optional[Any]			      =	self.get_value(a__       )
         return False if value is None else bool(a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Optional[Any]			      =	self.get_value(a__       )
         return False if value is None else not bool(a__       )
      def 		_lowerCamelCase      (		self       ):
         return self._stage == 2
      def 		_lowerCamelCase      (		self       ):
         return self._stage == 3






      def 		_lowerCamelCase      (		self       ):
         return self._offload






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__       ):
         A_     :				Any			      =	engine






      def 		_lowerCamelCase      (		self      ,       a__      ,       **a__       ):
         # runs backpropagation and handles mixed precision
         self.engine.backward(a__      ,       **a__       )

         # Deepspeed's `engine.step` performs the following operations:
         # - gradient accumulation check
         # - gradient clipping
         # - optimizer step
         # - zero grad
         # - checking overflow
         # - lr_scheduler step (only if engine.lr_scheduler is not None)
         self.engine.step()
         # and this plugin overrides the above calls with no-ops when Accelerate runs under
         # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
         # training loop that works transparently under many training regimes.






class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__       ):
         super().__init__(a__      ,       device_placement=a__      ,       scaler=a__       )
         A_     :				Dict			      =	hasattr(self.optimizer      ,       """overflow"""       )
      def 		_lowerCamelCase      (		self      ,       a__=None       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed
      def 		_lowerCamelCase      (		self       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






      @property
      def 		_lowerCamelCase      (		self       ):
         if self.__has_overflow__:
            return self.optimizer.overflow
         return False






class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__      ,       a__       ):
         super().__init__(a__      ,       a__       )






      def 		_lowerCamelCase      (		self       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__=0.001      ,       a__=0      ,       **a__       ):
         A_     :				List[str]			      =	params
         A_     :				Any			      =	lr
         A_     :				int			      =	weight_decay
         A_     :				Optional[int]			      =	kwargs






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__=None      ,       a__=0      ,       **a__       ):
         A_     :				Union[str, Any]			      =	optimizer
         A_     :				int			      =	total_num_steps
         A_     :				Any			      =	warmup_num_steps
         A_     :				int			      =	kwargs






