





import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
      from transformers.trainer import DEFAULT_CALLBACKS

      from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel

class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self       ):
         A_     :				Dict			      =	[]
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_init_end"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_train_begin"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_train_end"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_epoch_begin"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_epoch_end"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_step_begin"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_step_end"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_evaluate"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_predict"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_save"""       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_log"""       )






      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       a__      ,       **a__       ):
         self.events.append("""on_prediction_step"""       )






@require_torch
class    _UpperCAmelCase							(		unittest.TestCase     ):
      def 		_lowerCamelCase      (		self       ):
         A_     :				Optional[Any]			      =	tempfile.mkdtemp()
      def 		_lowerCamelCase      (		self       ):
         shutil.rmtree(self.output_dir       )
      def 		_lowerCamelCase      (		self      ,       a__=0      ,       a__=0      ,       a__=64      ,       a__=64      ,       a__=None      ,       a__=False      ,       **a__       ):
         # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
         # its set to False since the tests later on depend on its value.
         A_     :				Dict			      =	RegressionDataset(length=a__       )
         A_     :				List[str]			      =	RegressionDataset(length=a__       )
         A_     :				int			      =	RegressionModelConfig(a=a__      ,       b=a__       )
         A_     :				Optional[int]			      =	RegressionPreTrainedModel(a__       )

         A_     :				Any			      =	TrainingArguments(self.output_dir      ,       disable_tqdm=a__      ,       report_to=[]      ,       **a__       )
         return Trainer(
             a__      ,       a__      ,       train_dataset=a__      ,       eval_dataset=a__      ,       callbacks=a__      ,       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__       ):
         self.assertEqual(len(a__       )      ,       len(a__       )       )

         # Order doesn't matter
         A_     :				int			      =	sorted(a__      ,       key=lambda a__       : cb.__name__ if isinstance(a__      ,       a__       ) else cb.__class__.__name__       )
         A_     :				Any			      =	sorted(a__      ,       key=lambda a__       : cb.__name__ if isinstance(a__      ,       a__       ) else cb.__class__.__name__       )

         for cba, cba in zip(a__      ,       a__       ):
            if isinstance(a__      ,       a__       ) and isinstance(a__      ,       a__       ):
               self.assertEqual(a__      ,       a__       )
            elif isinstance(a__      ,       a__       ) and not isinstance(a__      ,       a__       ):
               self.assertEqual(a__      ,       cba.__class__       )
            elif not isinstance(a__      ,       a__       ) and isinstance(a__      ,       a__       ):
               self.assertEqual(cba.__class__      ,       a__       )
            else:
               self.assertEqual(a__      ,       a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Union[str, Any]			      =	["""on_init_end""", """on_train_begin"""]
         A_     :				str			      =	0
         A_     :				Dict			      =	len(trainer.get_eval_dataloader()       )
         A_     :				List[Any]			      =	["""on_prediction_step"""] * len(trainer.get_eval_dataloader()       ) + ["""on_log""", """on_evaluate"""]
         for _ in range(trainer.state.num_train_epochs       ):
            expected_events.append("""on_epoch_begin"""       )
            for _ in range(a__       ):
               step += 1
               expected_events += ["on_step_begin", "on_step_end"]
               if step % trainer.args.logging_steps == 0:
                  expected_events.append("""on_log"""       )
               if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                  expected_events += evaluation_events.copy()
               if step % trainer.args.save_steps == 0:
                  expected_events.append("""on_save"""       )
            expected_events.append("""on_epoch_end"""       )
            if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
               expected_events += evaluation_events.copy()
         expected_events += ["on_log", "on_train_end"]
         return expected_events
      def 		_lowerCamelCase      (		self       ):
         A_     :				List[str]			      =	self.get_trainer()
         A_     :				str			      =	DEFAULT_CALLBACKS.copy() + [ProgressCallback]
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         # Callbacks passed at init are added to the default callbacks
         A_     :				int			      =	self.get_trainer(callbacks=[MyTestTrainerCallback]       )
         expected_callbacks.append(a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
         A_     :				Optional[int]			      =	self.get_trainer(disable_tqdm=a__       )
         A_     :				Tuple			      =	DEFAULT_CALLBACKS.copy() + [PrinterCallback]
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )
      def 		_lowerCamelCase      (		self       ):
         A_     :				Tuple			      =	DEFAULT_CALLBACKS.copy() + [ProgressCallback]
         A_     :				Dict			      =	self.get_trainer()

         # We can add, pop, or remove by class name
         trainer.remove_callback(a__       )
         expected_callbacks.remove(a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         A_     :				Any			      =	self.get_trainer()
         A_     :				Any			      =	trainer.pop_callback(a__       )
         self.assertEqual(cb.__class__      ,       a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         trainer.add_callback(a__       )
         expected_callbacks.insert(0      ,       a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         # We can also add, pop, or remove by instance
         A_     :				str			      =	self.get_trainer()
         A_     :				Union[str, Any]			      =	trainer.callback_handler.callbacks[0]
         trainer.remove_callback(a__       )
         expected_callbacks.remove(a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         A_     :				Optional[Any]			      =	self.get_trainer()
         A_     :				Dict			      =	trainer.callback_handler.callbacks[0]
         A_     :				Optional[Any]			      =	trainer.pop_callback(a__       )
         self.assertEqual(a__      ,       a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )

         trainer.add_callback(a__       )
         expected_callbacks.insert(0      ,       a__       )
         self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       a__       )






      def 		_lowerCamelCase      (		self       ):
         import warnings

         # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
         warnings.simplefilter(action="""ignore"""      ,       category=a__       )

         A_     :				Any			      =	self.get_trainer(callbacks=[MyTestTrainerCallback]       )
         trainer.train()
         A_     :				Any			      =	trainer.callback_handler.callbacks[-2].events
         self.assertEqual(a__      ,       self.get_expected_events(a__       )       )

         # Independent log/save/eval
         A_     :				Union[str, Any]			      =	self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       logging_steps=5       )
         trainer.train()
         A_     :				List[Any]			      =	trainer.callback_handler.callbacks[-2].events
         self.assertEqual(a__      ,       self.get_expected_events(a__       )       )

         A_     :				Tuple			      =	self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       save_steps=5       )
         trainer.train()
         A_     :				Tuple			      =	trainer.callback_handler.callbacks[-2].events
         self.assertEqual(a__      ,       self.get_expected_events(a__       )       )

         A_     :				Any			      =	self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       eval_steps=5      ,       evaluation_strategy="""steps"""       )
         trainer.train()
         A_     :				Tuple			      =	trainer.callback_handler.callbacks[-2].events
         self.assertEqual(a__      ,       self.get_expected_events(a__       )       )

         A_     :				str			      =	self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       evaluation_strategy="""epoch"""       )
         trainer.train()
         A_     :				Tuple			      =	trainer.callback_handler.callbacks[-2].events
         self.assertEqual(a__      ,       self.get_expected_events(a__       )       )

         # A bit of everything
         A_     :				List[Any]			      =	self.get_trainer(
             callbacks=[MyTestTrainerCallback]      ,       logging_steps=3      ,       save_steps=10      ,       eval_steps=5      ,       evaluation_strategy="""steps"""      ,       )
         trainer.train()
         A_     :				List[Any]			      =	trainer.callback_handler.callbacks[-2].events
         self.assertEqual(a__      ,       self.get_expected_events(a__       )       )

         # warning should be emitted for duplicated callbacks
         with patch("""transformers.trainer_callback.logger.warning"""       ) as warn_mock:
            A_     :				Optional[int]			      =	self.get_trainer(
                callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]      ,       )
            assert str(a__       ) in warn_mock.call_args[0][0]









import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler

class    _UpperCAmelCase							:
      def __init__(		self      ,       a__       ):
         if isinstance(a__      ,       a__       ):
            # Don't modify user's data should they want to reuse it (e.g. in tests), because once we
            # modified it, it will not be accepted here again, since `auto` values would have been overridden
            A_     :				Optional[Any]			      =	deepcopy(a__       )
         elif os.path.exists(a__       ):
            with io.open(a__      ,       """r"""      ,       encoding="""utf-8"""       ) as f:
               A_     :				str			      =	json.load(a__       )
         else:
            try:
               A_     :				Dict			      =	baseaa.urlsafe_baadecode(a__       ).decode("""utf-8"""       )
               A_     :				List[Any]			      =	json.loads(a__       )
            except (UnicodeDecodeError, AttributeError, ValueError):
               raise ValueError(
                   F"""Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}"""       )

         A_     :				Any			      =	config

         self.set_stage_and_offload()
      def 		_lowerCamelCase      (		self       ):
         # zero stage - this is done as early as possible, before model is created, to allow
         # ``is_deepspeed_zero3_enabled`` query and getting to the early deepspeed config object
         # during ``zero.Init()`` which needs to know the dtype, and some other hparams.
         A_     :				List[str]			      =	self.get_value("""zero_optimization.stage"""      ,       -1       )

         # offload
         A_     :				Any			      =	False
         if self.is_zeroa() or self.is_zeroa():
            A_     :				Optional[int]			      =	set(["""cpu""", """nvme"""]       )
            A_     :				Dict			      =	set(
                [
                    self.get_value("""zero_optimization.offload_optimizer.device"""       ),
                    self.get_value("""zero_optimization.offload_param.device"""       ),
                ]       )
            if len(offload_devices & offload_devices_valid       ) > 0:
               A_     :				Tuple			      =	True
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				List[Any]			      =	self.config

         # find the config node of interest if it exists
         A_     :				Optional[Any]			      =	ds_key_long.split("""."""       )
         A_     :				Union[str, Any]			      =	nodes.pop()
         for node in nodes:
            A_     :				List[str]			      =	config.get(a__       )
            if config is None:
               return None, ds_key

         return config, ds_key
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=None       ):
         A_      ,				A_     :				Union[str, Any]			      =	self.find_config_node(a__       )
         if config is None:
            return default
         return config.get(a__      ,       a__       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=False       ):
         A_     :				Union[str, Any]			      =	self.config

         # find the config node of interest if it exists
         A_     :				str			      =	ds_key_long.split("""."""       )
         for node in nodes:
            A_     :				int			      =	config
            A_     :				int			      =	config.get(a__       )
            if config is None:
               if must_exist:
                  raise ValueError(F"""Can't find {ds_key_long} entry in the config: {self.config}"""       )
               else:
                  return

        # if found remove it
         if parent_config is not None:
            parent_config.pop(a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Optional[Any]			      =	self.get_value(a__       )
         return False if value is None else bool(a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Optional[Any]			      =	self.get_value(a__       )
         return False if value is None else not bool(a__       )
      def 		_lowerCamelCase      (		self       ):
         return self._stage == 2
      def 		_lowerCamelCase      (		self       ):
         return self._stage == 3






      def 		_lowerCamelCase      (		self       ):
         return self._offload






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__       ):
         A_     :				Any			      =	engine






      def 		_lowerCamelCase      (		self      ,       a__      ,       **a__       ):
         # runs backpropagation and handles mixed precision
         self.engine.backward(a__      ,       **a__       )

         # Deepspeed's `engine.step` performs the following operations:
         # - gradient accumulation check
         # - gradient clipping
         # - optimizer step
         # - zero grad
         # - checking overflow
         # - lr_scheduler step (only if engine.lr_scheduler is not None)
         self.engine.step()
         # and this plugin overrides the above calls with no-ops when Accelerate runs under
         # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
         # training loop that works transparently under many training regimes.






class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__       ):
         super().__init__(a__      ,       device_placement=a__      ,       scaler=a__       )
         A_     :				Dict			      =	hasattr(self.optimizer      ,       """overflow"""       )
      def 		_lowerCamelCase      (		self      ,       a__=None       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed
      def 		_lowerCamelCase      (		self       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






      @property
      def 		_lowerCamelCase      (		self       ):
         if self.__has_overflow__:
            return self.optimizer.overflow
         return False






class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__      ,       a__       ):
         super().__init__(a__      ,       a__       )






      def 		_lowerCamelCase      (		self       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__=0.001      ,       a__=0      ,       **a__       ):
         A_     :				List[str]			      =	params
         A_     :				Any			      =	lr
         A_     :				int			      =	weight_decay
         A_     :				Optional[int]			      =	kwargs






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__=None      ,       a__=0      ,       **a__       ):
         A_     :				Union[str, Any]			      =	optimizer
         A_     :				int			      =	total_num_steps
         A_     :				Any			      =	warmup_num_steps
         A_     :				int			      =	kwargs






