





'''simple docstring'''

# XXX: we want transformers master here - in the absense of conftest manipulating sys.path:
# hack it in for now:
import sys
from pathlib import Path


a        =						Path(__file__).resolve().parents[3] / "src"
sys.path.insert(1, str(git_repo_path))

import dataclasses  # noqa
import io  # noqa
import itertools  # noqa
import json  # noqa
import os  # noqa
import unittest  # noqa
from copy import deepcopy  # noqa

from parameterized import parameterized  # noqa
from transformers import TrainingArguments, is_torch_available  # noqa
from transformers.deepspeed import is_deepspeed_available  # noqa
from transformers.file_utils import WEIGHTS_NAME  # noqa
from transformers.testing_utils import (  # noqa
    CaptureLogger,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    mockenv_context,
    require_deepspeed,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)
from transformers.trainer_utils import set_seed  # noqa


set_seed(42)

a        =						{"base": "patrickvonplaten/wav2vec2_tiny_random", "robust": "patrickvonplaten/wav2vec2_tiny_random_robust"}

a        =						"zero2"
a        =						"zero3"
a        =						[ZEROa, ZEROa]

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					int:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       parameterized.to_safe_name("""_""".join(str(__UpperCAmelCase      ) for x in param.args      )      )
   return f"""{func.__name__}_{param_based_name}"""


# Cartesian-product of zero stages with models to test
a        =						list(itertools.product(stages, models.keys()))







@slow
@require_deepspeed
@require_torch_gpu
class __a (   _snake_case			):





   @parameterized.expand(lowerCamelCase				,name_func=lowerCamelCase    )
   def      UpperCAmelCase__     (					self				:			Dict				,lowerCamelCase				:			Optional[int]				,lowerCamelCase				:			Tuple    ):

      '''simple docstring'''

      self.run_and_check(
          stage=lowerCamelCase				,model=lowerCamelCase				,distributed=lowerCamelCase				,fpaa=lowerCamelCase				,)





   @require_torch_multi_gpu
   @parameterized.expand(lowerCamelCase				,name_func=lowerCamelCase    )
   def      UpperCAmelCase__     (					self				:			Dict				,lowerCamelCase				:			List[Any]				,lowerCamelCase				:			Dict    ):

      '''simple docstring'''

      self.run_and_check(
          stage=lowerCamelCase				,model=lowerCamelCase				,distributed=lowerCamelCase				,fpaa=lowerCamelCase				,)





   @parameterized.expand(lowerCamelCase				,name_func=lowerCamelCase    )
   def      UpperCAmelCase__     (					self				:			Optional[Any]				,lowerCamelCase				:			Union[str, Any]				,lowerCamelCase				:			List[Any]    ):

      '''simple docstring'''

      self.run_and_check(
          stage=lowerCamelCase				,model=lowerCamelCase				,distributed=lowerCamelCase				,fpaa=lowerCamelCase				,)





   @require_torch_multi_gpu
   @parameterized.expand(lowerCamelCase				,name_func=lowerCamelCase    )
   def      UpperCAmelCase__     (					self				:			Optional[int]				,lowerCamelCase				:			Optional[Any]				,lowerCamelCase				:			Optional[Any]    ):

      '''simple docstring'''

      self.run_and_check(
          stage=lowerCamelCase				,model=lowerCamelCase				,distributed=lowerCamelCase				,fpaa=lowerCamelCase				,)





   def      UpperCAmelCase__     (					self				:			Union[str, Any]				,lowerCamelCase				:			Dict    ):

      '''simple docstring'''

      pass





   def      UpperCAmelCase__     (					self				:			Tuple				,lowerCamelCase				:			str				,lowerCamelCase				:			str				,lowerCamelCase				:			int = 10				,lowerCamelCase				:			bool = True				,lowerCamelCase				:			bool = True				,lowerCamelCase				:			bool = True				,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       models[model]

      __SCREAMING_SNAKE_CASE      							=       self.run_trainer(
          stage=lowerCamelCase				,model_name=lowerCamelCase				,eval_steps=lowerCamelCase				,num_train_epochs=1				,distributed=lowerCamelCase				,fpaa=lowerCamelCase				,)

      self.do_checks(lowerCamelCase    )

      return output_dir





   def      UpperCAmelCase__     (					self				:			List[str]				,lowerCamelCase				:			str				,lowerCamelCase				:			str				,lowerCamelCase				:			int = 10				,lowerCamelCase				:			int = 1				,lowerCamelCase				:			bool = True				,lowerCamelCase				:			bool = True				,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self.get_auto_remove_tmp_dir("""./xxx"""				,after=lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       f"""
            --model_name_or_path {model_name}
            --dataset_name hf-internal-testing/librispeech_asr_dummy
            --dataset_config_name clean
            --train_split_name validation
            --validation_split_name validation
            --output_dir {output_dir}
            --num_train_epochs {str(lowerCamelCase    )}
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 2
            --evaluation_strategy steps
            --learning_rate 5e-4
            --warmup_steps 8
            --orthography timit
            --preprocessing_num_workers 1
            --group_by_length
            --freeze_feature_extractor
            --report_to none
            --save_steps 0
            --eval_steps {eval_steps}
            --report_to none
        """.split()

      if fpaa:
         args.extend(["""--fp16"""]    )

      # currently ds_config_wav2vec2_zero.json requires "zero_optimization.find_unused_parameters": true,
      # hence the separate config files
      __SCREAMING_SNAKE_CASE      							=       f"""--deepspeed {self.test_file_dir_str}/ds_config_wav2vec2_{stage}.json""".split()
      __SCREAMING_SNAKE_CASE      							=       [f"""{self.examples_dir_str}/research_projects/wav2vec2/run_asr.py"""]
      __SCREAMING_SNAKE_CASE      							=       self.get_launcher(lowerCamelCase    )

      __SCREAMING_SNAKE_CASE      							=       launcher + script + args + ds_args
      # keep for quick debug
      # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
      execute_subprocess_async(lowerCamelCase				,env=self.get_env()    )

      return output_dir





   def      UpperCAmelCase__     (					self				:			Tuple				,lowerCamelCase				:			List[Any]=False    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       min(2				,get_gpu_count()    ) if distributed else 1
      return f"""deepspeed --num_nodes 1 --num_gpus {num_gpus}""".split()












'''simple docstring'''

import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


a        =						logging.get_logger(__name__)







class __a (   _snake_case			):
   __UpperCamelCase   : int	      =	'linear'
   __UpperCamelCase   : Tuple	      =	'cosine'
   __UpperCamelCase   : Tuple	      =	'cosine_with_restarts'
   __UpperCamelCase   : List[Any]	      =	'polynomial'
   __UpperCamelCase   : Optional[Any]	      =	'constant'
   __UpperCamelCase   : Optional[int]	      =	'constant_with_warmup'
   __UpperCamelCase   : List[Any]	      =	'piecewise_constant'





def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase = -1      )						->					int:

   '''simple docstring'''







   return LambdaLR(__UpperCAmelCase							,	lambda __UpperCAmelCase      : 1							,	last_epoch=__UpperCAmelCase      )

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase = -1      )						->					List[Any]:

   '''simple docstring'''







   def lr_lambda(__UpperCAmelCase      ):
      if current_step < num_warmup_steps:
         return float(__UpperCAmelCase      ) / float(max(1.0							,	__UpperCAmelCase      )      )
      return 1.0

   return LambdaLR(__UpperCAmelCase							,	__UpperCAmelCase							,	last_epoch=__UpperCAmelCase      )

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase = -1      )						->					int:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       {}
   __SCREAMING_SNAKE_CASE      							=       step_rules.split(""","""      )
   for rule_str in rule_list[:-1]:
      __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       rule_str.split(""":"""      )
      __SCREAMING_SNAKE_CASE      							=       int(__UpperCAmelCase      )
      __SCREAMING_SNAKE_CASE      							=       float(__UpperCAmelCase      )
      __SCREAMING_SNAKE_CASE      							=       value
   __SCREAMING_SNAKE_CASE      							=       float(rule_list[-1]      )

   def create_rules_function(__UpperCAmelCase							,	__UpperCAmelCase      ):
      def rule_func(__UpperCAmelCase      ) -> float:
         __SCREAMING_SNAKE_CASE      							=       sorted(rules_dict.keys()      )
         for i, sorted_step in enumerate(__UpperCAmelCase      ):
            if steps < sorted_step:
               return rules_dict[sorted_steps[i]]
         return last_lr_multiple

      return rule_func

   __SCREAMING_SNAKE_CASE      							=       create_rules_function(__UpperCAmelCase							,	__UpperCAmelCase      )

   return LambdaLR(__UpperCAmelCase							,	__UpperCAmelCase							,	last_epoch=__UpperCAmelCase      )

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase=-1      )						->					int:

   '''simple docstring'''







   def lr_lambda(__UpperCAmelCase      ):
      if current_step < num_warmup_steps:
         return float(__UpperCAmelCase      ) / float(max(1							,	__UpperCAmelCase      )      )
      return max(
          0.0							,	float(num_training_steps - current_step      ) / float(max(1							,	num_training_steps - num_warmup_steps      )      )      )

   return LambdaLR(__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase = 0.5							,	__UpperCAmelCase = -1      )						->					Dict:

   '''simple docstring'''







   def lr_lambda(__UpperCAmelCase      ):
      if current_step < num_warmup_steps:
         return float(__UpperCAmelCase      ) / float(max(1							,	__UpperCAmelCase      )      )
      __SCREAMING_SNAKE_CASE      							=       float(current_step - num_warmup_steps      ) / float(max(1							,	num_training_steps - num_warmup_steps      )      )
      return max(0.0							,	0.5 * (1.0 + math.cos(math.pi * float(__UpperCAmelCase      ) * 2.0 * progress      ))      )

   return LambdaLR(__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase = 1							,	__UpperCAmelCase = -1      )						->					Tuple:

   '''simple docstring'''







   def lr_lambda(__UpperCAmelCase      ):
      if current_step < num_warmup_steps:
         return float(__UpperCAmelCase      ) / float(max(1							,	__UpperCAmelCase      )      )
      __SCREAMING_SNAKE_CASE      							=       float(current_step - num_warmup_steps      ) / float(max(1							,	num_training_steps - num_warmup_steps      )      )
      if progress >= 1.0:
         return 0.0
      return max(0.0							,	0.5 * (1.0 + math.cos(math.pi * ((float(__UpperCAmelCase      ) * progress) % 1.0)      ))      )

   return LambdaLR(__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase=1e-7							,	__UpperCAmelCase=1.0							,	__UpperCAmelCase=-1      )						->					Tuple:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       optimizer.defaults["""lr"""]
   if not (lr_init > lr_end):
      raise ValueError(f"""lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"""      )

   def lr_lambda(__UpperCAmelCase      ):
      if current_step < num_warmup_steps:
         return float(__UpperCAmelCase      ) / float(max(1							,	__UpperCAmelCase      )      )
      elif current_step > num_training_steps:
         return lr_end / lr_init  # as LambdaLR multiplies by lr_init
      else:
         __SCREAMING_SNAKE_CASE      							=       lr_init - lr_end
         __SCREAMING_SNAKE_CASE      							=       num_training_steps - num_warmup_steps
         __SCREAMING_SNAKE_CASE      							=       1 - (current_step - num_warmup_steps) / decay_steps
         __SCREAMING_SNAKE_CASE      							=       lr_range * pct_remaining**power + lr_end
         return decay / lr_init  # as LambdaLR multiplies by lr_init

   return LambdaLR(__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )


a        =						{
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}







def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase = None							,	__UpperCAmelCase = None							,	__UpperCAmelCase = None							,	__UpperCAmelCase = 1							,	__UpperCAmelCase = 1.0							,	__UpperCAmelCase = -1							,	)						->					str:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       SchedulerType(__UpperCAmelCase      )
   __SCREAMING_SNAKE_CASE      							=       TYPE_TO_SCHEDULER_FUNCTION[name]
   if name == SchedulerType.CONSTANT:
      return schedule_func(__UpperCAmelCase							,	last_epoch=__UpperCAmelCase      )

   if name == SchedulerType.PIECEWISE_CONSTANT:
      return schedule_func(__UpperCAmelCase							,	step_rules=__UpperCAmelCase							,	last_epoch=__UpperCAmelCase      )

   # All other schedulers require `num_warmup_steps`
   if num_warmup_steps is None:
      raise ValueError(f"""{name} requires `num_warmup_steps`, please provide that argument."""      )

   if name == SchedulerType.CONSTANT_WITH_WARMUP:
      return schedule_func(__UpperCAmelCase							,	num_warmup_steps=__UpperCAmelCase							,	last_epoch=__UpperCAmelCase      )

   # All other schedulers require `num_training_steps`
   if num_training_steps is None:
      raise ValueError(f"""{name} requires `num_training_steps`, please provide that argument."""      )

   if name == SchedulerType.COSINE_WITH_RESTARTS:
      return schedule_func(
          __UpperCAmelCase							,	num_warmup_steps=__UpperCAmelCase							,	num_training_steps=__UpperCAmelCase							,	num_cycles=__UpperCAmelCase							,	last_epoch=__UpperCAmelCase							,	)

   if name == SchedulerType.POLYNOMIAL:
      return schedule_func(
          __UpperCAmelCase							,	num_warmup_steps=__UpperCAmelCase							,	num_training_steps=__UpperCAmelCase							,	power=__UpperCAmelCase							,	last_epoch=__UpperCAmelCase							,	)

   return schedule_func(
       __UpperCAmelCase							,	num_warmup_steps=__UpperCAmelCase							,	num_training_steps=__UpperCAmelCase							,	last_epoch=__UpperCAmelCase      )





