





import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
	from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
	_SCREAMING_SNAKE_CASE					=    True
	from torch.cuda.amp import autocast


_SCREAMING_SNAKE_CASE					=    logging.getLogger(__name__)






@dataclass
class 							__lowercase       :







	'''simple docstring'''

	a    :				Optional[Any]											=       field(
	    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}						)
	a    :				int											=       field(
	    default=lowercase__ ,      metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"} ,      )
	a    :				Any											=       field(
	    default=lowercase__ ,      metadata={"help": "Whether to freeze the feature extractor layers of the model."}						)
	a    :				Optional[Any]											=       field(
	    default=lowercase__ ,      metadata={"help": "Whether to log verbose messages or not."} ,      )
	a    :				str											=       field(
	    default=2.0 ,      metadata={"help": "Maximum temperature for gumbel softmax."}						)
	a    :				int											=       field(
	    default=0.5 ,      metadata={"help": "Minimum temperature for gumbel softmax."}						)
	a    :				List[str]											=       field(
	    default=0.99_99_95 ,      metadata={"help": "Decay of gumbel temperature during training."}						)





def    _lowerCAmelCase						(				lowerCamelCase_			:  ModelArguments		,       lowerCamelCase_			:  TrainingArguments				):
			logging.basicConfig(
			    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''		,       datefmt='''%m/%d/%Y %H:%M:%S'''		,       handlers=[logging.StreamHandler(sys.stdout				)]		,       )
			__lowercase										=							logging.WARNING
			if model_args.verbose_logging:
						__lowercase										=							logging.DEBUG
			elif trainer_utils.is_main_process(training_args.local_rank				):
						__lowercase										=							logging.INFO
			logger.setLevel(_lowerCamelCase				)






@dataclass
class 							__lowercase       :







	'''simple docstring'''

	a    :				List[str]											=       field(
	    default=lowercase__ ,      metadata={"help": "The name of the dataset to use (via the datasets library)."}						)
	a    :				List[Any]											=       field(
	    default=lowercase__ ,      metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}						)
	a    :				Any											=       field(
	    default="train" ,      metadata={
	        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
	    } ,      )
	a    :				str											=       field(
	    default="validation" ,      metadata={
	        "help": (
	            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
	        )
	    } ,      )
	a    :				str											=       field(
	    default="file" ,      metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"} ,      )
	a    :				Tuple											=       field(
	    default=lowercase__ ,      metadata={"help": "Overwrite the cached preprocessed datasets or not."}						)
	a    :				Tuple											=       field(
	    default=1 ,      metadata={
	        "help": "The percentage of the train set used as validation set in case there's no validation split"
	    } ,      )
	a    :				Optional[Any]											=       field(
	    default=lowercase__ ,      metadata={"help": "The number of processes to use for the preprocessing."} ,      )
	a    :				Dict											=       field(
	    default=20.0 ,      metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}						)






@dataclass
class 							__lowercase       :







	'''simple docstring'''

	a    :				Dict											=       42
	a    :				List[Any]											=       42
	a    :				int											=       "longest"
	a    :				Tuple											=       None
	a    :				Union[str, Any]											=       None


	def __call__(self     ,_lowerCamelCase )							->							Dict[str, torch.Tensor]:







				'''simple docstring'''





				__lowercase										=							self.feature_extractor.pad(
				    _lowerCamelCase     ,max_length=self.max_length     ,padding=self.padding     ,pad_to_multiple_of=self.pad_to_multiple_of     ,return_tensors='''pt'''     ,)
				__lowercase										=							self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1] )

				__lowercase										=							batch["input_values"].shape[0]

				# make sure that no loss is computed on padded inputs
				if batch["attention_mask"] is not None:
							# compute real output lengths according to convolution formula
							__lowercase										=							self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1 ) ).to(
							    torch.long )

							__lowercase										=							torch.zeros(
							    (batch_size, mask_indices_seq_length)     ,dtype=torch.long     ,device=batch['''input_values'''].device )

							# these two operations makes sure that all values
							# before the output lengths indices are attended to
							__lowercase										=							1
							__lowercase										=							attention_mask.flip([-1] ).cumsum(-1 ).flip([-1] ).bool()

				# sample randomly masked indices
				__lowercase										=							_compute_mask_indices(
				    (batch_size, mask_indices_seq_length)     ,self.model.config.mask_time_prob     ,self.model.config.mask_time_length     ,attention_mask=_lowerCamelCase     ,min_masks=2     ,)

				return batch






class 							__lowercase       (      lowercase__						):







	'''simple docstring'''

	def __init__(self     ,*_lowerCamelCase     ,_lowerCamelCase=1     ,_lowerCamelCase=0     ,_lowerCamelCase=1.0     ,**_lowerCamelCase )							->							Optional[int]:







				'''simple docstring'''





				super().__init__(*_lowerCamelCase     ,**_lowerCamelCase )
				__lowercase										=							0
				__lowercase										=							max_gumbel_temp
				__lowercase										=							min_gumbel_temp
				__lowercase										=							gumbel_temp_decay


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							torch.Tensor:







				'''simple docstring'''





				model.train()
				__lowercase										=							self._prepare_inputs(_lowerCamelCase )

				if self.use_amp:
							with autocast():
										__lowercase										=							self.compute_loss(_lowerCamelCase     ,_lowerCamelCase )
				else:
							__lowercase										=							self.compute_loss(_lowerCamelCase     ,_lowerCamelCase )

				if self.args.n_gpu > 1 or self.deepspeed:
							if model.module.config.ctc_loss_reduction == "mean":
										__lowercase										=							loss.mean()
							elif model.module.config.ctc_loss_reduction == "sum":
										__lowercase										=							loss.sum() / (inputs["mask_time_indices"]).sum()
							else:
										raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']" )

				if self.args.gradient_accumulation_steps > 1:
							__lowercase										=							loss / self.args.gradient_accumulation_steps

				if self.use_amp:
							self.scaler.scale(_lowerCamelCase ).backward()
				elif self.use_apex:
							with amp.scale_loss(_lowerCamelCase     ,self.optimizer ) as scaled_loss:
										scaled_loss.backward()
				elif self.deepspeed:
							self.deepspeed.backward(_lowerCamelCase )
				else:
							loss.backward()

				self.num_update_step += 1
				# make sure gumbel softmax temperature is decayed
				if self.args.n_gpu > 1 or self.deepspeed:
							model.module.set_gumbel_temperature(
							    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,self.min_gumbel_temp ) )
				else:
							model.set_gumbel_temperature(
							    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,self.min_gumbel_temp ) )

				return loss.detach()





def    _lowerCAmelCase						(				):
			__lowercase										=							HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)

			__lowercase										=							parser.parse_args_into_dataclasses()
			configure_logger(_lowerCamelCase		,       _lowerCamelCase				)

			# Downloading and loading a dataset from the hub.
			__lowercase										=							load_dataset(data_args.dataset_name		,       data_args.dataset_config_name		,       cache_dir=model_args.cache_dir				)

			if "validation" not in datasets.keys():
						# make sure only "validation" and "train" keys remain"
						__lowercase										=							DatasetDict()
						__lowercase										=							load_dataset(
						    data_args.dataset_name		,       data_args.dataset_config_name		,       split=f"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"		,       cache_dir=model_args.cache_dir		,       )
						__lowercase										=							load_dataset(
						    data_args.dataset_name		,       data_args.dataset_config_name		,       split=f"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"		,       cache_dir=model_args.cache_dir		,       )
			else:
						# make sure only "validation" and "train" keys remain"
						__lowercase										=							DatasetDict()
						__lowercase										=							load_dataset(
						    data_args.dataset_name		,       data_args.dataset_config_name		,       split='''validation'''		,       cache_dir=model_args.cache_dir		,       )
						__lowercase										=							load_dataset(
						    data_args.dataset_name		,       data_args.dataset_config_name		,       split=f"{data_args.train_split_name}"		,       cache_dir=model_args.cache_dir		,       )

			# only normalized-inputs-training is supported
			__lowercase										=							WavaVecaFeatureExtractor.from_pretrained(
			    model_args.model_name_or_path		,       cache_dir=model_args.cache_dir		,       do_normalize=_lowerCamelCase				)

			def prepare_dataset(lowerCamelCase_			:  Any				):
						# check that all files have the correct sampling rate
						__lowercase										=							librosa.load(batch[data_args.speech_file_column]		,       sr=feature_extractor.sampling_rate				)
						return batch

			# load audio files into numpy arrays
			__lowercase										=							datasets.map(
			    _lowerCamelCase		,       num_proc=data_args.preprocessing_num_workers		,       remove_columns=datasets['''train'''].column_names				)

			# filter audio files that are too long
			__lowercase										=							vectorized_datasets.filter(
			    lambda lowerCamelCase_				: len(data['''speech''']				) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate				)				)

			def normalize(lowerCamelCase_			:  Union[str, Any]				):
						return feature_extractor(batch['''speech''']		,       sampling_rate=feature_extractor.sampling_rate				)

			# normalize and transform to `BatchFeatures`
			__lowercase										=							vectorized_datasets.map(
			    _lowerCamelCase		,       batched=_lowerCamelCase		,       num_proc=data_args.preprocessing_num_workers		,       load_from_cache_file=not data_args.overwrite_cache		,       remove_columns=vectorized_datasets['''train'''].column_names		,       )

			# pretraining is only supported for "newer" stable layer norm architecture
			# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
			__lowercase										=							WavaVecaConfig.from_pretrained(
			    model_args.model_name_or_path		,       cache_dir=model_args.cache_dir		,       gradient_checkpointing=training_args.gradient_checkpointing		,       )

			if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
						raise ValueError(
						    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
						    ''' ``config.feat_extract_norm=\'layer\''''				)

			__lowercase										=							WavaVecaForPreTraining(_lowerCamelCase				)

			__lowercase										=							DataCollatorForWavaVecaPretraining(model=_lowerCamelCase		,       feature_extractor=_lowerCamelCase				)

			__lowercase										=							WavaVecaPreTrainer(
			    model=_lowerCamelCase		,       data_collator=_lowerCamelCase		,       args=_lowerCamelCase		,       train_dataset=vectorized_datasets['''train''']		,       eval_dataset=vectorized_datasets['''validation''']		,       tokenizer=_lowerCamelCase		,       max_gumbel_temp=model_args.max_gumbel_temperature		,       min_gumbel_temp=model_args.min_gumbel_temperature		,       gumbel_temp_decay=model_args.gumbel_temperature_decay		,       )
			trainer.train()


if __name__ == "__main__":
	main()












'''simple docstring'''



from argparse import ArgumentParser

from .env import EnvironmentCommand




def    _lowerCAmelCase						(				):
			__lowercase										=							ArgumentParser('''Diffusers CLI tool'''		,       usage='''diffusers-cli <command> [<args>]'''				)
			__lowercase										=							parser.add_subparsers(help='''diffusers-cli command helpers'''				)

			# Register commands
			EnvironmentCommand.register_subcommand(lowerCamelCase_				)

			# Let's go
			__lowercase										=							parser.parse_args()

			if not hasattr(lowerCamelCase_		,       '''func'''				):
						parser.print_help()
						exit(1				)

			# Run
			__lowercase										=							args.func(lowerCamelCase_				)
			service.run()


if __name__ == "__main__":
	main()





