




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
		UpperCAmelCase_						=True
		from torch.cuda.amp import autocast


UpperCAmelCase_						=logging.getLogger(__name__)


@dataclass
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: str										=field(
				    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}  )
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}     ,					)
				__a							: Optional[bool]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}  )
				__a							: Optional[bool]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Whether to log verbose messages or not."""}     ,					)
				__a							: Optional[float]										=field(
				    default=2.0     ,					metadata={"""help""": """Maximum temperature for gumbel softmax."""}  )
				__a							: Optional[float]										=field(
				    default=0.5     ,					metadata={"""help""": """Minimum temperature for gumbel softmax."""}  )
				__a							: Optional[float]										=field(
				    default=0.99_99_95     ,					metadata={"""help""": """Decay of gumbel temperature during training."""}  )




def       UpperCAmelCase       (     _snake_case						, _snake_case							):
							logging.basicConfig(
							    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''						, datefmt='''%m/%d/%Y %H:%M:%S'''						, handlers=[logging.StreamHandler(sys.stdout							)]						, )
							lowerCAmelCase   =		logging.WARNING
							if model_args.verbose_logging:
														lowerCAmelCase   =		logging.DEBUG
							elif trainer_utils.is_main_process(training_args.local_rank							):
														lowerCAmelCase   =		logging.INFO
							logger.setLevel(_snake_case							)


@dataclass
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: str										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}  )
				__a							: Optional[str]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}  )
				__a							: Optional[str]										=field(
				    default="""train"""     ,					metadata={
				        """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
				    }     ,					)
				__a							: Optional[str]										=field(
				    default="""validation"""     ,					metadata={
				        """help""": (
				            """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
				        )
				    }     ,					)
				__a							: Optional[str]										=field(
				    default="""file"""     ,					metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}     ,					)
				__a							: bool										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}  )
				__a							: Optional[int]										=field(
				    default=1     ,					metadata={
				        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
				    }     ,					)
				__a							: Optional[int]										=field(
				    default=__UpperCAmelCase     ,					metadata={"""help""": """The number of processes to use for the preprocessing."""}     ,					)
				__a							: Optional[float]										=field(
				    default=20.0     ,					metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}  )




@dataclass
class 							__UpperCamelCase			:





				'''simple docstring'''



				__a							: WavaVecaForPreTraining
				__a							: WavaVecaFeatureExtractor
				__a							: Union[bool, str]										="longest"
				__a							: Optional[int]										=None
				__a							: Optional[int]										=None



				def __call__( self       , UpperCAmelCase_						):
											# reformat list to dict and set to pytorch format
											lowerCAmelCase   =		self.feature_extractor.pad(
											    UpperCAmelCase_       , max_length=self.max_length       , padding=self.padding       , pad_to_multiple_of=self.pad_to_multiple_of       , return_tensors='''pt'''       , )
											lowerCAmelCase   =		self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]						)

											lowerCAmelCase   =		batch['''input_values'''].shape[0]

											# make sure that no loss is computed on padded inputs
											if batch["attention_mask"] is not None:
																		# compute real output lengths according to convolution formula
																		lowerCAmelCase   =		self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1						)						).to(
																		    torch.long						)

																		lowerCAmelCase   =		torch.zeros(
																		    (batch_size, mask_indices_seq_length)       , dtype=torch.long       , device=batch['''input_values'''].device						)

																		# these two operations makes sure that all values
																		# before the output lengths indices are attended to
																		lowerCAmelCase   =		1
																		lowerCAmelCase   =		attention_mask.flip([-1]						).cumsum(-1						).flip([-1]						).bool()

											# sample randomly masked indices
											lowerCAmelCase   =		_compute_mask_indices(
											    (batch_size, mask_indices_seq_length)       , self.model.config.mask_time_prob       , self.model.config.mask_time_length       , attention_mask=UpperCAmelCase_       , min_masks=2       , )

											return batch




class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				def __init__( self       , *UpperCAmelCase_       , UpperCAmelCase_=1       , UpperCAmelCase_=0       , UpperCAmelCase_=1.0       , **UpperCAmelCase_						):
											super().__init__(*UpperCAmelCase_       , **UpperCAmelCase_						)
											lowerCAmelCase   =		0
											lowerCAmelCase   =		max_gumbel_temp
											lowerCAmelCase   =		min_gumbel_temp
											lowerCAmelCase   =		gumbel_temp_decay






				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_						):
											model.train()
											lowerCAmelCase   =		self._prepare_inputs(UpperCAmelCase_						)

											if self.use_amp:
																		with autocast():
																									lowerCAmelCase   =		self.compute_loss(UpperCAmelCase_       , UpperCAmelCase_						)
											else:
																		lowerCAmelCase   =		self.compute_loss(UpperCAmelCase_       , UpperCAmelCase_						)

											if self.args.n_gpu > 1 or self.deepspeed:
																		if model.module.config.ctc_loss_reduction == "mean":
																									lowerCAmelCase   =		loss.mean()
																		elif model.module.config.ctc_loss_reduction == "sum":
																									lowerCAmelCase   =		loss.sum() / (inputs['''mask_time_indices''']).sum()
																		else:
																									raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""						)

											if self.args.gradient_accumulation_steps > 1:
																		lowerCAmelCase   =		loss / self.args.gradient_accumulation_steps

											if self.use_amp:
																		self.scaler.scale(UpperCAmelCase_						).backward()
											elif self.use_apex:
																		with amp.scale_loss(UpperCAmelCase_       , self.optimizer						) as scaled_loss:
																									scaled_loss.backward()
											elif self.deepspeed:
																		self.deepspeed.backward(UpperCAmelCase_						)
											else:
																		loss.backward()

											self.num_update_step += 1
											# make sure gumbel softmax temperature is decayed
											if self.args.n_gpu > 1 or self.deepspeed:
																		model.module.set_gumbel_temperature(
																		    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       , self.min_gumbel_temp						)						)
											else:
																		model.set_gumbel_temperature(
																		    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       , self.min_gumbel_temp						)						)

											return loss.detach()




def       UpperCAmelCase       (     ):
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							lowerCAmelCase   =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)

							lowerCAmelCase   ,     lowerCAmelCase   ,     lowerCAmelCase   =		parser.parse_args_into_dataclasses()
							configure_logger(_snake_case						, _snake_case							)

							# Downloading and loading a dataset from the hub.
							lowerCAmelCase   =		load_dataset(data_args.dataset_name						, data_args.dataset_config_name						, cache_dir=model_args.cache_dir							)

							if "validation" not in datasets.keys():
														# make sure only "validation" and "train" keys remain"
														lowerCAmelCase   =		DatasetDict()
														lowerCAmelCase   =		load_dataset(
														    data_args.dataset_name						, data_args.dataset_config_name						, split=F"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""						, cache_dir=model_args.cache_dir						, )
														lowerCAmelCase   =		load_dataset(
														    data_args.dataset_name						, data_args.dataset_config_name						, split=F"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""						, cache_dir=model_args.cache_dir						, )
							else:
														# make sure only "validation" and "train" keys remain"
														lowerCAmelCase   =		DatasetDict()
														lowerCAmelCase   =		load_dataset(
														    data_args.dataset_name						, data_args.dataset_config_name						, split='''validation'''						, cache_dir=model_args.cache_dir						, )
														lowerCAmelCase   =		load_dataset(
														    data_args.dataset_name						, data_args.dataset_config_name						, split=F"""{data_args.train_split_name}"""						, cache_dir=model_args.cache_dir						, )

							# only normalized-inputs-training is supported
							lowerCAmelCase   =		WavaVecaFeatureExtractor.from_pretrained(
							    model_args.model_name_or_path						, cache_dir=model_args.cache_dir						, do_normalize=_snake_case							)

							def prepare_dataset(_snake_case							):
														# check that all files have the correct sampling rate
														lowerCAmelCase   ,     lowerCAmelCase   =		librosa.load(batch[data_args.speech_file_column]						, sr=feature_extractor.sampling_rate							)
														return batch

							# load audio files into numpy arrays
							lowerCAmelCase   =		datasets.map(
							    _snake_case						, num_proc=data_args.preprocessing_num_workers						, remove_columns=datasets['''train'''].column_names							)

							# filter audio files that are too long
							lowerCAmelCase   =		vectorized_datasets.filter(
							    lambda _snake_case							: len(data['''speech''']							) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate							)							)

							def normalize(_snake_case							):
														return feature_extractor(batch['''speech''']						, sampling_rate=feature_extractor.sampling_rate							)

							# normalize and transform to `BatchFeatures`
							lowerCAmelCase   =		vectorized_datasets.map(
							    _snake_case						, batched=_snake_case						, num_proc=data_args.preprocessing_num_workers						, load_from_cache_file=not data_args.overwrite_cache						, remove_columns=vectorized_datasets['''train'''].column_names						, )

							# pretraining is only supported for "newer" stable layer norm architecture
							# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
							lowerCAmelCase   =		WavaVecaConfig.from_pretrained(
							    model_args.model_name_or_path						, cache_dir=model_args.cache_dir						, gradient_checkpointing=training_args.gradient_checkpointing						, )

							if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
														raise ValueError(
														    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
														    ''' ``config.feat_extract_norm=\'layer\''''							)

							lowerCAmelCase   =		WavaVecaForPreTraining(_snake_case							)

							lowerCAmelCase   =		DataCollatorForWavaVecaPretraining(model=_snake_case						, feature_extractor=_snake_case							)

							lowerCAmelCase   =		WavaVecaPreTrainer(
							    model=_snake_case						, data_collator=_snake_case						, args=_snake_case						, train_dataset=vectorized_datasets['''train''']						, eval_dataset=vectorized_datasets['''validation''']						, tokenizer=_snake_case						, max_gumbel_temp=model_args.max_gumbel_temperature						, min_gumbel_temp=model_args.min_gumbel_temperature						, gumbel_temp_decay=model_args.gumbel_temperature_decay						, )
							trainer.train()


if __name__ == "__main__":
		main()









from collections.abc import Sequence



def       UpperCAmelCase       (     _snake_case						, _snake_case = False							):
							if not arr:
														return 0

							lowerCAmelCase   =		0 if allow_empty_subarrays else float('''-inf'''							)
							lowerCAmelCase   =		0.0
							for num in arr:
														lowerCAmelCase   =		max(0 if allow_empty_subarrays else num						, curr_sum + num							)
														lowerCAmelCase   =		max(_snake_case						, _snake_case							)

							return max_sum


if __name__ == "__main__":
		from doctest import testmod

		testmod()

		UpperCAmelCase_						=[-2, 1, -3, 4, -1, 2, 1, -5, 4]
		print(F'''{max_subarray_sum(nums) = }''')



