


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
       from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
       __lowerCamelCase        =     True
       from torch.cuda.amp import autocast


__lowerCamelCase        =     logging.getLogger(__name__)


@dataclass
class   a__	:
       lowerCamelCase__:     str							    =     field(
           metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}    )
       lowerCamelCase__:     Optional[str]							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}						,       )
       lowerCamelCase__:     Optional[bool]							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}    )
       lowerCamelCase__:     Optional[bool]							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """Whether to log verbose messages or not."""}						,       )
       lowerCamelCase__:     Optional[float]							    =     field(
           default=2.0						,       metadata={"""help""": """Maximum temperature for gumbel softmax."""}    )
       lowerCamelCase__:     Optional[float]							    =     field(
           default=0.5						,       metadata={"""help""": """Minimum temperature for gumbel softmax."""}    )
       lowerCamelCase__:     Optional[float]							    =     field(
           default=0.99_99_95						,       metadata={"""help""": """Decay of gumbel temperature during training."""}    )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     logging.basicConfig(
         format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""  ,  datefmt="""%m/%d/%Y %H:%M:%S"""  ,  handlers=[logging.StreamHandler(sys.stdout )]  ,  )
     a_    :     List[str]										=    logging.WARNING
     if model_args.verbose_logging:
          a_    :     Union[str, Any]										=    logging.DEBUG
     elif trainer_utils.is_main_process(training_args.local_rank ):
          a_    :     Any										=    logging.INFO
     logger.setLevel(__UpperCamelCase )


@dataclass
class   a__	:
       lowerCamelCase__:     str							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}    )
       lowerCamelCase__:     Optional[str]							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}    )
       lowerCamelCase__:     Optional[str]							    =     field(
           default="""train"""						,       metadata={
               """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
           }						,       )
       lowerCamelCase__:     Optional[str]							    =     field(
           default="""validation"""						,       metadata={
               """help""": (
                   """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
               )
           }						,       )
       lowerCamelCase__:     Optional[str]							    =     field(
           default="""file"""						,       metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}						,       )
       lowerCamelCase__:     bool							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}    )
       lowerCamelCase__:     Optional[int]							    =     field(
           default=1						,       metadata={
               """help""": """The percentage of the train set used as validation set in case there's no validation split"""
           }						,       )
       lowerCamelCase__:     Optional[int]							    =     field(
           default=lowerCAmelCase_						,       metadata={"""help""": """The number of processes to use for the preprocessing."""}						,       )
       lowerCamelCase__:     Optional[float]							    =     field(
           default=20.0						,       metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}    )


@dataclass
class   a__	:
       lowerCamelCase__:     WavaVecaForPreTraining
       lowerCamelCase__:     WavaVecaFeatureExtractor
       lowerCamelCase__:     Union[bool, str]							    =     "longest"
       lowerCamelCase__:     Optional[int]							    =     None
       lowerCamelCase__:     Optional[int]							    =     None

       def __call__(   self      :     List[str]      ,							lowerCamelCase_      :     List[Dict[str, Union[List[int], torch.Tensor]]]    ):
            # reformat list to dict and set to pytorch format
            a_    :     List[Any]										=    self.feature_extractor.pad(
                lowerCamelCase_      ,							max_length=self.max_length      ,							padding=self.padding      ,							pad_to_multiple_of=self.pad_to_multiple_of      ,							return_tensors="""pt"""      ,							)
            a_    :     List[str]										=    self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]    )

            a_    :     Any										=    batch["""input_values"""].shape[0]

            # make sure that no loss is computed on padded inputs
            if batch["attention_mask"] is not None:
                 # compute real output lengths according to convolution formula
                 a_    :     List[str]										=    self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1    )    ).to(
                     torch.long    )

                 a_    :     Tuple										=    torch.zeros(
                     (batch_size, mask_indices_seq_length)      ,							dtype=torch.long      ,							device=batch["""input_values"""].device    )

                 # these two operations makes sure that all values
                 # before the output lengths indices are attended to
                 a_    :     Any										=    1
                 a_    :     List[str]										=    attention_mask.flip([-1]    ).cumsum(-1    ).flip([-1]    ).bool()

            # sample randomly masked indices
            a_    :     Optional[Any]										=    _compute_mask_indices(
                (batch_size, mask_indices_seq_length)      ,							self.model.config.mask_time_prob      ,							self.model.config.mask_time_length      ,							attention_mask=lowerCamelCase_      ,							min_masks=2      ,							)

            return batch


class   a__	(						lowerCAmelCase_    ):

       def __init__(   self      :     Union[str, Any]      ,							*lowerCamelCase_      :     Any      ,							lowerCamelCase_      :     Any=1      ,							lowerCamelCase_      :     Optional[int]=0      ,							lowerCamelCase_      :     List[str]=1.0      ,							**lowerCamelCase_      :     Any    ):
            super().__init__(*lowerCamelCase_      ,							**lowerCamelCase_    )
            a_    :     str										=    0
            a_    :     List[Any]										=    max_gumbel_temp
            a_    :     Union[str, Any]										=    min_gumbel_temp
            a_    :     Optional[int]										=    gumbel_temp_decay

       def 						UpperCAmelCase(   self      :     Optional[Any]      ,							lowerCamelCase_      :     nn.Module      ,							lowerCamelCase_      :     Dict[str, Union[torch.Tensor, Any]]    ):
            model.train()
            a_    :     int										=    self._prepare_inputs(lowerCamelCase_    )

            if self.use_amp:
                 with autocast():
                      a_    :     List[str]										=    self.compute_loss(lowerCamelCase_      ,							lowerCamelCase_    )
            else:
                 a_    :     Any										=    self.compute_loss(lowerCamelCase_      ,							lowerCamelCase_    )

            if self.args.n_gpu > 1 or self.deepspeed:
                 if model.module.config.ctc_loss_reduction == "mean":
                      a_    :     Union[str, Any]										=    loss.mean()
                 elif model.module.config.ctc_loss_reduction == "sum":
                      a_    :     Optional[int]										=    loss.sum() / (inputs["""mask_time_indices"""]).sum()
                 else:
                      raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''    )

            if self.args.gradient_accumulation_steps > 1:
                 a_    :     Optional[Any]										=    loss / self.args.gradient_accumulation_steps

            if self.use_amp:
                 self.scaler.scale(lowerCamelCase_    ).backward()
            elif self.use_apex:
                 with amp.scale_loss(lowerCamelCase_      ,							self.optimizer    ) as scaled_loss:
                      scaled_loss.backward()
            elif self.deepspeed:
                 self.deepspeed.backward(lowerCamelCase_    )
            else:
                 loss.backward()

            self.num_update_step += 1
            # make sure gumbel softmax temperature is decayed
            if self.args.n_gpu > 1 or self.deepspeed:
                 model.module.set_gumbel_temperature(
                     max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,							self.min_gumbel_temp    )    )
            else:
                 model.set_gumbel_temperature(
                     max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,							self.min_gumbel_temp    )    )

            return loss.detach()



def  _a			(						):
     # See all possible arguments in src/transformers/training_args.py
     # or by passing the --help flag to this script.
     # We now keep distinct sets of args, for a cleaner separation of concerns.

     a_    :     Optional[Any]										=    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

     a_    :     Tuple										=    parser.parse_args_into_dataclasses()
     configure_logger(__UpperCamelCase  ,  __UpperCamelCase )

     # Downloading and loading a dataset from the hub.
     a_    :     Dict										=    load_dataset(data_args.dataset_name  ,  data_args.dataset_config_name  ,  cache_dir=model_args.cache_dir )

     if "validation" not in datasets.keys():
          # make sure only "validation" and "train" keys remain"
          a_    :     Optional[Any]										=    DatasetDict()
          a_    :     Dict										=    load_dataset(
              data_args.dataset_name  ,  data_args.dataset_config_name  ,  split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''  ,  cache_dir=model_args.cache_dir  ,  )
          a_    :     Union[str, Any]										=    load_dataset(
              data_args.dataset_name  ,  data_args.dataset_config_name  ,  split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''  ,  cache_dir=model_args.cache_dir  ,  )
     else:
          # make sure only "validation" and "train" keys remain"
          a_    :     Dict										=    DatasetDict()
          a_    :     Optional[Any]										=    load_dataset(
              data_args.dataset_name  ,  data_args.dataset_config_name  ,  split="""validation"""  ,  cache_dir=model_args.cache_dir  ,  )
          a_    :     str										=    load_dataset(
              data_args.dataset_name  ,  data_args.dataset_config_name  ,  split=F'''{data_args.train_split_name}'''  ,  cache_dir=model_args.cache_dir  ,  )

     # only normalized-inputs-training is supported
     a_    :     Dict										=    WavaVecaFeatureExtractor.from_pretrained(
         model_args.model_name_or_path  ,  cache_dir=model_args.cache_dir  ,  do_normalize=__UpperCamelCase )

     def prepare_dataset(__UpperCamelCase ):
          # check that all files have the correct sampling rate
          a_    :     Dict										=    librosa.load(batch[data_args.speech_file_column]  ,  sr=feature_extractor.sampling_rate )
          return batch

     # load audio files into numpy arrays
     a_    :     int										=    datasets.map(
         __UpperCamelCase  ,  num_proc=data_args.preprocessing_num_workers  ,  remove_columns=datasets["""train"""].column_names )

     # filter audio files that are too long
     a_    :     Optional[Any]										=    vectorized_datasets.filter(
         lambda __UpperCamelCase : len(data["""speech"""] ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate ) )

     def normalize(__UpperCamelCase ):
          return feature_extractor(batch["""speech"""]  ,  sampling_rate=feature_extractor.sampling_rate )

     # normalize and transform to `BatchFeatures`
     a_    :     Optional[Any]										=    vectorized_datasets.map(
         __UpperCamelCase  ,  batched=__UpperCamelCase  ,  num_proc=data_args.preprocessing_num_workers  ,  load_from_cache_file=not data_args.overwrite_cache  ,  remove_columns=vectorized_datasets["""train"""].column_names  ,  )

     # pretraining is only supported for "newer" stable layer norm architecture
     # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
     a_    :     Tuple										=    WavaVecaConfig.from_pretrained(
         model_args.model_name_or_path  ,  cache_dir=model_args.cache_dir  ,  gradient_checkpointing=training_args.gradient_checkpointing  ,  )

     if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
          raise ValueError(
              """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
              """ ``config.feat_extract_norm='layer'""" )

     a_    :     str										=    WavaVecaForPreTraining(__UpperCamelCase )

     a_    :     Union[str, Any]										=    DataCollatorForWavaVecaPretraining(model=__UpperCamelCase  ,  feature_extractor=__UpperCamelCase )

     a_    :     Optional[int]										=    WavaVecaPreTrainer(
         model=__UpperCamelCase  ,  data_collator=__UpperCamelCase  ,  args=__UpperCamelCase  ,  train_dataset=vectorized_datasets["""train"""]  ,  eval_dataset=vectorized_datasets["""validation"""]  ,  tokenizer=__UpperCamelCase  ,  max_gumbel_temp=model_args.max_gumbel_temperature  ,  min_gumbel_temp=model_args.min_gumbel_temperature  ,  gumbel_temp_decay=model_args.gumbel_temperature_decay  ,  )
     trainer.train()


if __name__ == "__main__":
       main()






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


__lowerCamelCase        =     {'''configuration_vit''': ['''VIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ViTConfig''', '''ViTOnnxConfig''']}

try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowerCamelCase        =     ['''ViTFeatureExtractor''']
       __lowerCamelCase        =     ['''ViTImageProcessor''']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowerCamelCase        =     [
           '''VIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''ViTForImageClassification''',
           '''ViTForMaskedImageModeling''',
           '''ViTModel''',
           '''ViTPreTrainedModel''',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowerCamelCase        =     [
           '''TFViTForImageClassification''',
           '''TFViTModel''',
           '''TFViTPreTrainedModel''',
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowerCamelCase        =     [
           '''FlaxViTForImageClassification''',
           '''FlaxViTModel''',
           '''FlaxViTPreTrainedModel''',
       ]

if TYPE_CHECKING:
       from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .feature_extraction_vit import ViTFeatureExtractor
              from .image_processing_vit import ViTImageProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_vit import (
                  VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  ViTForImageClassification,
                  ViTForMaskedImageModeling,
                  ViTModel,
                  ViTPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
       import sys

       __lowerCamelCase        =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



