



import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
     from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
     a_						:     Union[str, Any]	     =		True
     from torch.cuda.amp import autocast


a_						:     int	     =		logging.getLogger(__name__)





@dataclass
class    lowerCamelCase__  :
     """simple docstring"""



     _A       =			field(
         metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'})
     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}       ,    )
     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'Whether to freeze the feature extractor layers of the model.'})
     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'Whether to log verbose messages or not.'}       ,    )
     _A       =			field(
         default=2.0       ,    metadata={'help': 'Maximum temperature for gumbel softmax.'})
     _A       =			field(
         default=0.5       ,    metadata={'help': 'Minimum temperature for gumbel softmax.'})
     _A       =			field(
         default=0.99_99_95       ,    metadata={'help': 'Decay of gumbel temperature during training.'})



def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"   ,   datefmt="%m/%d/%Y %H:%M:%S"   ,   handlers=[logging.StreamHandler(sys.stdout )]   ,   )
      lowerCamelCase								=		logging.WARNING
      if model_args.verbose_logging:
            lowerCamelCase								=		logging.DEBUG
      elif trainer_utils.is_main_process(training_args.local_rank ):
            lowerCamelCase								=		logging.INFO
      logger.setLevel(UpperCAmelCase__ )





@dataclass
class    lowerCamelCase__  :
     """simple docstring"""



     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'The name of the dataset to use (via the datasets library).'})
     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'})
     _A       =			field(
         default='train'       ,    metadata={
             'help': 'The name of the training data set split to use (via the datasets library). Defaults to \'train\''
         }       ,    )
     _A       =			field(
         default='validation'       ,    metadata={
             'help': (
                 'The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''
             )
         }       ,    )
     _A       =			field(
         default='file'       ,    metadata={'help': 'Column in the dataset that contains speech file path. Defaults to \'file\''}       ,    )
     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'Overwrite the cached preprocessed datasets or not.'})
     _A       =			field(
         default=1       ,    metadata={
             'help': 'The percentage of the train set used as validation set in case there\'s no validation split'
         }       ,    )
     _A       =			field(
         default=UpperCAmelCase_       ,    metadata={'help': 'The number of processes to use for the preprocessing.'}       ,    )
     _A       =			field(
         default=20.0       ,    metadata={'help': 'Filter audio files that are longer than `max_duration_in_seconds` seconds'})





@dataclass
class    lowerCamelCase__  :
     """simple docstring"""



     _A       =			42
     _A       =			42
     _A       =			"longest"
     _A       =			None
     _A       =			None







     def __call__(self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.feature_extractor.pad(
               __a				,			max_length=self.max_length				,			padding=self.padding				,			pad_to_multiple_of=self.pad_to_multiple_of				,			return_tensors="pt"				,			)
           lowerCamelCase								=		self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]    )

           lowerCamelCase								=		batch["input_values"].shape[0]

           # make sure that no loss is computed on padded inputs
           if batch["attention_mask"] is not None:
                 # compute real output lengths according to convolution formula
                 lowerCamelCase								=		self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1    )    ).to(
                     torch.long    )

                 lowerCamelCase								=		torch.zeros(
                     (batch_size, mask_indices_seq_length)				,			dtype=torch.long				,			device=batch["input_values"].device    )

                 # these two operations makes sure that all values
                 # before the output lengths indices are attended to
                 lowerCamelCase								=		1
                 lowerCamelCase								=		attention_mask.flip([-1]    ).cumsum(-1    ).flip([-1]    ).bool()

           # sample randomly masked indices
           lowerCamelCase								=		_compute_mask_indices(
               (batch_size, mask_indices_seq_length)				,			self.model.config.mask_time_prob				,			self.model.config.mask_time_length				,			attention_mask=__a				,			min_masks=2				,			)

           return batch





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     def __init__(self				,			*__a				,			__a=1				,			__a=0				,			__a=1.0				,			**__a    ):






           '''simple docstring'''



           super().__init__(*__a				,			**__a    )
           lowerCamelCase								=		0
           lowerCamelCase								=		max_gumbel_temp
           lowerCamelCase								=		min_gumbel_temp
           lowerCamelCase								=		gumbel_temp_decay







     def 						_a      (self				,			__a				,			__a    ):






           '''simple docstring'''



           model.train()
           lowerCamelCase								=		self._prepare_inputs(__a    )

           if self.use_amp:
                 with autocast():
                       lowerCamelCase								=		self.compute_loss(__a				,			__a    )
           else:
                 lowerCamelCase								=		self.compute_loss(__a				,			__a    )

           if self.args.n_gpu > 1 or self.deepspeed:
                 if model.module.config.ctc_loss_reduction == "mean":
                       lowerCamelCase								=		loss.mean()
                 elif model.module.config.ctc_loss_reduction == "sum":
                       lowerCamelCase								=		loss.sum() / (inputs["mask_time_indices"]).sum()
                 else:
                       raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""    )

           if self.args.gradient_accumulation_steps > 1:
                 lowerCamelCase								=		loss / self.args.gradient_accumulation_steps

           if self.use_amp:
                 self.scaler.scale(__a    ).backward()
           elif self.use_apex:
                 with amp.scale_loss(__a				,			self.optimizer    ) as scaled_loss:
                       scaled_loss.backward()
           elif self.deepspeed:
                 self.deepspeed.backward(__a    )
           else:
                 loss.backward()

           self.num_update_step += 1
           # make sure gumbel softmax temperature is decayed
           if self.args.n_gpu > 1 or self.deepspeed:
                 model.module.set_gumbel_temperature(
                     max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,			self.min_gumbel_temp    )    )
           else:
                 model.set_gumbel_temperature(
                     max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,			self.min_gumbel_temp    )    )

           return loss.detach()



def     __lowercase(	):


      """simple docstring"""



      lowerCamelCase								=		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

      lowerCamelCase			,							lowerCamelCase			,							lowerCamelCase								=		parser.parse_args_into_dataclasses()
      configure_logger(UpperCAmelCase__   ,   UpperCAmelCase__ )

      # Downloading and loading a dataset from the hub.
      lowerCamelCase								=		load_dataset(data_args.dataset_name   ,   data_args.dataset_config_name   ,   cache_dir=model_args.cache_dir )

      if "validation" not in datasets.keys():
            # make sure only "validation" and "train" keys remain"
            lowerCamelCase								=		DatasetDict()
            lowerCamelCase								=		load_dataset(
                data_args.dataset_name   ,   data_args.dataset_config_name   ,   split=F"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""   ,   cache_dir=model_args.cache_dir   ,   )
            lowerCamelCase								=		load_dataset(
                data_args.dataset_name   ,   data_args.dataset_config_name   ,   split=F"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""   ,   cache_dir=model_args.cache_dir   ,   )
      else:
            # make sure only "validation" and "train" keys remain"
            lowerCamelCase								=		DatasetDict()
            lowerCamelCase								=		load_dataset(
                data_args.dataset_name   ,   data_args.dataset_config_name   ,   split="validation"   ,   cache_dir=model_args.cache_dir   ,   )
            lowerCamelCase								=		load_dataset(
                data_args.dataset_name   ,   data_args.dataset_config_name   ,   split=F"""{data_args.train_split_name}"""   ,   cache_dir=model_args.cache_dir   ,   )

      # only normalized-inputs-training is supported
      lowerCamelCase								=		WavaVecaFeatureExtractor.from_pretrained(
          model_args.model_name_or_path   ,   cache_dir=model_args.cache_dir   ,   do_normalize=UpperCAmelCase__ )

      def prepare_dataset(UpperCAmelCase__ ):
            # check that all files have the correct sampling rate
            lowerCamelCase			,							lowerCamelCase								=		librosa.load(batch[data_args.speech_file_column]   ,   sr=feature_extractor.sampling_rate )
            return batch

      # load audio files into numpy arrays
      lowerCamelCase								=		datasets.map(
          UpperCAmelCase__   ,   num_proc=data_args.preprocessing_num_workers   ,   remove_columns=datasets["train"].column_names )

      # filter audio files that are too long
      lowerCamelCase								=		vectorized_datasets.filter(
          lambda UpperCAmelCase__ : len(data["speech"] ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate ) )

      def normalize(UpperCAmelCase__ ):
            return feature_extractor(batch["speech"]   ,   sampling_rate=feature_extractor.sampling_rate )

      # normalize and transform to `BatchFeatures`
      lowerCamelCase								=		vectorized_datasets.map(
          UpperCAmelCase__   ,   batched=UpperCAmelCase__   ,   num_proc=data_args.preprocessing_num_workers   ,   load_from_cache_file=not data_args.overwrite_cache   ,   remove_columns=vectorized_datasets["train"].column_names   ,   )

      # pretraining is only supported for "newer" stable layer norm architecture
      # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
      lowerCamelCase								=		WavaVecaConfig.from_pretrained(
          model_args.model_name_or_path   ,   cache_dir=model_args.cache_dir   ,   gradient_checkpointing=training_args.gradient_checkpointing   ,   )

      if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
            raise ValueError(
                "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
                " ``config.feat_extract_norm='layer'" )

      lowerCamelCase								=		WavaVecaForPreTraining(UpperCAmelCase__ )

      lowerCamelCase								=		DataCollatorForWavaVecaPretraining(model=UpperCAmelCase__   ,   feature_extractor=UpperCAmelCase__ )

      lowerCamelCase								=		WavaVecaPreTrainer(
          model=UpperCAmelCase__   ,   data_collator=UpperCAmelCase__   ,   args=UpperCAmelCase__   ,   train_dataset=vectorized_datasets["train"]   ,   eval_dataset=vectorized_datasets["validation"]   ,   tokenizer=UpperCAmelCase__   ,   max_gumbel_temp=model_args.max_gumbel_temperature   ,   min_gumbel_temp=model_args.min_gumbel_temperature   ,   gumbel_temp_decay=model_args.gumbel_temperature_decay   ,   )
      trainer.train()


if __name__ == "__main__":
     main()




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_rembert import RemBertTokenizer
else:
     a_						:     Any	     =		None

a_						:     Optional[Any]	     =		logging.get_logger(__name__)
a_						:     int	     =		{'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

a_						:     Dict	     =		{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

a_						:     List[str]	     =		{
    'google/rembert': 2_5_6,
}

a_						:     str	     =		'▁'





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     _A       =			VOCAB_FILES_NAMES
     _A       =			PRETRAINED_VOCAB_FILES_MAP
     _A       =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A       =			RemBertTokenizer







     def __init__(self				,			__a=None				,			__a=None				,			__a=True				,			__a=True				,			__a=False				,			__a="[CLS]"				,			__a="[SEP]"				,			__a="<unk>"				,			__a="[SEP]"				,			__a="<pad>"				,			__a="[CLS]"				,			__a="[MASK]"				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		AddedToken(__a				,			lstrip=__a				,			rstrip=__a    ) if isinstance(__a				,			__a    ) else mask_token

           super().__init__(
               __a				,			tokenizer_file=__a				,			do_lower_case=__a				,			remove_space=__a				,			keep_accents=__a				,			bos_token=__a				,			eos_token=__a				,			unk_token=__a				,			sep_token=__a				,			pad_token=__a				,			cls_token=__a				,			mask_token=__a				,			**__a				,			)

           lowerCamelCase								=		do_lower_case
           lowerCamelCase								=		remove_space
           lowerCamelCase								=		keep_accents
           lowerCamelCase								=		vocab_file
           lowerCamelCase								=		False if not self.vocab_file else True







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep







     def 						_a      (self				,			__a				,			__a = None				,			__a = False    ):






           '''simple docstring'''



           if already_has_special_tokens:
                 if token_ids_a is not None:
                       raise ValueError(
                           "You should not supply a second sequence if the provided sequence of "
                           "ids is already formatted with special tokens for the model."    )
                 return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

           if token_ids_a is not None:
                 return [1] + ([0] * len(__a    )) + [1] + ([0] * len(__a    )) + [1]
           return [1] + ([0] * len(__a    )) + [1]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep    ) * [0]
           return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           if not os.path.isdir(__a    ):
                 logger.error("Vocabulary path ({}) should be a directory".format(__a    )    )
                 return
           lowerCamelCase								=		os.path.join(
               __a				,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

           if os.path.abspath(self.vocab_file    ) != os.path.abspath(__a    ):
                 copyfile(self.vocab_file				,			__a    )

           return (out_vocab_file,)