


'''simple docstring'''

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_						:       List[str]      			=				logging.get_logger(__name__)

a_						:       Dict      			=				{
    """microsoft/unispeech-large-1500h-cv""": (
        """https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"""
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}





class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						"unispeech"






  def __init__(					self    ,						UpperCamelCase=32    ,						UpperCamelCase=768    ,						UpperCamelCase=12    ,						UpperCamelCase=12    ,						UpperCamelCase=3072    ,						UpperCamelCase="gelu"    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.0    ,						UpperCamelCase=0.0    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.02    ,						UpperCamelCase=1e-5    ,						UpperCamelCase="group"    ,						UpperCamelCase="gelu"    ,						UpperCamelCase=(512, 512, 512, 512, 512, 512, 512)    ,						UpperCamelCase=(5, 2, 2, 2, 2, 2, 2)    ,						UpperCamelCase=(10, 3, 3, 3, 3, 2, 2)    ,						UpperCamelCase=False    ,						UpperCamelCase=128    ,						UpperCamelCase=16    ,						UpperCamelCase=False    ,						UpperCamelCase=True    ,						UpperCamelCase=0.05    ,						UpperCamelCase=10    ,						UpperCamelCase=2    ,						UpperCamelCase=0.0    ,						UpperCamelCase=10    ,						UpperCamelCase=0    ,						UpperCamelCase=320    ,						UpperCamelCase=2    ,						UpperCamelCase=0.1    ,						UpperCamelCase=100    ,						UpperCamelCase=256    ,						UpperCamelCase=256    ,						UpperCamelCase=0.1    ,						UpperCamelCase="mean"    ,						UpperCamelCase=False    ,						UpperCamelCase=False    ,						UpperCamelCase=256    ,						UpperCamelCase=80    ,						UpperCamelCase=0    ,						UpperCamelCase=1    ,						UpperCamelCase=2    ,						UpperCamelCase=0.5    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   super().__init__(**UpperCamelCase    ,						pad_token_id=UpperCamelCase    ,						bos_token_id=UpperCamelCase    ,						eos_token_id=UpperCamelCase  )
   lowerCamelCase_  				=						hidden_size
   lowerCamelCase_  				=						feat_extract_norm
   lowerCamelCase_  				=						feat_extract_activation
   lowerCamelCase_  				=						list(UpperCamelCase  )
   lowerCamelCase_  				=						list(UpperCamelCase  )
   lowerCamelCase_  				=						list(UpperCamelCase  )
   lowerCamelCase_  				=						conv_bias
   lowerCamelCase_  				=						num_conv_pos_embeddings
   lowerCamelCase_  				=						num_conv_pos_embedding_groups
   lowerCamelCase_  				=						len(self.conv_dim  )
   lowerCamelCase_  				=						num_hidden_layers
   lowerCamelCase_  				=						intermediate_size
   lowerCamelCase_  				=						hidden_act
   lowerCamelCase_  				=						num_attention_heads
   lowerCamelCase_  				=						hidden_dropout
   lowerCamelCase_  				=						attention_dropout
   lowerCamelCase_  				=						activation_dropout
   lowerCamelCase_  				=						feat_proj_dropout
   lowerCamelCase_  				=						final_dropout
   lowerCamelCase_  				=						layerdrop
   lowerCamelCase_  				=						layer_norm_eps
   lowerCamelCase_  				=						initializer_range
   lowerCamelCase_  				=						num_ctc_classes
   lowerCamelCase_  				=						vocab_size
   lowerCamelCase_  				=						do_stable_layer_norm
   lowerCamelCase_  				=						use_weighted_layer_sum
   lowerCamelCase_  				=						classifier_proj_size

   if (
       (len(self.conv_stride  ) != self.num_feat_extract_layers)
       or (len(self.conv_kernel  ) != self.num_feat_extract_layers)
       or (len(self.conv_dim  ) != self.num_feat_extract_layers)
   ):
    raise ValueError(
        "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
        " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
        f''' {len(self.conv_dim  )}`, `len(config.conv_stride) = {len(self.conv_stride  )}`,'''
        f''' `len(config.conv_kernel) = {len(self.conv_kernel  )}`.'''  )

   # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
   lowerCamelCase_  				=						apply_spec_augment
   lowerCamelCase_  				=						mask_time_prob
   lowerCamelCase_  				=						mask_time_length
   lowerCamelCase_  				=						mask_time_min_masks
   lowerCamelCase_  				=						mask_feature_prob
   lowerCamelCase_  				=						mask_feature_length
   lowerCamelCase_  				=						mask_feature_min_masks

   # parameters for pretraining with codevector quantized representations
   lowerCamelCase_  				=						num_codevectors_per_group
   lowerCamelCase_  				=						num_codevector_groups
   lowerCamelCase_  				=						contrastive_logits_temperature
   lowerCamelCase_  				=						feat_quantizer_dropout
   lowerCamelCase_  				=						num_negatives
   lowerCamelCase_  				=						codevector_dim
   lowerCamelCase_  				=						proj_codevector_dim
   lowerCamelCase_  				=						diversity_loss_weight

   # ctc loss
   lowerCamelCase_  				=						ctc_loss_reduction
   lowerCamelCase_  				=						ctc_zero_infinity

   # pretraining loss
   lowerCamelCase_  				=						replace_prob







  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   return functools.reduce(operator.mul    ,						self.conv_stride    ,						1  )







'''simple docstring'''

from dataclasses import dataclass
from typing import Optional, Tuple

import torch
from torch import nn
from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel
from transformers.utils import ModelOutput





@dataclass
class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						None
  _lowerCamelCase						=						None
  _lowerCamelCase						=						None
  _lowerCamelCase						=						None





class   snake_case					(					lowercase    ):





  """simple docstring"""




  def __init__(					self    ,						UpperCamelCase=1    ,						UpperCamelCase=0    ,						UpperCamelCase=2    ,						UpperCamelCase=512    ,						UpperCamelCase="cls"    ,						UpperCamelCase=False    ,						UpperCamelCase=True    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   super().__init__(pad_token_id=UpperCamelCase    ,						bos_token_id=UpperCamelCase    ,						eos_token_id=UpperCamelCase    ,						**UpperCamelCase  )
   lowerCamelCase_  				=						project_dim
   lowerCamelCase_  				=						pooler_fn
   lowerCamelCase_  				=						learn_encoder
   lowerCamelCase_  				=						use_attention_mask





class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						[r"pooler", r"logit_scale"]
  _lowerCamelCase						=						[r"position_ids", r"predictions.decoder.bias"]
  _lowerCamelCase						=						"roberta"
  _lowerCamelCase						=						RobertaSeriesConfig






  def __init__(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   super().__init__(UpperCamelCase  )
   lowerCamelCase_  				=						XLMRobertaModel(UpperCamelCase  )
   lowerCamelCase_  				=						nn.Linear(config.hidden_size    ,						config.project_dim  )
   lowerCamelCase_  				=						getattr(UpperCamelCase    ,						"has_pre_transformation"    ,						UpperCamelCase  )
   if self.has_pre_transformation:
    lowerCamelCase_  				=						nn.Linear(config.hidden_size    ,						config.project_dim  )
    lowerCamelCase_  				=						nn.LayerNorm(config.hidden_size    ,						eps=config.layer_norm_eps  )
   self.post_init()







  def 						snake_case	(					self    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						UpperCamelCase = None    ,						):







   """simple docstring"""
   lowerCamelCase_  				=						return_dict if return_dict is not None else self.config.use_return_dict

   lowerCamelCase_  				=						self.base_model(
       input_ids=UpperCamelCase    ,						attention_mask=UpperCamelCase    ,						token_type_ids=UpperCamelCase    ,						position_ids=UpperCamelCase    ,						head_mask=UpperCamelCase    ,						inputs_embeds=UpperCamelCase    ,						encoder_hidden_states=UpperCamelCase    ,						encoder_attention_mask=UpperCamelCase    ,						output_attentions=UpperCamelCase    ,						output_hidden_states=True if self.has_pre_transformation else output_hidden_states    ,						return_dict=UpperCamelCase    ,						)

   if self.has_pre_transformation:
    lowerCamelCase_  				=						outputs["hidden_states"][-2]
    lowerCamelCase_  				=						self.pre_LN(UpperCamelCase  )
    lowerCamelCase_  				=						self.transformation_pre(UpperCamelCase  )

    return TransformationModelOutput(
        projection_state=UpperCamelCase    ,						last_hidden_state=outputs.last_hidden_state    ,						hidden_states=outputs.hidden_states    ,						attentions=outputs.attentions    ,						)
   else:
    lowerCamelCase_  				=						self.transformation(outputs.last_hidden_state  )
    return TransformationModelOutput(
        projection_state=UpperCamelCase    ,						last_hidden_state=outputs.last_hidden_state    ,						hidden_states=outputs.hidden_states    ,						attentions=outputs.attentions    ,						)



