






'''simple docstring'''



from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


SCREAMING_SNAKE_CASE						=	logging.get_logger(__name__)
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			['''input_features''', '''attention_mask''']






      def __init__( self			:		Dict     ,      UpperCAmelCase			:		List[str]=80     ,      UpperCAmelCase			:		Optional[int]=1_6000     ,      UpperCAmelCase			:		Tuple=80     ,      UpperCAmelCase			:		Dict=0.0     ,      UpperCAmelCase			:		str=True     ,      UpperCAmelCase			:		Union[str, Any]=True     ,      UpperCAmelCase			:		str=True     ,      **UpperCAmelCase			:		Union[str, Any]     ,      )  ->  Optional[Any]:


       '''simple docstring'''

       super().__init__(feature_size=UpperCAmelCase     ,      sampling_rate=UpperCAmelCase     ,      padding_value=UpperCAmelCase     ,      **UpperCAmelCase		)
       lowercase					:   Dict				  =num_mel_bins
       lowercase					:   str				  =do_ceptral_normalize
       lowercase					:   Union[str, Any]				  =normalize_means
       lowercase					:   Optional[Any]				  =normalize_vars
       lowercase					:   List[str]				  =True






      def 				A__      ( self			:		List[str]     ,      UpperCAmelCase			:		np.ndarray     ,      )  ->  np.ndarray:


       '''simple docstring'''

       lowercase					:   Tuple				  =waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
       lowercase					:   List[str]				  =torch.from_numpy(UpperCAmelCase		).unsqueeze(0		)
       lowercase					:   Dict				  =ta_kaldi.fbank(UpperCAmelCase     ,      num_mel_bins=self.num_mel_bins     ,      sample_frequency=self.sampling_rate		)
       return features.numpy()






      @staticmethod
      def 				A__      ( UpperCAmelCase			:		np.ndarray     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Optional[bool] = True     ,      UpperCAmelCase			:		Optional[bool] = True     ,      UpperCAmelCase			:		float = 0.0     ,      )  ->  np.ndarray:


       '''simple docstring'''

       if normalize_means:
        lowercase					:   str				  =x[:input_length].mean(axis=0		)
        lowercase					:   Union[str, Any]				  =np.subtract(UpperCAmelCase     ,      UpperCAmelCase		)
       if normalize_vars:
        lowercase					:   List[str]				  =x[:input_length].std(axis=0		)
        lowercase					:   Optional[int]				  =np.divide(UpperCAmelCase     ,      UpperCAmelCase		)

       if input_length < x.shape[0]:
        lowercase					:   Any				  =padding_value

       # make sure array is in float32
       lowercase					:   Any				  =x.astype(np.floataa		)

       return x






      def 				A__      ( self			:		Optional[Any]     ,      UpperCAmelCase			:		List[np.ndarray]     ,      UpperCAmelCase			:		Optional[np.ndarray] = None		)  ->  List[np.ndarray]:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =attention_mask.sum(-1		) if attention_mask is not None else [x.shape[0] for x in input_features]
       return [
           self.utterance_cmvn(UpperCAmelCase     ,      UpperCAmelCase     ,      self.normalize_means     ,      self.normalize_vars     ,      self.padding_value		)
           for x, n in zip(UpperCAmelCase     ,      UpperCAmelCase		)
       ]






      def __call__( self			:		Optional[Any]     ,      UpperCAmelCase			:		Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]     ,      UpperCAmelCase			:		Union[bool, str, PaddingStrategy] = False     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		bool = False     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		Optional[Union[str, TensorType]] = None     ,      UpperCAmelCase			:		Optional[int] = None     ,      UpperCAmelCase			:		Optional[bool] = None     ,      **UpperCAmelCase			:		List[Any]     ,      )  ->  BatchFeature:


       '''simple docstring'''

       if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
         raise ValueError(
             f'The model corresponding to this feature extractor: {self} was trained using a sampling rate of'
             f' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'
             f' {self.sampling_rate} and not {sampling_rate}.'		)
       else:
        logger.warning(
            '''It is strongly recommended to pass the `sampling_rate` argument to this function. '''
            '''Failing to do so can result in silent errors that might be hard to debug.'''		)

       lowercase					:   Optional[Any]				  =isinstance(UpperCAmelCase     ,      np.ndarray		) and len(raw_speech.shape		) > 1
       if is_batched_numpy and len(raw_speech.shape		) > 2:
        raise ValueError(f'Only mono-channel audio is supported for input to {self}'		)
       lowercase					:   Optional[int]				  =is_batched_numpy or (
           isinstance(UpperCAmelCase     ,      (list, tuple)		) and (isinstance(raw_speech[0]     ,      (np.ndarray, tuple, list)		))
       )

       if is_batched:
        lowercase					:   List[str]				  =[np.asarray(UpperCAmelCase     ,      dtype=np.floataa		) for speech in raw_speech]
       elif not is_batched and not isinstance(UpperCAmelCase     ,      np.ndarray		):
        lowercase					:   int				  =np.asarray(UpperCAmelCase     ,      dtype=np.floataa		)
       elif isinstance(UpperCAmelCase     ,      np.ndarray		) and raw_speech.dtype is np.dtype(np.floataa		):
        lowercase					:   Tuple				  =raw_speech.astype(np.floataa		)

       # always return batch
       if not is_batched:
        lowercase					:   Any				  =[raw_speech]

       # extract fbank features
       lowercase					:   Any				  =[self._extract_fbank_features(UpperCAmelCase		) for waveform in raw_speech]

       # convert into correct format for padding
       lowercase					:   Optional[Any]				  =BatchFeature({'''input_features''': features}		)

       lowercase					:   List[str]				  =self.pad(
           UpperCAmelCase     ,      padding=UpperCAmelCase     ,      max_length=UpperCAmelCase     ,      truncation=UpperCAmelCase     ,      pad_to_multiple_of=UpperCAmelCase     ,      return_attention_mask=UpperCAmelCase     ,      **UpperCAmelCase     ,      )

       # make sure list is in array format
       lowercase					:   Any				  =padded_inputs.get('''input_features'''		)
       if isinstance(input_features[0]     ,      UpperCAmelCase		):
        lowercase					:   Optional[Any]				  =[np.asarray(UpperCAmelCase     ,      dtype=np.floataa		) for feature in input_features]

       lowercase					:   Union[str, Any]				  =padded_inputs.get('''attention_mask'''		)
       if attention_mask is not None:
        lowercase					:   int				  =[np.asarray(UpperCAmelCase     ,      dtype=np.intaa		) for array in attention_mask]

       # Utterance-level cepstral mean and variance normalization
       if self.do_ceptral_normalize:
        lowercase					:   List[str]				  =(
            np.array(UpperCAmelCase     ,      dtype=np.intaa		)
            if self._get_padding_strategies(UpperCAmelCase     ,      max_length=UpperCAmelCase		) is not PaddingStrategy.DO_NOT_PAD
            else None
        )
        lowercase					:   int				  =self.normalize(
            padded_inputs['''input_features''']     ,      attention_mask=UpperCAmelCase		)

       if return_tensors is not None:
        lowercase					:   Dict				  =padded_inputs.convert_to_tensors(UpperCAmelCase		)

       return padded_inputs











'''simple docstring'''



from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


SCREAMING_SNAKE_CASE						=	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						=	{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE						=	{
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

SCREAMING_SNAKE_CASE						=	{'allegro/herbert-base-cased': 514}
SCREAMING_SNAKE_CASE						=	{}
class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			VOCAB_FILES_NAMES
      UpperCamelCase_								=			PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_								=			PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase_								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_								=			HerbertTokenizer






      def __init__( self			:		Dict     ,      UpperCAmelCase			:		Optional[Any]=None     ,      UpperCAmelCase			:		Optional[int]=None     ,      UpperCAmelCase			:		Any=None     ,      UpperCAmelCase			:		Tuple="<s>"     ,      UpperCAmelCase			:		List[Any]="<unk>"     ,      UpperCAmelCase			:		str="<pad>"     ,      UpperCAmelCase			:		Optional[Any]="<mask>"     ,      UpperCAmelCase			:		List[str]="</s>"     ,      **UpperCAmelCase			:		List[str]     ,      )  ->  Dict:


       '''simple docstring'''

       super().__init__(
           UpperCAmelCase     ,      UpperCAmelCase     ,      tokenizer_file=UpperCAmelCase     ,      cls_token=UpperCAmelCase     ,      unk_token=UpperCAmelCase     ,      pad_token=UpperCAmelCase     ,      mask_token=UpperCAmelCase     ,      sep_token=UpperCAmelCase     ,      **UpperCAmelCase     ,      )






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		List[int]     ,      UpperCAmelCase			:		Optional[List[int]] = None		)  ->  List[int]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =[self.cls_token_id]
       lowercase					:   Any				  =[self.sep_token_id]
       if token_ids_a is None:
        return cls + token_ids_a + sep

       return cls + token_ids_a + sep + token_ids_a + sep






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		List[int]     ,      UpperCAmelCase			:		Optional[List[int]] = None     ,      UpperCAmelCase			:		bool = False		)  ->  List[int]:


       '''simple docstring'''

       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=UpperCAmelCase     ,      token_ids_a=UpperCAmelCase     ,      already_has_special_tokens=UpperCAmelCase		)

       if token_ids_a is None:
        return [1] + ([0] * len(UpperCAmelCase		)) + [1]
       return [1] + ([0] * len(UpperCAmelCase		)) + [1] + ([0] * len(UpperCAmelCase		)) + [1]






      def 				A__      ( self			:		Optional[int]     ,      UpperCAmelCase			:		List[int]     ,      UpperCAmelCase			:		Optional[List[int]] = None		)  ->  List[int]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =[self.sep_token_id]
       lowercase					:   Dict				  =[self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






      def 				A__      ( self			:		Union[str, Any]     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		Optional[str] = None		)  ->  Tuple[str]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =self._tokenizer.model.save(UpperCAmelCase     ,      name=UpperCAmelCase		)
       return tuple(UpperCAmelCase		)



