

'''simple docstring'''





import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__ :     Optional[Any]				    =    logging.get_logger(__name__)


UpperCAmelCase__ :     str				    =    {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

UpperCAmelCase__ :     List[Any]				    =    {
    """vocab_file""": {
        """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"""
    },
    """merges_file""": {
        """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"""
    },
    """tokenizer_config_file""": {
        """facebook/blenderbot_small-90M""": (
            """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"""
        )
    },
}

UpperCAmelCase__ :     Any				    =    {"""facebook/blenderbot_small-90M""": 512}






def 					__lowercase      (     _A			)     ->						Union[str, Any]:
      SCREAMING_SNAKE_CASE :      Any       =   set()
      SCREAMING_SNAKE_CASE :      Dict       =   word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)			)
            SCREAMING_SNAKE_CASE :      Union[str, Any]       =   char

      SCREAMING_SNAKE_CASE :      int       =   set(_snake_case			)
      return pairs







class        a__       (   __UpperCAmelCase       ):







  """simple docstring"""




  UpperCAmelCase__      : int			       =VOCAB_FILES_NAMES
  UpperCAmelCase__      : Optional[Any]			       =PRETRAINED_VOCAB_FILES_MAP
  UpperCAmelCase__      : Optional[int]			       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCAmelCase__      : Union[str, Any]			       =["""input_ids""", """attention_mask"""]


  def __init__(			self    : int				,				UpperCAmelCase__    : Optional[int]				,				UpperCAmelCase__    : str				,				UpperCAmelCase__    : str="__start__"				,				UpperCAmelCase__    : Union[str, Any]="__end__"				,				UpperCAmelCase__    : Dict="__unk__"				,				UpperCAmelCase__    : List[Any]="__null__"				,				**UpperCAmelCase__    : Any				,				)					->List[str]:
        """simple docstring"""







        super().__init__(unk_token=UpperCAmelCase_				,				bos_token=UpperCAmelCase_				,				eos_token=UpperCAmelCase_				,				pad_token=UpperCAmelCase_				,				**UpperCAmelCase_		)

        with open(UpperCAmelCase_				,				encoding="""utf-8"""		) as vocab_handle:
              SCREAMING_SNAKE_CASE :      List[Any]       =   json.load(UpperCAmelCase_		)
        SCREAMING_SNAKE_CASE :      str       =   {v: k for k, v in self.encoder.items()}
        with open(UpperCAmelCase_				,				encoding="""utf-8"""		) as merges_handle:
              SCREAMING_SNAKE_CASE :      Optional[int]       =   merges_handle.read().split("""\n"""		)[1:-1]
        SCREAMING_SNAKE_CASE :      int       =   [tuple(merge.split()		) for merge in merges]
        SCREAMING_SNAKE_CASE :      List[Any]       =   dict(zip(UpperCAmelCase_				,				range(len(UpperCAmelCase_		)		)		)		)
        SCREAMING_SNAKE_CASE :      List[Any]       =   {}


  @property
  def  _lowercase      (			self    : Optional[int]		)					->List[Any]:
        """simple docstring"""







        return len(self.encoder		)


  def  _lowercase      (			self    : Tuple		)					->Any:
        """simple docstring"""







        return dict(self.encoder				,				**self.added_tokens_encoder		)


  def  _lowercase      (			self    : Dict				,				UpperCAmelCase__    : str		)					->str:
        """simple docstring"""







        if token in self.cache:
              return self.cache[token]
        SCREAMING_SNAKE_CASE :      int       =   re.sub("""([.,!?()])"""				,				R""" \1"""				,				UpperCAmelCase_		)
        SCREAMING_SNAKE_CASE :      Any       =   re.sub("""(\')"""				,				R""" \1 """				,				UpperCAmelCase_		)
        SCREAMING_SNAKE_CASE :      Dict       =   re.sub(R"""\s{2,}"""				,				""" """				,				UpperCAmelCase_		)
        if "\n" in token:
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   token.replace("""\n"""				,				""" __newln__"""		)

        SCREAMING_SNAKE_CASE :      Dict       =   token.split(""" """		)
        SCREAMING_SNAKE_CASE :      Optional[int]       =   []
        for token in tokens:
              if not len(UpperCAmelCase_		):
                    continue

              SCREAMING_SNAKE_CASE :      int       =   token.lower()
              SCREAMING_SNAKE_CASE :      List[Any]       =   tuple(UpperCAmelCase_		)
              SCREAMING_SNAKE_CASE :      Optional[int]       =   tuple(list(word[:-1]		) + [word[-1] + """</w>"""]		)
              SCREAMING_SNAKE_CASE :      Tuple       =   get_pairs(UpperCAmelCase_		)

              if not pairs:
                    words.append(UpperCAmelCase_		)
                    continue

              while True:
                    SCREAMING_SNAKE_CASE :      Dict       =   min(UpperCAmelCase_				,				key=lambda UpperCAmelCase__		: self.bpe_ranks.get(UpperCAmelCase_				,				float("""inf"""		)		)		)
                    if bigram not in self.bpe_ranks:
                          break
                    SCREAMING_SNAKE_CASE     ,	SCREAMING_SNAKE_CASE :      str       =   bigram
                    SCREAMING_SNAKE_CASE :      Any       =   []
                    SCREAMING_SNAKE_CASE :      Tuple       =   0

                    while i < len(UpperCAmelCase_		):
                          try:
                                SCREAMING_SNAKE_CASE :      List[Any]       =   word.index(UpperCAmelCase_				,				UpperCAmelCase_		)
                                new_word.extend(word[i:j]		)
                                SCREAMING_SNAKE_CASE :      Optional[int]       =   j
                          except ValueError:
                                new_word.extend(word[i:]		)
                                break

                          if word[i] == first and i < len(UpperCAmelCase_		) - 1 and word[i + 1] == second:
                                new_word.append(first + second		)
                                i += 2
                          else:
                                new_word.append(word[i]		)
                                i += 1
                    SCREAMING_SNAKE_CASE :      List[str]       =   tuple(UpperCAmelCase_		)
                    SCREAMING_SNAKE_CASE :      List[Any]       =   new_word
                    if len(UpperCAmelCase_		) == 1:
                          break
                    else:
                          SCREAMING_SNAKE_CASE :      List[Any]       =   get_pairs(UpperCAmelCase_		)
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   """@@ """.join(UpperCAmelCase_		)
              SCREAMING_SNAKE_CASE :      List[Any]       =   word[:-4]

              SCREAMING_SNAKE_CASE :      Tuple       =   word
              words.append(UpperCAmelCase_		)
        return " ".join(UpperCAmelCase_		)


  def  _lowercase      (			self    : Tuple				,				UpperCAmelCase__    : int		)					->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[Any]       =   []

        SCREAMING_SNAKE_CASE :      List[Any]       =   re.findall(R"""\S+\n?"""				,				UpperCAmelCase_		)

        for token in words:
              split_tokens.extend(list(self.bpe(UpperCAmelCase_		).split(""" """		)		)		)
        return split_tokens


  def  _lowercase      (			self    : List[str]				,				UpperCAmelCase__    : Tuple		)					->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Tuple       =   token.lower()
        return self.encoder.get(UpperCAmelCase_				,				self.encoder.get(self.unk_token		)		)


  def  _lowercase      (			self    : Optional[Any]				,				UpperCAmelCase__    : List[Any]		)					->Any:
        """simple docstring"""







        return self.decoder.get(UpperCAmelCase_				,				self.unk_token		)


  def  _lowercase      (			self    : str				,				UpperCAmelCase__    : Optional[Any]		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   """ """.join(UpperCAmelCase_		).replace("""@@ """				,				""""""		).strip()
        return out_string




  def  _lowercase      (			self    : Any				,				UpperCAmelCase__    : Any				,				UpperCAmelCase__    : Optional[Any] = None		)					->Any:
        """simple docstring"""







        if not os.path.isdir(UpperCAmelCase_		):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
              return
        SCREAMING_SNAKE_CASE :      str       =   os.path.join(
            UpperCAmelCase_				,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
        SCREAMING_SNAKE_CASE :      Any       =   os.path.join(
            UpperCAmelCase_				,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

        with open(UpperCAmelCase_				,				"""w"""				,				encoding="""utf-8"""		) as f:
              f.write(json.dumps(self.encoder				,				indent=2				,				sort_keys=UpperCAmelCase_				,				ensure_ascii=UpperCAmelCase_		) + """\n"""		)

        SCREAMING_SNAKE_CASE :      List[str]       =   0
        with open(UpperCAmelCase_				,				"""w"""				,				encoding="""utf-8"""		) as writer:
              writer.write("""#version: 0.2\n"""		)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,				key=lambda UpperCAmelCase__		: kv[1]		):
                    if index != token_index:
                          logger.warning(
                              f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                              """ Please check that the tokenizer is not corrupted!"""		)
                          SCREAMING_SNAKE_CASE :      Union[str, Any]       =   token_index
                    writer.write(""" """.join(UpperCAmelCase_		) + """\n"""		)
                    index += 1

        return vocab_file, merge_file







from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


UpperCAmelCase__ :     Tuple				    =    logging.get_logger(__name__)







class        a__       (   UpperCAmelCase       ):







  """simple docstring"""




  UpperCAmelCase__      : Optional[int]			       =["""input_features""", """attention_mask"""]


  def __init__(			self    : List[str]				,				UpperCAmelCase__    : Optional[Any]=8_0				,				UpperCAmelCase__    : List[str]=1_6_0_0_0				,				UpperCAmelCase__    : Optional[int]=8_0				,				UpperCAmelCase__    : List[str]=0.0				,				UpperCAmelCase__    : List[Any]=True				,				UpperCAmelCase__    : Tuple=True				,				UpperCAmelCase__    : Tuple=True				,				**UpperCAmelCase__    : Optional[int]				,				)					->Optional[int]:
        """simple docstring"""







        super().__init__(feature_size=UpperCAmelCase__				,				sampling_rate=UpperCAmelCase__				,				padding_value=UpperCAmelCase__				,				**UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Optional[int]       =   num_mel_bins
        SCREAMING_SNAKE_CASE :      Dict       =   do_ceptral_normalize
        SCREAMING_SNAKE_CASE :      Optional[int]       =   normalize_means
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   normalize_vars
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   True


  def  _lowercase      (			self    : Tuple				,				UpperCAmelCase__    : np.ndarray				,				)					->np.ndarray:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   waveform * (2**1_5)  # Kaldi compliance: 16-bit signed integers
        SCREAMING_SNAKE_CASE :      Dict       =   torch.from_numpy(UpperCAmelCase__		).unsqueeze(0		)
        SCREAMING_SNAKE_CASE :      List[Any]       =   ta_kaldi.fbank(UpperCAmelCase__				,				num_mel_bins=self.num_mel_bins				,				sample_frequency=self.sampling_rate		)
        return features.numpy()


  @staticmethod
  def  _lowercase      (			UpperCAmelCase__    : np.ndarray				,				UpperCAmelCase__    : int				,				UpperCAmelCase__    : Optional[bool] = True				,				UpperCAmelCase__    : Optional[bool] = True				,				UpperCAmelCase__    : float = 0.0				,				)					->np.ndarray:
        """simple docstring"""







        if normalize_means:
              SCREAMING_SNAKE_CASE :      Tuple       =   x[:input_length].mean(axis=0		)
              SCREAMING_SNAKE_CASE :      List[Any]       =   np.subtract(UpperCAmelCase__				,				UpperCAmelCase__		)
        if normalize_vars:
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   x[:input_length].std(axis=0		)
              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   np.divide(UpperCAmelCase__				,				UpperCAmelCase__		)

        if input_length < x.shape[0]:
              SCREAMING_SNAKE_CASE :      Tuple       =   padding_value

        # make sure array is in float32
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   x.astype(np.floataa		)

        return x


  def  _lowercase      (			self    : List[str]				,				UpperCAmelCase__    : List[np.ndarray]				,				UpperCAmelCase__    : Optional[np.ndarray] = None		)					->List[np.ndarray]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      str       =   attention_mask.sum(-1		) if attention_mask is not None else [x.shape[0] for x in input_features]
        return [
            self.utterance_cmvn(UpperCAmelCase__				,				UpperCAmelCase__				,				self.normalize_means				,				self.normalize_vars				,				self.padding_value		)
            for x, n in zip(UpperCAmelCase__				,				UpperCAmelCase__		)
        ]




  def __call__(			self    : str				,				UpperCAmelCase__    : Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]				,				UpperCAmelCase__    : Union[bool, str, PaddingStrategy] = False				,				UpperCAmelCase__    : Optional[int] = None				,				UpperCAmelCase__    : bool = False				,				UpperCAmelCase__    : Optional[int] = None				,				UpperCAmelCase__    : Optional[Union[str, TensorType]] = None				,				UpperCAmelCase__    : Optional[int] = None				,				UpperCAmelCase__    : Optional[bool] = None				,				**UpperCAmelCase__    : Optional[int]				,				)					->BatchFeature:
        """simple docstring"""







        if sampling_rate is not None:
              if sampling_rate != self.sampling_rate:
                    raise ValueError(
                        f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                        f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
                        f" {self.sampling_rate} and not {sampling_rate}."		)
        else:
              logger.warning(
                  """It is strongly recommended to pass the `sampling_rate` argument to this function. """
                  """Failing to do so can result in silent errors that might be hard to debug."""		)

        SCREAMING_SNAKE_CASE :      Tuple       =   isinstance(UpperCAmelCase__				,				np.ndarray		) and len(raw_speech.shape		) > 1
        if is_batched_numpy and len(raw_speech.shape		) > 2:
              raise ValueError(f"Only mono-channel audio is supported for input to {self}"		)
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   is_batched_numpy or (
            isinstance(UpperCAmelCase__				,				(list, tuple)		) and (isinstance(raw_speech[0]				,				(np.ndarray, tuple, list)		))
        )

        if is_batched:
              SCREAMING_SNAKE_CASE :      Dict       =   [np.asarray(UpperCAmelCase__				,				dtype=np.floataa		) for speech in raw_speech]
        elif not is_batched and not isinstance(UpperCAmelCase__				,				np.ndarray		):
              SCREAMING_SNAKE_CASE :      Any       =   np.asarray(UpperCAmelCase__				,				dtype=np.floataa		)
        elif isinstance(UpperCAmelCase__				,				np.ndarray		) and raw_speech.dtype is np.dtype(np.floataa		):
              SCREAMING_SNAKE_CASE :      Tuple       =   raw_speech.astype(np.floataa		)

        # always return batch
        if not is_batched:
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   [raw_speech]

        # extract fbank features
        SCREAMING_SNAKE_CASE :      Tuple       =   [self._extract_fbank_features(UpperCAmelCase__		) for waveform in raw_speech]

        # convert into correct format for padding
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   BatchFeature({"""input_features""": features}		)

        SCREAMING_SNAKE_CASE :      str       =   self.pad(
            UpperCAmelCase__				,				padding=UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				truncation=UpperCAmelCase__				,				pad_to_multiple_of=UpperCAmelCase__				,				return_attention_mask=UpperCAmelCase__				,				**UpperCAmelCase__				,				)

        # make sure list is in array format
        SCREAMING_SNAKE_CASE :      Dict       =   padded_inputs.get("""input_features"""		)
        if isinstance(input_features[0]				,				UpperCAmelCase__		):
              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   [np.asarray(UpperCAmelCase__				,				dtype=np.floataa		) for feature in input_features]

        SCREAMING_SNAKE_CASE :      Dict       =   padded_inputs.get("""attention_mask"""		)
        if attention_mask is not None:
              SCREAMING_SNAKE_CASE :      int       =   [np.asarray(UpperCAmelCase__				,				dtype=np.intaa		) for array in attention_mask]

        # Utterance-level cepstral mean and variance normalization
        if self.do_ceptral_normalize:
              SCREAMING_SNAKE_CASE :      Tuple       =   (
                  np.array(UpperCAmelCase__				,				dtype=np.intaa		)
                  if self._get_padding_strategies(UpperCAmelCase__				,				max_length=UpperCAmelCase__		) is not PaddingStrategy.DO_NOT_PAD
                  else None
              )
              SCREAMING_SNAKE_CASE :      str       =   self.normalize(
                  padded_inputs["""input_features"""]				,				attention_mask=UpperCAmelCase__		)

        if return_tensors is not None:
              SCREAMING_SNAKE_CASE :      Any       =   padded_inputs.convert_to_tensors(UpperCAmelCase__		)

        return padded_inputs




