

"""simple docstring"""






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


_lowerCAmelCase						=		logging.get_logger(__name__)

_lowerCAmelCase						=		{
    """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/config.json""",
    # See all BlenderbotSmall models at https://huggingface.co/models?filter=blenderbot_small
}







class 							__UpperCamelCase    (    a__						):
  _UpperCAmelCase          =		"blenderbot-small"
  _UpperCAmelCase          =		["past_key_values"]
  _UpperCAmelCase          =		{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

  def __init__(		self    ,_A=5_0265    ,_A=512    ,_A=8    ,_A=2048    ,_A=16    ,_A=8    ,_A=2048    ,_A=16    ,_A=0.0    ,_A=0.0    ,_A=True    ,_A=True    ,_A="gelu"    ,_A=512    ,_A=0.1    ,_A=0.0    ,_A=0.0    ,_A=0.0_2    ,_A=1    ,_A=False    ,_A=0    ,_A=1    ,_A=2    ,_A=2    ,**_A    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    vocab_size
      _lowerCAmelCase       :   Dict  =    max_position_embeddings
      _lowerCAmelCase       :   List[str]  =    d_model
      _lowerCAmelCase       :   List[Any]  =    encoder_ffn_dim
      _lowerCAmelCase       :   int  =    encoder_layers
      _lowerCAmelCase       :   List[str]  =    encoder_attention_heads
      _lowerCAmelCase       :   List[Any]  =    decoder_ffn_dim
      _lowerCAmelCase       :   Dict  =    decoder_layers
      _lowerCAmelCase       :   Tuple  =    decoder_attention_heads
      _lowerCAmelCase       :   str  =    dropout
      _lowerCAmelCase       :   Any  =    attention_dropout
      _lowerCAmelCase       :   str  =    activation_dropout
      _lowerCAmelCase       :   Dict  =    activation_function
      _lowerCAmelCase       :   Optional[Any]  =    init_std
      _lowerCAmelCase       :   Optional[Any]  =    encoder_layerdrop
      _lowerCAmelCase       :   Tuple  =    decoder_layerdrop
      _lowerCAmelCase       :   List[str]  =    use_cache
      _lowerCAmelCase       :   Union[str, Any]  =    encoder_layers
      _lowerCAmelCase       :   Union[str, Any]  =    scale_embedding  # scale factor will be sqrt(d_model) if True

      super().__init__(
          pad_token_id=_A    ,bos_token_id=_A    ,eos_token_id=_A    ,is_encoder_decoder=_A    ,decoder_start_token_id=_A    ,forced_eos_token_id=_A    ,**_A    ,)







class 							__UpperCamelCase    (    a__						):

  @property
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      if self.task in ["default", "seq2seq-lm"]:
          _lowerCAmelCase       :   Union[str, Any]  =    OrderedDict(
              [
                  ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                  ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
              ]      )

          if self.use_past:
              _lowerCAmelCase       :   str  =    {0: 'batch'}
              _lowerCAmelCase       :   Optional[Any]  =    {0: 'batch', 1: 'past_decoder_sequence + sequence'}
          else:
              _lowerCAmelCase       :   Optional[Any]  =    {0: 'batch', 1: 'decoder_sequence'}
              _lowerCAmelCase       :   Optional[int]  =    {0: 'batch', 1: 'decoder_sequence'}

          if self.use_past:
              self.fill_with_past_key_values_(_A    ,direction='inputs'      )
      elif self.task == "causal-lm":
          # TODO: figure this case out.
          _lowerCAmelCase       :   Optional[Any]  =    OrderedDict(
              [
                  ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                  ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
              ]      )
          if self.use_past:
              _lowerCAmelCase       :   Any  =    self.num_layers
              for i in range(_A      ):
                  _lowerCAmelCase       :   List[Any]  =    {0: 'batch', 2: 'past_sequence + sequence'}
                  _lowerCAmelCase       :   int  =    {0: 'batch', 2: 'past_sequence + sequence'}
      else:
          _lowerCAmelCase       :   List[Any]  =    OrderedDict(
              [
                  ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                  ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                  ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
                  ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
              ]      )

      return common_inputs

  @property
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      if self.task in ["default", "seq2seq-lm"]:
          _lowerCAmelCase       :   Union[str, Any]  =    super().outputs
      else:
          _lowerCAmelCase       :   Optional[int]  =    super(_A    ,self      ).outputs
          if self.use_past:
              _lowerCAmelCase       :   Optional[Any]  =    self.num_layers
              for i in range(_A      ):
                  _lowerCAmelCase       :   Any  =    {0: 'batch', 2: 'past_sequence + sequence'}
                  _lowerCAmelCase       :   Tuple  =    {0: 'batch', 2: 'past_sequence + sequence'}
      return common_outputs

  def __lowerCamelCase		(		self    ,_A    ,_A = -1    ,_A = -1    ,_A = False    ,_A = None    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   List[str]  =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          _A    ,_A    ,_A    ,_A    ,_A      )

      # Generate decoder inputs
      _lowerCAmelCase       :   List[str]  =    seq_length if not self.use_past else 1
      _lowerCAmelCase       :   List[str]  =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          _A    ,_A    ,_A    ,_A    ,_A      )
      _lowerCAmelCase       :   str  =    {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
      _lowerCAmelCase       :   Optional[int]  =    dict(**_A    ,**_A      )

      if self.use_past:
          if not is_torch_available():
              raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
          else:
              import torch
          _lowerCAmelCase       :   str  =    common_inputs['input_ids'].shape
          _lowerCAmelCase       :   Tuple  =    common_inputs['decoder_input_ids'].shape[1]
          _lowerCAmelCase       :   int  =    self.num_attention_heads
          _lowerCAmelCase       :   Any  =    (
              batch,
              num_encoder_attention_heads,
              encoder_seq_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )
          _lowerCAmelCase       :   List[Any]  =    decoder_seq_length + 3
          _lowerCAmelCase       :   List[str]  =    (
              batch,
              num_decoder_attention_heads,
              decoder_past_length,
              self._config.hidden_size // num_decoder_attention_heads,
          )

          _lowerCAmelCase       :   Optional[Any]  =    torch.cat(
              [common_inputs['decoder_attention_mask'], torch.ones(_A    ,_A      )]    ,dim=1      )

          _lowerCAmelCase       :   str  =    []
          # If the number of encoder and decoder layers are present in the model configuration, both are considered
          _lowerCAmelCase       :   Optional[Any]  =    self.num_layers
          _lowerCAmelCase       :   List[Any]  =    min(_A    ,_A      )
          _lowerCAmelCase       :   str  =    max(_A    ,_A      ) - min_num_layers
          _lowerCAmelCase       :   Any  =    'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

          for _ in range(_A      ):
              common_inputs["past_key_values"].append(
                  (
                      torch.zeros(_A      ),
                      torch.zeros(_A      ),
                      torch.zeros(_A      ),
                      torch.zeros(_A      ),
                  )      )
          # TODO: test this.
          _lowerCAmelCase       :   Dict  =    encoder_shape if remaining_side_name == 'encoder' else decoder_shape
          for _ in range(_A    ,_A      ):
              common_inputs["past_key_values"].append((torch.zeros(_A      ), torch.zeros(_A      ))      )
      return common_inputs

  def __lowerCamelCase		(		self    ,_A    ,_A = -1    ,_A = -1    ,_A = False    ,_A = None    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          _A    ,_A    ,_A    ,_A    ,_A      )

      if self.use_past:
          if not is_torch_available():
              raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
          else:
              import torch
          _lowerCAmelCase       :   List[Any]  =    common_inputs['input_ids'].shape
          # Not using the same length for past_key_values
          _lowerCAmelCase       :   str  =    seqlen + 2
          _lowerCAmelCase       :   Dict  =    self.num_layers
          _lowerCAmelCase       :   Any  =    self.num_attention_heads
          _lowerCAmelCase       :   Optional[int]  =    (
              batch,
              num_encoder_attention_heads,
              past_key_values_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )

          _lowerCAmelCase       :   Optional[int]  =    common_inputs['attention_mask'].dtype
          _lowerCAmelCase       :   Dict  =    torch.cat(
              [common_inputs['attention_mask'], torch.ones(_A    ,_A    ,dtype=_A      )]    ,dim=1      )
          _lowerCAmelCase       :   str  =    [
              (torch.zeros(_A      ), torch.zeros(_A      )) for _ in range(_A      )
          ]
      return common_inputs

  def __lowerCamelCase		(		self    ,_A    ,_A = -1    ,_A = -1    ,_A = False    ,_A = None    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   str  =    compute_effective_axis_dimension(
          _A    ,fixed_dimension=OnnxConfig.default_fixed_batch    ,num_token_to_add=0      )

      # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
      _lowerCAmelCase       :   Optional[int]  =    tokenizer.num_special_tokens_to_add(_A      )
      _lowerCAmelCase       :   str  =    compute_effective_axis_dimension(
          _A    ,fixed_dimension=OnnxConfig.default_fixed_sequence    ,num_token_to_add=_A      )

      # Generate dummy inputs according to compute batch and sequence
      _lowerCAmelCase       :   str  =    [' '.join([tokenizer.unk_token]      ) * seq_length] * batch_size
      _lowerCAmelCase       :   Tuple  =    dict(tokenizer(_A    ,return_tensors=_A      )      )
      return common_inputs

  def __lowerCamelCase		(		self    ,_A    ,_A = -1    ,_A = -1    ,_A = False    ,_A = None    ,):
      '''simple docstring'''

      if self.task in ["default", "seq2seq-lm"]:
          _lowerCAmelCase       :   Dict  =    self._generate_dummy_inputs_for_default_and_seqaseq_lm(
              _A    ,batch_size=_A    ,seq_length=_A    ,is_pair=_A    ,framework=_A      )

      elif self.task == "causal-lm":
          _lowerCAmelCase       :   Optional[Any]  =    self._generate_dummy_inputs_for_causal_lm(
              _A    ,batch_size=_A    ,seq_length=_A    ,is_pair=_A    ,framework=_A      )
      else:
          _lowerCAmelCase       :   str  =    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
              _A    ,batch_size=_A    ,seq_length=_A    ,is_pair=_A    ,framework=_A      )

      return common_inputs







  def __lowerCamelCase		(		self    ,_A    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      if self.task in ["default", "seq2seq-lm"]:
          _lowerCAmelCase       :   Optional[Any]  =    super()._flatten_past_key_values_(_A    ,_A    ,_A    ,_A      )
      else:
          _lowerCAmelCase       :   str  =    super(_A    ,self      )._flatten_past_key_values_(
              _A    ,_A    ,_A    ,_A      )








"""simple docstring"""






import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase						=		logging.get_logger(__name__)


_lowerCAmelCase						=		{
    """vocab_file""": """vocab.json""",
    """tokenizer_config_file""": """tokenizer_config.json""",
    """merges_file""": """merges.txt""",
}

_lowerCAmelCase						=		{
    """vocab_file""": {
        """facebook/s2t-wav2vec2-large-en-de""": (
            """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json"""
        ),
    },
    """tokenizer_config_file""": {
        """facebook/s2t-wav2vec2-large-en-de""": (
            """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json"""
        ),
    },
    """merges_file""": {
        """facebook/s2t-wav2vec2-large-en-de""": (
            """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt"""
        ),
    },
}

_lowerCAmelCase						=		"""</w>"""
_lowerCAmelCase						=		"""@@ """

def 		lowerCamelCase__		(					_lowerCamelCase   ):

    '''simple docstring'''



    _lowerCAmelCase       :   List[str]  =    set()
    _lowerCAmelCase       :   Dict  =    word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)   )
        _lowerCAmelCase       :   Any  =    char
    return pairs


# Speech2Text2 has no max input length
_lowerCAmelCase						=		{"""facebook/s2t-wav2vec2-large-en-de""": 1_0_2_4}







class 	__UpperCamelCase    (    a__						):
  _UpperCAmelCase          =		VOCAB_FILES_NAMES
  _UpperCAmelCase          =		PRETRAINED_VOCAB_FILES_MAP
  _UpperCAmelCase          =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCAmelCase          =		["input_ids", "attention_mask"]

  def __init__(		self    ,_A    ,_A="<s>"    ,_A="<pad>"    ,_A="</s>"    ,_A="<unk>"    ,_A=False    ,_A=None    ,**_A    ,):
      '''simple docstring'''

      super().__init__(
          unk_token=_A    ,bos_token=_A    ,eos_token=_A    ,pad_token=_A    ,do_lower_case=_A    ,**_A    ,)

      _lowerCAmelCase       :   List[Any]  =    do_lower_case

      with open(_A    ,encoding='utf-8'      ) as vocab_handle:
          _lowerCAmelCase       :   Optional[int]  =    json.load(_A      )
      _lowerCAmelCase       :   Tuple  =    {v: k for k, v in self.encoder.items()}

      if merges_file is None:
          logger.info(F"""No merges files provided. {self.__class__.__name__} can only be used for decoding."""      )

          _lowerCAmelCase       :   Optional[Any]  =    None
          _lowerCAmelCase       :   Tuple  =    None
      else:
          with open(_A    ,encoding='utf-8'      ) as merges_handle:
              _lowerCAmelCase       :   Optional[Any]  =    merges_handle.read().split('\n'      )[:-1]

          _lowerCAmelCase       :   List[str]  =    [tuple(merge.split()[:2]      ) for merge in merges]
          _lowerCAmelCase       :   List[Any]  =    dict(zip(_A    ,range(len(_A      )      )      )      )
          _lowerCAmelCase       :   Union[str, Any]  =    {}

  @property
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      return len(self.decoder      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      return dict(self.encoder    ,**self.added_tokens_encoder      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   str  =    tuple(token[:-1]      ) + (token[-1] + BPE_TOKEN_MERGES,)
      if token in self.cache:
          return self.cache[token]
      _lowerCAmelCase       :   str  =    get_pairs(_A      )

      if not pairs:
          return token

      while True:
          _lowerCAmelCase       :   List[str]  =    min(_A    ,key=lambda _A      : self.bpe_ranks.get(_A    ,float('inf'      )      )      )
          if bigram not in self.bpe_ranks:
              break
          _lowerCAmelCase,					_lowerCAmelCase       :   Optional[int]  =    bigram
          _lowerCAmelCase       :   Union[str, Any]  =    []
          _lowerCAmelCase       :   Dict  =    0
          while i < len(_A      ):
              try:
                  _lowerCAmelCase       :   Dict  =    word.index(_A    ,_A      )
              except ValueError:
                  new_word.extend(word[i:]      )
                  break
              else:
                  new_word.extend(word[i:j]      )
                  _lowerCAmelCase       :   Optional[Any]  =    j

              if word[i] == first and i < len(_A      ) - 1 and word[i + 1] == second:
                  new_word.append(first + second      )
                  i += 2
              else:
                  new_word.append(word[i]      )
                  i += 1
          _lowerCAmelCase       :   Optional[Any]  =    tuple(_A      )
          _lowerCAmelCase       :   List[str]  =    new_word
          if len(_A      ) == 1:
              break
          else:
              _lowerCAmelCase       :   List[str]  =    get_pairs(_A      )
      _lowerCAmelCase       :   Any  =    ' '.join(_A      )
      if word == "\n  " + BPE_TOKEN_MERGES:
          _lowerCAmelCase       :   str  =    '\n' + BPE_TOKEN_MERGES

      if word.endswith(_A      ):
          _lowerCAmelCase       :   Dict  =    word.replace(_A    ,''      )

      _lowerCAmelCase       :   str  =    word.replace(' '    ,_A      )
      _lowerCAmelCase       :   str  =    word
      return word

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      if self.bpe_ranks is None:
          raise ValueError(
              'This tokenizer was instantiated without a `merges.txt` file, so'
              ' that it can only be used for decoding, not for encoding.'
              'Make sure to provide `merges.txt` file at instantiation to enable '
              'encoding.'      )

      if self.do_lower_case:
          _lowerCAmelCase       :   Optional[Any]  =    text.lower()

      _lowerCAmelCase       :   Tuple  =    text.split()

      _lowerCAmelCase       :   Union[str, Any]  =    []
      for token in text:
          if token:
              split_tokens.extend(list(self.bpe(_A      ).split(' '      )      )      )

      return split_tokens

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      return self.encoder.get(_A    ,self.encoder.get(self.unk_token      )      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   int  =    self.decoder.get(_A    ,self.unk_token      )
      return result

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Optional[Any]  =    ' '.join(_A      )

      # make sure @@ tokens are concatenated
      _lowerCAmelCase       :   int  =    ''.join(string.split(_A      )      )

      return string







  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      if not os.path.isdir(_A      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      _lowerCAmelCase       :   List[Any]  =    os.path.join(
          _A    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
      _lowerCAmelCase       :   str  =    os.path.join(
          _A    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

      with open(_A    ,'w'    ,encoding='utf-8'      ) as f:
          f.write(json.dumps(self.encoder    ,indent=2    ,sort_keys=_A    ,ensure_ascii=_A      ) + '\n'      )

      _lowerCAmelCase       :   str  =    0
      if self.bpe_ranks is None:
          return (vocab_file,)

      with open(_A    ,'w'    ,encoding='utf-8'      ) as writer:
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,key=lambda _A      : kv[1]      ):
              if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."""
                      ' Please check that the tokenizer is not corrupted!'      )
                  _lowerCAmelCase       :   Dict  =    token_index
              writer.write(' '.join(_A      ) + '\n'      )
              index += 1

      return (vocab_file, merges_file)





