'''simple docstring'''






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_	   =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	   =			'▁'

SCREAMING_SNAKE_CASE_	   =			{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

SCREAMING_SNAKE_CASE_	   =			{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

SCREAMING_SNAKE_CASE_	   =			{
    'facebook/m2m100_418M': 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE_	   =			{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}




class    a (       __lowerCAmelCase				):



  """simple docstring"""
  __lowerCAmelCase				      =					VOCAB_FILES_NAMES
  __lowerCAmelCase				      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase				      =					PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase				      =					["""input_ids""", """attention_mask"""]

  __lowerCAmelCase				      =					[]
  __lowerCAmelCase				      =					[]







  def __init__(							self   ,    snake_case_   ,    snake_case_   ,    snake_case_=None   ,    snake_case_=None   ,    snake_case_="<s>"   ,    snake_case_="</s>"   ,    snake_case_="</s>"   ,    snake_case_="<pad>"   ,    snake_case_="<unk>"   ,    snake_case_="m2m100"   ,    snake_case_ = None   ,    snake_case_=8   ,    **snake_case_   ,    ):

    '''simple docstring'''







    __UpperCAmelCase:					Tuple		= {} if sp_model_kwargs is None else sp_model_kwargs

    __UpperCAmelCase:					Optional[int]		= language_codes
    __UpperCAmelCase:					Tuple		= FAIRSEQ_LANGUAGE_CODES[language_codes]
    __UpperCAmelCase:					List[Any]		= {lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

    __UpperCAmelCase:					List[Any]		= kwargs.get("""additional_special_tokens"""   ,    []					)
    kwargs["additional_special_tokens"] += [
        self.get_lang_token(snake_case_					)
        for lang_code in fairseq_language_code
        if self.get_lang_token(snake_case_					) not in kwargs["additional_special_tokens"]
    ]

    super().__init__(
        src_lang=snake_case_   ,    tgt_lang=snake_case_   ,    bos_token=snake_case_   ,    eos_token=snake_case_   ,    sep_token=snake_case_   ,    unk_token=snake_case_   ,    pad_token=snake_case_   ,    language_codes=snake_case_   ,    sp_model_kwargs=self.sp_model_kwargs   ,    num_madeup_words=snake_case_   ,    **snake_case_   ,    )

    __UpperCAmelCase:					Dict		= vocab_file
    __UpperCAmelCase:					Tuple		= load_json(snake_case_					)
    __UpperCAmelCase:					List[Any]		= {v: k for k, v in self.encoder.items()}
    __UpperCAmelCase:					Dict		= spm_file
    __UpperCAmelCase:					Optional[Any]		= load_spm(snake_case_   ,    self.sp_model_kwargs					)

    __UpperCAmelCase:					Optional[int]		= len(self.encoder					)

    __UpperCAmelCase:					List[str]		= {
        self.get_lang_token(snake_case_					): self.encoder_size + i for i, lang_code in enumerate(snake_case_					)
    }
    __UpperCAmelCase:					Tuple		= {lang_code: self.encoder_size + i for i, lang_code in enumerate(snake_case_					)}
    __UpperCAmelCase:					Union[str, Any]		= {v: k for k, v in self.lang_token_to_id.items()}

    __UpperCAmelCase:					Optional[Any]		= src_lang if src_lang is not None else """en"""
    __UpperCAmelCase:					List[Any]		= tgt_lang
    __UpperCAmelCase:					Optional[Any]		= self.get_lang_id(self._src_lang					)
    self.set_src_lang_special_tokens(self._src_lang					)

    __UpperCAmelCase:					Optional[Any]		= num_madeup_words







  @property
  def    lowercase_  (							self					):

    '''simple docstring'''







    return len(self.encoder					) + len(self.lang_token_to_id					)







  @property
  def    lowercase_  (							self					):

    '''simple docstring'''







    return self._src_lang







  @src_lang.setter
  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					str		= new_src_lang
    self.set_src_lang_special_tokens(self._src_lang					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    return self.sp_model.encode(snake_case_   ,    out_type=snake_case_					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    if token in self.lang_token_to_id:
      return self.lang_token_to_id[token]
    return self.encoder.get(snake_case_   ,    self.encoder[self.unk_token]					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    if index in self.id_to_lang_token:
      return self.id_to_lang_token[index]
    return self.decoder.get(snake_case_   ,    self.unk_token					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					List[Any]		= []
    __UpperCAmelCase:					Dict		= """"""
    for token in tokens:
      # make sure that special tokens are not decoded using sentencepiece model
      if token in self.all_special_tokens:
        out_string += self.sp_model.decode(snake_case_					) + token
        __UpperCAmelCase:					List[str]		= []
      else:
        current_sub_tokens.append(snake_case_					)
    out_string += self.sp_model.decode(snake_case_					)
    return out_string.strip()







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None   ,    snake_case_ = False					):

    '''simple docstring'''







    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=snake_case_   ,    token_ids_a=snake_case_   ,    already_has_special_tokens=snake_case_					)

    __UpperCAmelCase:					Union[str, Any]		= [1] * len(self.prefix_tokens					)
    __UpperCAmelCase:					Tuple		= [1] * len(self.suffix_tokens					)
    if token_ids_a is None:
      return prefix_ones + ([0] * len(snake_case_					)) + suffix_ones
    return prefix_ones + ([0] * len(snake_case_					)) + ([0] * len(snake_case_					)) + suffix_ones







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    if token_ids_a is None:
      return self.prefix_tokens + token_ids_a + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







  def    lowercase_  (							self					):

    '''simple docstring'''







    __UpperCAmelCase:					Optional[int]		= {self.convert_ids_to_tokens(snake_case_					): i for i in range(self.vocab_size					)}
    vocab.update(self.added_tokens_encoder					)
    return vocab







  def __getstate__(							self					):

    '''simple docstring'''







    __UpperCAmelCase:					Union[str, Any]		= self.__dict__.copy()
    __UpperCAmelCase:					str		= None
    return state







  def __setstate__(							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Any		= d

    # for backward compatibility
    if not hasattr(self   ,    """sp_model_kwargs"""					):
      __UpperCAmelCase:					Union[str, Any]		= {}

    __UpperCAmelCase:					Tuple		= load_spm(self.spm_file   ,    self.sp_model_kwargs					)







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    __UpperCAmelCase:					str		= Path(snake_case_					)
    if not save_dir.is_dir():
      raise OSError(F'''{save_directory} should be a directory'''					)
    __UpperCAmelCase:					int		= save_dir / (
        (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
    )
    __UpperCAmelCase:					List[Any]		= save_dir / (
        (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
    )

    save_json(self.encoder   ,    snake_case_					)

    if os.path.abspath(self.spm_file					) != os.path.abspath(snake_case_					) and os.path.isfile(self.spm_file					):
      copyfile(self.spm_file   ,    snake_case_					)
    elif not os.path.isfile(self.spm_file					):
      with open(snake_case_   ,    """wb"""					) as fi:
        __UpperCAmelCase:					Dict		= self.sp_model.serialized_model_proto()
        fi.write(snake_case_					)

    return (str(snake_case_					), str(snake_case_					))







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = "en"   ,    snake_case_ = None   ,    snake_case_ = "ro"   ,    **snake_case_   ,    ):

    '''simple docstring'''







    __UpperCAmelCase:					Any		= src_lang
    __UpperCAmelCase:					List[Any]		= tgt_lang
    self.set_src_lang_special_tokens(self.src_lang					)
    return super().prepare_seqaseq_batch(snake_case_   ,    snake_case_   ,    **snake_case_					)







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_   ,    snake_case_   ,    **snake_case_					):

    '''simple docstring'''







    if src_lang is None or tgt_lang is None:
      raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""					)
    __UpperCAmelCase:					Any		= src_lang
    __UpperCAmelCase:					Tuple		= self(snake_case_   ,    add_special_tokens=snake_case_   ,    **snake_case_					)
    __UpperCAmelCase:					Dict		= self.get_lang_id(snake_case_					)
    __UpperCAmelCase:					Dict		= tgt_lang_id
    return inputs







  def    lowercase_  (							self					):

    '''simple docstring'''







    self.set_src_lang_special_tokens(self.src_lang					)







  def    lowercase_  (							self					):

    '''simple docstring'''







    self.set_tgt_lang_special_tokens(self.tgt_lang					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Union[str, Any]		= self.get_lang_token(snake_case_					)
    __UpperCAmelCase:					int		= self.lang_token_to_id[lang_token]
    __UpperCAmelCase:					List[Any]		= [self.cur_lang_id]
    __UpperCAmelCase:					List[str]		= [self.eos_token_id]







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Optional[int]		= self.get_lang_token(snake_case_					)
    __UpperCAmelCase:					Optional[Any]		= self.lang_token_to_id[lang_token]
    __UpperCAmelCase:					List[str]		= [self.cur_lang_id]
    __UpperCAmelCase:					Dict		= [self.eos_token_id]







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    return self.lang_code_to_token[lang]







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					List[Any]		= self.get_lang_token(snake_case_					)
    return self.lang_token_to_id[lang_token]






def  UpperCamelCase__					(					_lowercase    :				str   ,    _lowercase    :				Dict[str, Any]     )							->	sentencepiece.SentencePieceProcessor:
  __UpperCAmelCase:					List[str]		= sentencepiece.SentencePieceProcessor(**_lowercase     )
  spm.Load(str(_lowercase     )     )
  return spm






def  UpperCamelCase__					(					_lowercase    :				str     )							->	Union[Dict, List]:
  with open(_lowercase   ,    """r"""     ) as f:
    return json.load(_lowercase     )







def  UpperCamelCase__					(					_lowercase    :				Union[str, Any]   ,    _lowercase    :				str     )							->	None:
  with open(_lowercase   ,    """w"""     ) as f:
    json.dump(_lowercase   ,    _lowercase   ,    indent=2     )
'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_	   =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	   =			{
    'facebook/timesformer': 'https://huggingface.co/facebook/timesformer/resolve/main/config.json',
}




class    a (       __lowerCAmelCase				):



  """simple docstring"""
  __lowerCAmelCase				      =					"""timesformer"""







  def __init__(							self   ,    snake_case_=224   ,    snake_case_=16   ,    snake_case_=3   ,    snake_case_=8   ,    snake_case_=768   ,    snake_case_=12   ,    snake_case_=12   ,    snake_case_=3072   ,    snake_case_="gelu"   ,    snake_case_=0.0   ,    snake_case_=0.0   ,    snake_case_=0.0_2   ,    snake_case_=1e-6   ,    snake_case_=True   ,    snake_case_="divided_space_time"   ,    snake_case_=0   ,    **snake_case_   ,    ):

    '''simple docstring'''







    super().__init__(**snake_case_					)

    __UpperCAmelCase:					Tuple		= image_size
    __UpperCAmelCase:					List[Any]		= patch_size
    __UpperCAmelCase:					Optional[Any]		= num_channels
    __UpperCAmelCase:					int		= num_frames

    __UpperCAmelCase:					List[str]		= hidden_size
    __UpperCAmelCase:					List[str]		= num_hidden_layers
    __UpperCAmelCase:					Dict		= num_attention_heads
    __UpperCAmelCase:					List[str]		= intermediate_size
    __UpperCAmelCase:					Union[str, Any]		= hidden_act
    __UpperCAmelCase:					Dict		= hidden_dropout_prob
    __UpperCAmelCase:					Optional[Any]		= attention_probs_dropout_prob
    __UpperCAmelCase:					Union[str, Any]		= initializer_range
    __UpperCAmelCase:					List[Any]		= layer_norm_eps
    __UpperCAmelCase:					int		= qkv_bias

    __UpperCAmelCase:					str		= attention_type
    __UpperCAmelCase:					Tuple		= drop_path_rate