


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase    :						str     =							logging.get_logger(__name__)

__UpperCamelCase    :						Any     =							{'vocab_file': 'sentencepiece.bpe.model'}

__UpperCamelCase    :						Any     =							{
    'vocab_file': {
        'moussaKam/mbarthez': 'https://huggingface.co/moussaKam/mbarthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez': 'https://huggingface.co/moussaKam/barthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez-orangesum-title': (
            'https://huggingface.co/moussaKam/barthez-orangesum-title/resolve/main/sentencepiece.bpe.model'
        ),
    },
}

__UpperCamelCase    :						str     =							{
    'moussaKam/mbarthez': 1024,
    'moussaKam/barthez': 1024,
    'moussaKam/barthez-orangesum-title': 1024,
}

__UpperCamelCase    :						List[str]     =							'▁'



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   a_				:							Any    = VOCAB_FILES_NAMES
   a_				:							List[Any]    = PRETRAINED_VOCAB_FILES_MAP
   a_				:							Union[str, Any]    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a_				:							int    = ["input_ids", "attention_mask"]

   def __init__(  self  : Optional[int]       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : List[Any]="<s>"       ,  _lowerCamelCase  : str="</s>"       ,  _lowerCamelCase  : Dict="</s>"       ,  _lowerCamelCase  : Union[str, Any]="<s>"       ,  _lowerCamelCase  : int="<unk>"       ,  _lowerCamelCase  : Optional[Any]="<pad>"       ,  _lowerCamelCase  : str="<mask>"       ,  _lowerCamelCase  : Optional[Dict[str, Any]] = None       ,  **_lowerCamelCase  : List[Any]       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[int]       						=						AddedToken(_lowerCamelCase       ,  lstrip=_lowerCamelCase       ,  rstrip=_lowerCamelCase							) if isinstance(_lowerCamelCase       ,  _lowerCamelCase							) else mask_token

         __lowerCamelCase     :      int       						=						{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=_lowerCamelCase       ,  eos_token=_lowerCamelCase       ,  unk_token=_lowerCamelCase       ,  sep_token=_lowerCamelCase       ,  cls_token=_lowerCamelCase       ,  pad_token=_lowerCamelCase       ,  mask_token=_lowerCamelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  **_lowerCamelCase       ,  )

         __lowerCamelCase     :      Any       						=						vocab_file
         __lowerCamelCase     :      str       						=						spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(str(_lowerCamelCase							)							)

         __lowerCamelCase     :      List[Any]       						=						{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

         __lowerCamelCase     :      List[Any]       						=						len(self.sp_model							) - 1
         __lowerCamelCase     :      Dict       						=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}

   def   _snake_case    (  self  : str       ,  _lowerCamelCase  : List[int]       ,  _lowerCamelCase  : Optional[List[int]] = None							):






         '''simple docstring'''



         if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __lowerCamelCase     :      Optional[Any]       						=						[self.cls_token_id]
         __lowerCamelCase     :      Any       						=						[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep

   def   _snake_case    (  self  : Optional[int]       ,  _lowerCamelCase  : List[int]       ,  _lowerCamelCase  : Optional[List[int]] = None       ,  _lowerCamelCase  : bool = False							):






         '''simple docstring'''



         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_lowerCamelCase       ,  token_ids_a=_lowerCamelCase       ,  already_has_special_tokens=_lowerCamelCase							)

         if token_ids_a is None:
               return [1] + ([0] * len(_lowerCamelCase							)) + [1]
         return [1] + ([0] * len(_lowerCamelCase							)) + [1, 1] + ([0] * len(_lowerCamelCase							)) + [1]

   def   _snake_case    (  self  : Union[str, Any]       ,  _lowerCamelCase  : List[int]       ,  _lowerCamelCase  : Optional[List[int]] = None							):






         '''simple docstring'''



         __lowerCamelCase     :      int       						=						[self.sep_token_id]
         __lowerCamelCase     :      Optional[int]       						=						[self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]

   @property
   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         return len(self.sp_model							)

   def   _snake_case    (  self  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						{self.convert_ids_to_tokens(_lowerCamelCase							): i for i in range(self.vocab_size							)}
         vocab.update(self.added_tokens_encoder							)
         return vocab

   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : str							):






         '''simple docstring'''



         return self.sp_model.encode(_lowerCamelCase       ,  out_type=_lowerCamelCase							)

   def   _snake_case    (  self  : Any       ,  _lowerCamelCase  : List[Any]							):






         '''simple docstring'''



         if token in self.fairseq_tokens_to_ids:
               return self.fairseq_tokens_to_ids[token]
         __lowerCamelCase     :      str       						=						self.sp_model.PieceToId(_lowerCamelCase							)

         return spm_id if spm_id else self.unk_token_id

   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : List[str]							):






         '''simple docstring'''



         if index in self.fairseq_ids_to_tokens:
               return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(_lowerCamelCase							)

   def   _snake_case    (  self  : str       ,  _lowerCamelCase  : Optional[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						[]
         __lowerCamelCase     :      Any       						=						""""""
         __lowerCamelCase     :      Tuple       						=						False
         for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                     if not prev_is_special:
                           out_string += " "
                     out_string += self.sp_model.decode(_lowerCamelCase							) + token
                     __lowerCamelCase     :      Union[str, Any]       						=						True
                     __lowerCamelCase     :      Dict       						=						[]
               else:
                     current_sub_tokens.append(_lowerCamelCase							)
                     __lowerCamelCase     :      Optional[int]       						=						False
         out_string += self.sp_model.decode(_lowerCamelCase							)
         return out_string.strip()

   def __getstate__(  self  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      str       						=						self.__dict__.copy()
         __lowerCamelCase     :      Union[str, Any]       						=						None
         return state

   def __setstate__(  self  : Any       ,  _lowerCamelCase  : str							):






         '''simple docstring'''



         __lowerCamelCase     :      Union[str, Any]       						=						d

         # for backward compatibility
         if not hasattr(self       ,  """sp_model_kwargs"""							):
               __lowerCamelCase     :      Union[str, Any]       						=						{}

         __lowerCamelCase     :      int       						=						spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(self.vocab_file							)


   def   _snake_case    (  self  : Optional[int]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Optional[str] = None							):






         '''simple docstring'''



         if not os.path.isdir(_lowerCamelCase							):
               logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
               return
         __lowerCamelCase     :      Union[str, Any]       						=						os.path.join(
             _lowerCamelCase       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(_lowerCamelCase							) and os.path.isfile(self.vocab_file							):
               copyfile(self.vocab_file       ,  _lowerCamelCase							)
         elif not os.path.isfile(self.vocab_file							):
               with open(_lowerCamelCase       ,  """wb"""							) as fi:
                     __lowerCamelCase     :      Union[str, Any]       						=						self.sp_model.serialized_model_proto()
                     fi.write(_lowerCamelCase							)

         return (out_vocab_file,)







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase    :						Tuple     =							logging.get_logger(__name__)

__UpperCamelCase    :						Any     =							{
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   a_				:							int    = "nllb-moe"
   a_				:							Optional[int]    = ["past_key_values"]
   a_				:							Optional[Any]    = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

   def __init__(  self  : int       ,  _lowerCamelCase  : Union[str, Any]=1_2_8_1_1_2       ,  _lowerCamelCase  : List[str]=1_0_2_4       ,  _lowerCamelCase  : Optional[int]=1_2       ,  _lowerCamelCase  : int=4_0_9_6       ,  _lowerCamelCase  : Union[str, Any]=1_6       ,  _lowerCamelCase  : int=1_2       ,  _lowerCamelCase  : Optional[int]=4_0_9_6       ,  _lowerCamelCase  : Any=1_6       ,  _lowerCamelCase  : Any=0.05       ,  _lowerCamelCase  : List[Any]=0.05       ,  _lowerCamelCase  : Tuple=True       ,  _lowerCamelCase  : Any=True       ,  _lowerCamelCase  : Optional[int]="relu"       ,  _lowerCamelCase  : Dict=1_0_2_4       ,  _lowerCamelCase  : Optional[int]=0.1       ,  _lowerCamelCase  : Optional[int]=0.1       ,  _lowerCamelCase  : Optional[int]=0.0       ,  _lowerCamelCase  : List[Any]=0.02       ,  _lowerCamelCase  : List[str]=2       ,  _lowerCamelCase  : List[str]=True       ,  _lowerCamelCase  : Union[str, Any]=False       ,  _lowerCamelCase  : Optional[Any]="float32"       ,  _lowerCamelCase  : Union[str, Any]=False       ,  _lowerCamelCase  : Tuple=1_2_8       ,  _lowerCamelCase  : Dict=6_4       ,  _lowerCamelCase  : Any=4       ,  _lowerCamelCase  : List[Any]=4       ,  _lowerCamelCase  : str=0.001       ,  _lowerCamelCase  : Dict=0.001       ,  _lowerCamelCase  : Any="all"       ,  _lowerCamelCase  : Optional[Any]=False       ,  _lowerCamelCase  : List[Any]=False       ,  _lowerCamelCase  : Optional[Any]=1.0       ,  _lowerCamelCase  : List[str]=0.2       ,  _lowerCamelCase  : List[Any]=1       ,  _lowerCamelCase  : str=0       ,  _lowerCamelCase  : int=2       ,  _lowerCamelCase  : int=False       ,  **_lowerCamelCase  : List[str]       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						vocab_size
         __lowerCamelCase     :      str       						=						max_position_embeddings
         __lowerCamelCase     :      int       						=						d_model
         __lowerCamelCase     :      Optional[Any]       						=						encoder_ffn_dim
         __lowerCamelCase     :      str       						=						encoder_layers
         __lowerCamelCase     :      Tuple       						=						encoder_attention_heads
         __lowerCamelCase     :      Union[str, Any]       						=						decoder_ffn_dim
         __lowerCamelCase     :      Dict       						=						decoder_layers
         __lowerCamelCase     :      Any       						=						decoder_attention_heads
         __lowerCamelCase     :      Union[str, Any]       						=						dropout
         __lowerCamelCase     :      Dict       						=						attention_dropout
         __lowerCamelCase     :      int       						=						activation_dropout
         __lowerCamelCase     :      Union[str, Any]       						=						activation_function
         __lowerCamelCase     :      Optional[int]       						=						init_std
         __lowerCamelCase     :      Union[str, Any]       						=						encoder_layerdrop
         __lowerCamelCase     :      Dict       						=						decoder_layerdrop
         __lowerCamelCase     :      List[str]       						=						use_cache
         __lowerCamelCase     :      Any       						=						encoder_layers
         __lowerCamelCase     :      List[Any]       						=						scale_embedding  # scale factor will be sqrt(d_model) if True
         __lowerCamelCase     :      Optional[Any]       						=						router_z_loss_coef
         __lowerCamelCase     :      Union[str, Any]       						=						router_aux_loss_coef
         __lowerCamelCase     :      Dict       						=						decoder_sparse_step
         __lowerCamelCase     :      Optional[int]       						=						encoder_sparse_step
         __lowerCamelCase     :      Any       						=						num_experts
         __lowerCamelCase     :      int       						=						expert_capacity
         __lowerCamelCase     :      Dict       						=						router_bias
         if router_dtype not in ["float32", "float16", "bfloat16"]:
               raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""							)
         __lowerCamelCase     :      Tuple       						=						router_dtype

         __lowerCamelCase     :      Dict       						=						router_ignore_padding_tokens
         __lowerCamelCase     :      str       						=						batch_prioritized_routing
         __lowerCamelCase     :      List[str]       						=						second_expert_policy
         __lowerCamelCase     :      Tuple       						=						normalize_router_prob_before_dropping
         __lowerCamelCase     :      Dict       						=						moe_eval_capacity_token_fraction
         __lowerCamelCase     :      Optional[int]       						=						moe_token_dropout
         __lowerCamelCase     :      Dict       						=						output_router_logits
         super().__init__(
             pad_token_id=_lowerCamelCase       ,  bos_token_id=_lowerCamelCase       ,  eos_token_id=_lowerCamelCase       ,  is_encoder_decoder=_lowerCamelCase       ,  decoder_start_token_id=_lowerCamelCase       ,  **_lowerCamelCase       ,  )



