


'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase    :						str     =							logging.get_logger(__name__)

__UpperCamelCase    :						Union[str, Any]     =							'▁'

__UpperCamelCase    :						Union[str, Any]     =							{'vocab_file': 'sentencepiece.bpe.model'}

__UpperCamelCase    :						List[Any]     =							{
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

__UpperCamelCase    :						List[Any]     =							{
    'facebook/xglm-564M': 2048,
}



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   a_				:							Dict    = VOCAB_FILES_NAMES
   a_				:							List[Any]    = PRETRAINED_VOCAB_FILES_MAP
   a_				:							str    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a_				:							List[Any]    = ["input_ids", "attention_mask"]

   def __init__(  self  : Any       ,  _lowerCamelCase  : Optional[Any]       ,  _lowerCamelCase  : List[Any]="<s>"       ,  _lowerCamelCase  : int="</s>"       ,  _lowerCamelCase  : List[str]="</s>"       ,  _lowerCamelCase  : Optional[int]="<s>"       ,  _lowerCamelCase  : Any="<unk>"       ,  _lowerCamelCase  : Tuple="<pad>"       ,  _lowerCamelCase  : Optional[Dict[str, Any]] = None       ,  **_lowerCamelCase  : Optional[Any]       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      str       						=						{} if sp_model_kwargs is None else sp_model_kwargs

         # Compatibility with the original tokenizer
         __lowerCamelCase     :      str       						=						7
         __lowerCamelCase     :      Any       						=						[F"""<madeupword{i}>""" for i in range(self.num_madeup_words							)]

         __lowerCamelCase     :      List[Any]       						=						kwargs.get("""additional_special_tokens"""       ,  []							)
         kwargs["additional_special_tokens"] += [
             word for word in madeup_words if word not in kwargs["additional_special_tokens"]
         ]

         super().__init__(
             bos_token=_lowerCamelCase       ,  eos_token=_lowerCamelCase       ,  unk_token=_lowerCamelCase       ,  sep_token=_lowerCamelCase       ,  cls_token=_lowerCamelCase       ,  pad_token=_lowerCamelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  **_lowerCamelCase       ,  )

         __lowerCamelCase     :      List[Any]       						=						spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(str(_lowerCamelCase							)							)
         __lowerCamelCase     :      Optional[Any]       						=						vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         __lowerCamelCase     :      Any       						=						1

         # Mimic fairseq token-to-id alignment for the first 4 token
         __lowerCamelCase     :      List[str]       						=						{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

         __lowerCamelCase     :      Optional[int]       						=						len(self.sp_model							)
         __lowerCamelCase     :      Optional[int]       						=						{F"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words							)}
         self.fairseq_tokens_to_ids.update(_lowerCamelCase							)

         __lowerCamelCase     :      Optional[Any]       						=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}

   def __getstate__(  self  : int							):






         '''simple docstring'''



         __lowerCamelCase     :      Tuple       						=						self.__dict__.copy()
         __lowerCamelCase     :      int       						=						None
         __lowerCamelCase     :      Dict       						=						self.sp_model.serialized_model_proto()
         return state

   def __setstate__(  self  : Dict       ,  _lowerCamelCase  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						d

         # for backward compatibility
         if not hasattr(self       ,  """sp_model_kwargs"""							):
               __lowerCamelCase     :      Union[str, Any]       						=						{}

         __lowerCamelCase     :      List[str]       						=						spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)

   def   _snake_case    (  self  : str       ,  _lowerCamelCase  : List[int]       ,  _lowerCamelCase  : Optional[List[int]] = None							):






         '''simple docstring'''



         if token_ids_a is None:
               return [self.sep_token_id] + token_ids_a
         __lowerCamelCase     :      Any       						=						[self.sep_token_id]
         return sep + token_ids_a + sep + sep + token_ids_a

   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : List[int]       ,  _lowerCamelCase  : Optional[List[int]] = None       ,  _lowerCamelCase  : bool = False							):






         '''simple docstring'''



         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_lowerCamelCase       ,  token_ids_a=_lowerCamelCase       ,  already_has_special_tokens=_lowerCamelCase							)

         if token_ids_a is None:
               return [1] + ([0] * len(_lowerCamelCase							))
         return [1] + ([0] * len(_lowerCamelCase							)) + [1, 1] + ([0] * len(_lowerCamelCase							))

   def   _snake_case    (  self  : List[Any]       ,  _lowerCamelCase  : List[int]       ,  _lowerCamelCase  : Optional[List[int]] = None							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						[self.sep_token_id]

         if token_ids_a is None:
               return len(sep + token_ids_a							) * [0]
         return len(sep + token_ids_a + sep + sep + token_ids_a							) * [0]

   @property
   def   _snake_case    (  self  : Union[str, Any]							):






         '''simple docstring'''



         return len(self.sp_model							) + self.fairseq_offset + self.num_madeup_words

   def   _snake_case    (  self  : Optional[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Any       						=						{self.convert_ids_to_tokens(_lowerCamelCase							): i for i in range(self.vocab_size							)}
         vocab.update(self.added_tokens_encoder							)
         return vocab

   def   _snake_case    (  self  : str       ,  _lowerCamelCase  : str							):






         '''simple docstring'''



         return self.sp_model.encode(_lowerCamelCase       ,  out_type=_lowerCamelCase							)

   def   _snake_case    (  self  : Any       ,  _lowerCamelCase  : Tuple							):






         '''simple docstring'''



         if token in self.fairseq_tokens_to_ids:
               return self.fairseq_tokens_to_ids[token]
         __lowerCamelCase     :      Tuple       						=						self.sp_model.PieceToId(_lowerCamelCase							)

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

   def   _snake_case    (  self  : List[str]       ,  _lowerCamelCase  : Dict							):






         '''simple docstring'''



         if index in self.fairseq_ids_to_tokens:
               return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset							)

   def   _snake_case    (  self  : Optional[int]       ,  _lowerCamelCase  : Union[str, Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						"""""".join(_lowerCamelCase							).replace(_lowerCamelCase       ,  """ """							).strip()
         return out_string


   def   _snake_case    (  self  : Dict       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Optional[str] = None							):






         '''simple docstring'''



         if not os.path.isdir(_lowerCamelCase							):
               logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
               return
         __lowerCamelCase     :      str       						=						os.path.join(
             _lowerCamelCase       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(_lowerCamelCase							) and os.path.isfile(self.vocab_file							):
               copyfile(self.vocab_file       ,  _lowerCamelCase							)
         elif not os.path.isfile(self.vocab_file							):
               with open(_lowerCamelCase       ,  """wb"""							) as fi:
                     __lowerCamelCase     :      Optional[int]       						=						self.sp_model.serialized_model_proto()
                     fi.write(_lowerCamelCase							)

         return (out_vocab_file,)







def   _UpperCAmelCase       (     UpperCAmelCase      :			str      ):
      """simple docstring"""




      __lowerCamelCase     :      List[Any]       						=						hex_num.strip()
      if not hex_num:
            raise ValueError("""No value was passed to the function"""      )

      __lowerCamelCase     :      Dict       						=						hex_num[0] == """-"""
      if is_negative:
            __lowerCamelCase     :      Optional[Any]       						=						hex_num[1:]

      try:
            __lowerCamelCase     :      Any       						=						int(UpperCAmelCase						,	16      )
      except ValueError:
            raise ValueError("""Invalid value was passed to the function"""      )

      __lowerCamelCase     :      List[str]       						=						""""""
      while int_num > 0:
            __lowerCamelCase     :      Dict       						=						str(int_num % 2      ) + bin_str
            int_num >>= 1

      return int(("""-""" + bin_str) if is_negative else bin_str      )


if __name__ == "__main__":
    import doctest

    doctest.testmod()



