



"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase     =			logging.get_logger(__name__)

_UpperCAmelCase     =			{"""vocab_file""": """spm_char.model"""}

_UpperCAmelCase     =			{
    """vocab_file""": {
        """microsoft/speecht5_asr""": """https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model""",
        """microsoft/speecht5_tts""": """https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model""",
        """microsoft/speecht5_vc""": """https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model""",
    }
}

_UpperCAmelCase     =			{
    """microsoft/speecht5_asr""": 1_0_2_4,
    """microsoft/speecht5_tts""": 1_0_2_4,
    """microsoft/speecht5_vc""": 1_0_2_4,
}
class 					a	(  UpperCAmelCase__       ):
  UpperCamelCase	:				Union[str, Any]     						=					VOCAB_FILES_NAMES
  UpperCamelCase	:				str     						=					PRETRAINED_VOCAB_FILES_MAP
  UpperCamelCase	:				Optional[Any]     						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCamelCase	:				Tuple     						=					['input_ids', 'attention_mask']



  def __init__(						self :					str       ,		lowerCAmelCase :					Any       ,		lowerCAmelCase :					Union[str, Any]="<s>"       ,		lowerCAmelCase :					Union[str, Any]="</s>"       ,		lowerCAmelCase :					Any="<unk>"       ,		lowerCAmelCase :					List[str]="<pad>"       ,		lowerCAmelCase :					Optional[Dict[str, Any]] = None       ,		**lowerCAmelCase :					Dict       ,		)     ->					None:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_: Optional[Any]     			={} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         bos_token=lowerCAmelCase       ,		eos_token=lowerCAmelCase       ,		unk_token=lowerCAmelCase       ,		pad_token=lowerCAmelCase       ,		sp_model_kwargs=self.sp_model_kwargs       ,		**lowerCAmelCase       ,		)

     SCREAMING_SNAKE_CASE_: str     			=vocab_file

     SCREAMING_SNAKE_CASE_: str     			=spm.SentencePieceProcessor(**self.sp_model_kwargs		)
     self.sp_model.Load(lowerCAmelCase		)



  @property
  def 	lowerCamelCase__       (						self :					str		)     ->					List[str]:


     '''simple docstring'''




     return self.sp_model.get_piece_size()



  def 	lowerCamelCase__       (						self :					Tuple		)     ->					int:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_: List[Any]     			={self.convert_ids_to_tokens(lowerCAmelCase		): i for i in range(self.vocab_size		)}
     vocab.update(self.added_tokens_encoder		)
     return vocab



  def __getstate__(						self :					List[str]		)     ->					str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_: int     			=self.__dict__.copy()
     SCREAMING_SNAKE_CASE_: Optional[Any]     			=None
     return state



  def __setstate__(						self :					Union[str, Any]       ,		lowerCAmelCase :					Union[str, Any]		)     ->					Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_: Union[str, Any]     			=d

     # for backward compatibility
     if not hasattr(self       ,		"""sp_model_kwargs"""		):
        SCREAMING_SNAKE_CASE_: Union[str, Any]     			={}

     SCREAMING_SNAKE_CASE_: Optional[int]     			=spm.SentencePieceProcessor(**self.sp_model_kwargs		)
     self.sp_model.Load(self.vocab_file		)



  def 	lowerCamelCase__       (						self :					List[Any]       ,		lowerCAmelCase :					str		)     ->					List[str]:


     '''simple docstring'''




     return self.sp_model.encode(lowerCAmelCase       ,		out_type=lowerCAmelCase		)



  def 	lowerCamelCase__       (						self :					List[str]       ,		lowerCAmelCase :					str		)     ->					Tuple:


     '''simple docstring'''




     return self.sp_model.piece_to_id(lowerCAmelCase		)



  def 	lowerCamelCase__       (						self :					List[Any]       ,		lowerCAmelCase :					Optional[int]		)     ->					Optional[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_: str     			=self.sp_model.IdToPiece(lowerCAmelCase		)
     return token



  def 	lowerCamelCase__       (						self :					Optional[Any]       ,		lowerCAmelCase :					Optional[Any]		)     ->					str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_: List[Any]     			=[]
     SCREAMING_SNAKE_CASE_: Any     			=""""""
     for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
           out_string += self.sp_model.decode(lowerCAmelCase		) + token
           SCREAMING_SNAKE_CASE_: Dict     			=[]
        else:
           current_sub_tokens.append(lowerCAmelCase		)
     out_string += self.sp_model.decode(lowerCAmelCase		)
     return out_string.strip()



  def 	lowerCamelCase__       (						self :					Union[str, Any]       ,		lowerCAmelCase :					List[str]       ,		lowerCAmelCase :					str=None		)     ->					List[int]:


     '''simple docstring'''




     if token_ids_a is None:
        return token_ids_a + [self.eos_token_id]
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return token_ids_a + token_ids_a + [self.eos_token_id]



  def 	lowerCamelCase__       (						self :					Optional[Any]       ,		lowerCAmelCase :					List[int]       ,		lowerCAmelCase :					Optional[List[int]] = None       ,		lowerCAmelCase :					bool = False		)     ->					List[int]:


     '''simple docstring'''




     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowerCAmelCase       ,		token_ids_a=lowerCAmelCase       ,		already_has_special_tokens=lowerCAmelCase		)

     SCREAMING_SNAKE_CASE_: List[str]     			=[1]
     if token_ids_a is None:
        return ([0] * len(lowerCAmelCase		)) + suffix_ones
     return ([0] * len(lowerCAmelCase		)) + ([0] * len(lowerCAmelCase		)) + suffix_ones





  def 	lowerCamelCase__       (						self :					Optional[int]       ,		lowerCAmelCase :					str       ,		lowerCAmelCase :					Optional[str] = None		)     ->					Tuple[str]:


     '''simple docstring'''




     if not os.path.isdir(lowerCAmelCase		):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
        return
     SCREAMING_SNAKE_CASE_: int     			=os.path.join(
         lowerCAmelCase       ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

     if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCAmelCase		) and os.path.isfile(self.vocab_file		):
        copyfile(self.vocab_file       ,		lowerCAmelCase		)
     elif not os.path.isfile(self.vocab_file		):
        with open(lowerCAmelCase       ,		"""wb"""		) as fi:
           SCREAMING_SNAKE_CASE_: Optional[Any]     			=self.sp_model.serialized_model_proto()
           fi.write(lowerCAmelCase		)

     return (out_vocab_file,)








"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig


_UpperCAmelCase     =			{
    """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/config.json""",
    """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/config.json""",
    """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/config.json""",
    """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/config.json""",
    """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/config.json""",
    """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/config.json""",
    """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/config.json""",
    """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/config.json""",
}
class 					a	(  UpperCAmelCase__       ):
  UpperCamelCase	:				Any     						=					'albert'



  def __init__(						self :					Dict       ,		lowerCAmelCase :					List[str]=3_0000       ,		lowerCAmelCase :					List[Any]=128       ,		lowerCAmelCase :					List[str]=4096       ,		lowerCAmelCase :					str=12       ,		lowerCAmelCase :					str=1       ,		lowerCAmelCase :					Tuple=64       ,		lowerCAmelCase :					Dict=1_6384       ,		lowerCAmelCase :					int=1       ,		lowerCAmelCase :					str="gelu_new"       ,		lowerCAmelCase :					Dict=0       ,		lowerCAmelCase :					Optional[Any]=0       ,		lowerCAmelCase :					str=512       ,		lowerCAmelCase :					Optional[int]=2       ,		lowerCAmelCase :					List[Any]=0.0_2       ,		lowerCAmelCase :					Union[str, Any]=1E-12       ,		lowerCAmelCase :					Tuple=0.1       ,		lowerCAmelCase :					List[Any]="absolute"       ,		lowerCAmelCase :					List[Any]=0       ,		lowerCAmelCase :					int=2       ,		lowerCAmelCase :					Optional[int]=3       ,		**lowerCAmelCase :					int       ,		)     ->					Tuple:


     '''simple docstring'''




     super().__init__(pad_token_id=lowerCAmelCase       ,		bos_token_id=lowerCAmelCase       ,		eos_token_id=lowerCAmelCase       ,		**lowerCAmelCase		)

     SCREAMING_SNAKE_CASE_: List[str]     			=vocab_size
     SCREAMING_SNAKE_CASE_: Optional[int]     			=embedding_size
     SCREAMING_SNAKE_CASE_: Optional[int]     			=hidden_size
     SCREAMING_SNAKE_CASE_: Tuple     			=num_hidden_layers
     SCREAMING_SNAKE_CASE_: Any     			=num_hidden_groups
     SCREAMING_SNAKE_CASE_: List[Any]     			=num_attention_heads
     SCREAMING_SNAKE_CASE_: List[Any]     			=inner_group_num
     SCREAMING_SNAKE_CASE_: Optional[int]     			=hidden_act
     SCREAMING_SNAKE_CASE_: int     			=intermediate_size
     SCREAMING_SNAKE_CASE_: Any     			=hidden_dropout_prob
     SCREAMING_SNAKE_CASE_: Union[str, Any]     			=attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE_: int     			=max_position_embeddings
     SCREAMING_SNAKE_CASE_: Any     			=type_vocab_size
     SCREAMING_SNAKE_CASE_: int     			=initializer_range
     SCREAMING_SNAKE_CASE_: List[Any]     			=layer_norm_eps
     SCREAMING_SNAKE_CASE_: Dict     			=classifier_dropout_prob
     SCREAMING_SNAKE_CASE_: int     			=position_embedding_type







class 					a	(  UpperCAmelCase__       ):



  @property
  def 	lowerCamelCase__       (						self :					Optional[int]		)     ->					Mapping[str, Mapping[int, str]]:


     '''simple docstring'''




     if self.task == "multiple-choice":
        SCREAMING_SNAKE_CASE_: str     			={0: """batch""", 1: """choice""", 2: """sequence"""}
     else:
        SCREAMING_SNAKE_CASE_: Dict     			={0: """batch""", 1: """sequence"""}
     return OrderedDict(
         [
             ("""input_ids""", dynamic_axis),
             ("""attention_mask""", dynamic_axis),
             ("""token_type_ids""", dynamic_axis),
         ]		)



