


'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a_						:       Union[str, Any]      			=				logging.get_logger(__name__)

a_						:       Tuple      			=				{"""vocab_file""": """spiece.model"""}

a_						:       Optional[int]      			=				{
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    }
}





class   snake_case					(					lowercase    ):





  """simple docstring"""




  def __init__(					self    ,						UpperCamelCase    ,						UpperCamelCase=False    ,						UpperCamelCase=True    ,						UpperCamelCase=False    ,						UpperCamelCase="<s>"    ,						UpperCamelCase="</s>"    ,						UpperCamelCase="<unk>"    ,						UpperCamelCase="<sep>"    ,						UpperCamelCase="<pad>"    ,						UpperCamelCase="<cls>"    ,						UpperCamelCase="<mask>"    ,						UpperCamelCase=["<eop>", "<eod>"]    ,						UpperCamelCase = None    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   lowerCamelCase_  				=						AddedToken(UpperCamelCase    ,						lstrip=UpperCamelCase    ,						rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase    ,						UpperCamelCase  ) else mask_token

   lowerCamelCase_  				=						{} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=UpperCamelCase    ,						remove_space=UpperCamelCase    ,						keep_accents=UpperCamelCase    ,						bos_token=UpperCamelCase    ,						eos_token=UpperCamelCase    ,						unk_token=UpperCamelCase    ,						sep_token=UpperCamelCase    ,						pad_token=UpperCamelCase    ,						cls_token=UpperCamelCase    ,						mask_token=UpperCamelCase    ,						additional_special_tokens=UpperCamelCase    ,						sp_model_kwargs=self.sp_model_kwargs    ,						**UpperCamelCase    ,						)

   lowerCamelCase_  				=						3

   lowerCamelCase_  				=						do_lower_case
   lowerCamelCase_  				=						remove_space
   lowerCamelCase_  				=						keep_accents
   lowerCamelCase_  				=						vocab_file

   lowerCamelCase_  				=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
   self.sp_model.Load(UpperCamelCase  )

   try:
    import jieba
   except ModuleNotFoundError as error:
    raise error.__class__(
        "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
        "See https://pypi.org/project/jieba/ for installation."  )
   lowerCamelCase_  				=						jieba
   lowerCamelCase_  				=						str.maketrans(" \n"    ,						"\u2582\u2583"  )






  @property
  # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
  def 						snake_case	(					self  ):







   """simple docstring"""
   return len(self.sp_model  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						{self.convert_ids_to_tokens(UpperCamelCase  ): i for i in range(self.vocab_size  )}
   vocab.update(self.added_tokens_encoder  )
   return vocab






  def __getstate__(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.__dict__.copy()
   lowerCamelCase_  				=						None
   return state






  def __setstate__(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						d

   # for backward compatibility
   if not hasattr(self    ,						"sp_model_kwargs"  ):
    lowerCamelCase_  				=						{}

   lowerCamelCase_  				=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
   self.sp_model.Load(self.vocab_file  )






  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   if self.remove_space:
    lowerCamelCase_  				=						" ".join(inputs.strip().split()  )
   else:
    lowerCamelCase_  				=						inputs
   lowerCamelCase_  				=						outputs.replace("``"    ,						"\""  ).replace("''"    ,						"\""  )

   if not self.keep_accents:
    lowerCamelCase_  				=						unicodedata.normalize("NFKD"    ,						UpperCamelCase  )
    lowerCamelCase_  				=						"".join([c for c in outputs if not unicodedata.combining(UpperCamelCase  )]  )
   if self.do_lower_case:
    lowerCamelCase_  				=						outputs.lower()

   return outputs






  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.preprocess_text(UpperCamelCase  )
   lowerCamelCase_  				=						self.sp_model.encode(UpperCamelCase    ,						out_type=UpperCamelCase  )
   lowerCamelCase_  				=						[]
   for piece in pieces:
    if len(UpperCamelCase  ) > 1 and piece[-1] == str(","  ) and piece[-2].isdigit():
     lowerCamelCase_  				=						self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase    ,						""  )  )
     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
      if len(cur_pieces[0]  ) == 1:
       lowerCamelCase_  				=						cur_pieces[1:]
      else:
       lowerCamelCase_  				=						cur_pieces[0][1:]
     cur_pieces.append(piece[-1]  )
     new_pieces.extend(UpperCamelCase  )
    else:
     new_pieces.append(UpperCamelCase  )

   return new_pieces






  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   return self.sp_model.PieceToId(UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   return self.sp_model.IdToPiece(UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						"".join(UpperCamelCase  ).replace(UpperCamelCase    ,						" "  ).strip()
   return out_string






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   lowerCamelCase_  				=						[self.sep_token_id]
   lowerCamelCase_  				=						[self.cls_token_id]
   if token_ids_a is None:
    return token_ids_a + sep + cls
   return token_ids_a + sep + token_ids_a + sep + cls






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None    ,						UpperCamelCase = False  ):







   """simple docstring"""

   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=UpperCamelCase    ,						token_ids_a=UpperCamelCase    ,						already_has_special_tokens=UpperCamelCase  )

   if token_ids_a is not None:
    return ([0] * len(UpperCamelCase  )) + [1] + ([0] * len(UpperCamelCase  )) + [1, 1]
   return ([0] * len(UpperCamelCase  )) + [1, 1]






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   lowerCamelCase_  				=						[self.sep_token_id]
   lowerCamelCase_  				=						[2]

   if token_ids_a is None:
    return len(token_ids_a + sep  ) * [0] + cls_segment_id
   return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   if not os.path.isdir(UpperCamelCase  ):
    logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
    return
   lowerCamelCase_  				=						os.path.join(
       UpperCamelCase    ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

   if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
    copyfile(self.vocab_file    ,						UpperCamelCase  )
   elif not os.path.isfile(self.vocab_file  ):
    with open(UpperCamelCase    ,						"wb"  ) as fi:
     lowerCamelCase_  				=						self.sp_model.serialized_model_proto()
     fi.write(UpperCamelCase  )

   return (out_vocab_file,)







  def 						snake_case	(					self    ,						*UpperCamelCase    ,						**UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						super()._decode(*UpperCamelCase    ,						**UpperCamelCase  )
   lowerCamelCase_  				=						text.replace(" "    ,						""  ).replace("\u2582"    ,						" "  ).replace("\u2583"    ,						"\n"  )
   return text







'''simple docstring'''

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_						:       List[str]      			=				logging.get_logger(__name__)

a_						:       Dict      			=				{
    """microsoft/unispeech-large-1500h-cv""": (
        """https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"""
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}





class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						"unispeech"






  def __init__(					self    ,						UpperCamelCase=32    ,						UpperCamelCase=768    ,						UpperCamelCase=12    ,						UpperCamelCase=12    ,						UpperCamelCase=3072    ,						UpperCamelCase="gelu"    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.0    ,						UpperCamelCase=0.0    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.1    ,						UpperCamelCase=0.02    ,						UpperCamelCase=1e-5    ,						UpperCamelCase="group"    ,						UpperCamelCase="gelu"    ,						UpperCamelCase=(512, 512, 512, 512, 512, 512, 512)    ,						UpperCamelCase=(5, 2, 2, 2, 2, 2, 2)    ,						UpperCamelCase=(10, 3, 3, 3, 3, 2, 2)    ,						UpperCamelCase=False    ,						UpperCamelCase=128    ,						UpperCamelCase=16    ,						UpperCamelCase=False    ,						UpperCamelCase=True    ,						UpperCamelCase=0.05    ,						UpperCamelCase=10    ,						UpperCamelCase=2    ,						UpperCamelCase=0.0    ,						UpperCamelCase=10    ,						UpperCamelCase=0    ,						UpperCamelCase=320    ,						UpperCamelCase=2    ,						UpperCamelCase=0.1    ,						UpperCamelCase=100    ,						UpperCamelCase=256    ,						UpperCamelCase=256    ,						UpperCamelCase=0.1    ,						UpperCamelCase="mean"    ,						UpperCamelCase=False    ,						UpperCamelCase=False    ,						UpperCamelCase=256    ,						UpperCamelCase=80    ,						UpperCamelCase=0    ,						UpperCamelCase=1    ,						UpperCamelCase=2    ,						UpperCamelCase=0.5    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   super().__init__(**UpperCamelCase    ,						pad_token_id=UpperCamelCase    ,						bos_token_id=UpperCamelCase    ,						eos_token_id=UpperCamelCase  )
   lowerCamelCase_  				=						hidden_size
   lowerCamelCase_  				=						feat_extract_norm
   lowerCamelCase_  				=						feat_extract_activation
   lowerCamelCase_  				=						list(UpperCamelCase  )
   lowerCamelCase_  				=						list(UpperCamelCase  )
   lowerCamelCase_  				=						list(UpperCamelCase  )
   lowerCamelCase_  				=						conv_bias
   lowerCamelCase_  				=						num_conv_pos_embeddings
   lowerCamelCase_  				=						num_conv_pos_embedding_groups
   lowerCamelCase_  				=						len(self.conv_dim  )
   lowerCamelCase_  				=						num_hidden_layers
   lowerCamelCase_  				=						intermediate_size
   lowerCamelCase_  				=						hidden_act
   lowerCamelCase_  				=						num_attention_heads
   lowerCamelCase_  				=						hidden_dropout
   lowerCamelCase_  				=						attention_dropout
   lowerCamelCase_  				=						activation_dropout
   lowerCamelCase_  				=						feat_proj_dropout
   lowerCamelCase_  				=						final_dropout
   lowerCamelCase_  				=						layerdrop
   lowerCamelCase_  				=						layer_norm_eps
   lowerCamelCase_  				=						initializer_range
   lowerCamelCase_  				=						num_ctc_classes
   lowerCamelCase_  				=						vocab_size
   lowerCamelCase_  				=						do_stable_layer_norm
   lowerCamelCase_  				=						use_weighted_layer_sum
   lowerCamelCase_  				=						classifier_proj_size

   if (
       (len(self.conv_stride  ) != self.num_feat_extract_layers)
       or (len(self.conv_kernel  ) != self.num_feat_extract_layers)
       or (len(self.conv_dim  ) != self.num_feat_extract_layers)
   ):
    raise ValueError(
        "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
        " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
        f''' {len(self.conv_dim  )}`, `len(config.conv_stride) = {len(self.conv_stride  )}`,'''
        f''' `len(config.conv_kernel) = {len(self.conv_kernel  )}`.'''  )

   # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
   lowerCamelCase_  				=						apply_spec_augment
   lowerCamelCase_  				=						mask_time_prob
   lowerCamelCase_  				=						mask_time_length
   lowerCamelCase_  				=						mask_time_min_masks
   lowerCamelCase_  				=						mask_feature_prob
   lowerCamelCase_  				=						mask_feature_length
   lowerCamelCase_  				=						mask_feature_min_masks

   # parameters for pretraining with codevector quantized representations
   lowerCamelCase_  				=						num_codevectors_per_group
   lowerCamelCase_  				=						num_codevector_groups
   lowerCamelCase_  				=						contrastive_logits_temperature
   lowerCamelCase_  				=						feat_quantizer_dropout
   lowerCamelCase_  				=						num_negatives
   lowerCamelCase_  				=						codevector_dim
   lowerCamelCase_  				=						proj_codevector_dim
   lowerCamelCase_  				=						diversity_loss_weight

   # ctc loss
   lowerCamelCase_  				=						ctc_loss_reduction
   lowerCamelCase_  				=						ctc_zero_infinity

   # pretraining loss
   lowerCamelCase_  				=						replace_prob







  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   return functools.reduce(operator.mul    ,						self.conv_stride    ,						1  )



