





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase_					     =	logging.get_logger(__name__)

lowercase_					     =	{"""vocab_file""": """vocab.txt"""}

lowercase_					     =	{
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

lowercase_					     =	{
    """YituTech/conv-bert-base""": 512,
    """YituTech/conv-bert-medium-small""": 512,
    """YituTech/conv-bert-small""": 512,
}


lowercase_					     =	{
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}
class        SCREAMING_SNAKE_CASE     (UpperCAmelCase   ):
   _UpperCamelCase					:	Optional[int]				      = VOCAB_FILES_NAMES
   _UpperCamelCase					:	List[str]				      = PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase					:	Any				      = PRETRAINED_INIT_CONFIGURATION
   _UpperCamelCase					:	Union[str, Any]				      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase					:	Any				      = ConvBertTokenizer


   def __init__(					self    :       Any    ,  a    :       int=None    ,  a    :       Union[str, Any]=None    ,  a    :       List[Any]=True    ,  a    :       Optional[Any]="[UNK]"    ,  a    :       Optional[Any]="[SEP]"    ,  a    :       int="[PAD]"    ,  a    :       Any="[CLS]"    ,  a    :       List[str]="[MASK]"    ,  a    :       Optional[Any]=True    ,  a    :       Optional[int]=None    ,  **a    :       Optional[Any]    ,  )->     Dict:





     """simple docstring"""







     super().__init__(
         a    ,  tokenizer_file=a    ,  do_lower_case=a    ,  unk_token=a    ,  sep_token=a    ,  pad_token=a    ,  cls_token=a    ,  mask_token=a    ,  tokenize_chinese_chars=a    ,  strip_accents=a    ,  **a    ,  )

     lowercase__			    =						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
     if (
         normalizer_state.get('lowercase'    ,  a ) != do_lower_case
         or normalizer_state.get('strip_accents'    ,  a ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'    ,  a ) != tokenize_chinese_chars
     ):
       lowercase__			    =						getattr(a    ,  normalizer_state.pop('type' ) )
       lowercase__			    =						do_lower_case
       lowercase__			    =						strip_accents
       lowercase__			    =						tokenize_chinese_chars
       lowercase__			    =						normalizer_class(**a )

     lowercase__			    =						do_lower_case


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[str]    ,  a    :       Any    ,  a    :       List[str]=None )->     str:





     """simple docstring"""







     lowercase__			    =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple    ,  a    :       List[int]    ,  a    :       Optional[List[int]] = None )->     List[int]:





     """simple docstring"""







     lowercase__			    =						[self.sep_token_id]
     lowercase__			    =						[self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep ) * [0]
     return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]


   def  SCREAMING_SNAKE_CASE_							(					self    :       Union[str, Any]    ,  a    :       str    ,  a    :       Optional[str] = None )->     Tuple[str]:





     """simple docstring"""







     lowercase__			    =						self._tokenizer.model.save(a    ,  name=a )
     return tuple(a )









from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class        SCREAMING_SNAKE_CASE     (UpperCAmelCase   ):
   _UpperCamelCase					:	Tuple				      = 'ClapFeatureExtractor'
   _UpperCamelCase					:	Union[str, Any]				      = ('RobertaTokenizer', 'RobertaTokenizerFast')


   def __init__(					self    :       List[Any]    ,  a    :       int    ,  a    :       str )->     Any:





     """simple docstring"""







     super().__init__(a    ,  a )


   def __call__(					self    :       Any    ,  a    :       Tuple=None    ,  a    :       Optional[int]=None    ,  a    :       int=None    ,  **a    :       Optional[int] )->     Union[str, Any]:





     """simple docstring"""







     lowercase__			    =						kwargs.pop('sampling_rate'    ,  a )

     if text is None and audios is None:
       raise ValueError('You have to specify either text or audios. Both cannot be none.' )

     if text is not None:
       lowercase__			    =						self.tokenizer(a    ,  return_tensors=a    ,  **a )

     if audios is not None:
       lowercase__			    =						self.feature_extractor(
           a    ,  sampling_rate=a    ,  return_tensors=a    ,  **a )

     if text is not None and audios is not None:
       lowercase__			    =						audio_features.input_features
       return encoding
     elif text is not None:
       return encoding
     else:
       return BatchEncoding(data=dict(**a )    ,  tensor_type=a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       str    ,  *a    :       Dict    ,  **a    :       int )->     Optional[int]:





     """simple docstring"""







     return self.tokenizer.batch_decode(*a    ,  **a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[Any]    ,  *a    :       int    ,  **a    :       Dict )->     Dict:





     """simple docstring"""







     return self.tokenizer.decode(*a    ,  **a )


   @property
   def  SCREAMING_SNAKE_CASE_							(					self    :       Union[str, Any] )->     Optional[int]:





     """simple docstring"""







     lowercase__			    =						self.tokenizer.model_input_names
     lowercase__			    =						self.feature_extractor.model_input_names
     return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names ) )


