


"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__lowerCamelCase      :Optional[Any]	    =		logging.get_logger(__name__)

__lowerCamelCase      :List[Any]	    =		{'vocab_file': 'vocab.txt'}

__lowerCamelCase      :Dict	    =		{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

__lowerCamelCase      :Union[str, Any]	    =		{
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


__lowerCamelCase      :str	    =		{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}


class 			A__	(						__lowercase):


     """simple docstring"""
     snake_case__	:					Dict													=VOCAB_FILES_NAMES
     snake_case__	:					Optional[int]													=PRETRAINED_VOCAB_FILES_MAP
     snake_case__	:					List[Any]													=PRETRAINED_INIT_CONFIGURATION
     snake_case__	:					Optional[int]													=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     snake_case__	:					Optional[int]													=ConvBertTokenizer

     def __init__(	self:						Optional[Any]     ,		__a:						Optional[int]=None     ,		__a:						List[str]=None     ,		__a:						Optional[Any]=True     ,		__a:						List[Any]="[UNK]"     ,		__a:						Optional[Any]="[SEP]"     ,		__a:						Optional[Any]="[PAD]"     ,		__a:						int="[CLS]"     ,		__a:						str="[MASK]"     ,		__a:						Tuple=True     ,		__a:						Optional[int]=None     ,		**__a:						Any     ,		)->    Optional[int]:
           super().__init__(
               __a     ,		tokenizer_file=__a     ,		do_lower_case=__a     ,		unk_token=__a     ,		sep_token=__a     ,		pad_token=__a     ,		cls_token=__a     ,		mask_token=__a     ,		tokenize_chinese_chars=__a     ,		strip_accents=__a     ,		**__a     ,		)

           lowerCamelCase					:					Optional[int]        =	json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
           if (
               normalizer_state.get("""lowercase"""     ,		__a	) != do_lower_case
               or normalizer_state.get("""strip_accents"""     ,		__a	) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""     ,		__a	) != tokenize_chinese_chars
           ):
                 lowerCamelCase					:					List[str]        =	getattr(__a     ,		normalizer_state.pop("""type"""	)	)
                 lowerCamelCase					:					Tuple        =	do_lower_case
                 lowerCamelCase					:					List[Any]        =	strip_accents
                 lowerCamelCase					:					List[Any]        =	tokenize_chinese_chars
                 lowerCamelCase					:					List[str]        =	normalizer_class(**__a	)

           lowerCamelCase					:					Any        =	do_lower_case

     def 		a__						(	self:						Union[str, Any]     ,		__a:						str     ,		__a:						Optional[Any]=None	)->    Any:
           lowerCamelCase					:					Any        =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output

     def 		a__						(	self:						Any     ,		__a:						List[int]     ,		__a:						Optional[List[int]] = None	)->    List[int]:
           lowerCamelCase					:					Optional[int]        =	[self.sep_token_id]
           lowerCamelCase					:					Optional[Any]        =	[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep	) * [0]
           return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







     def 		a__						(	self:						int     ,		__a:						str     ,		__a:						Optional[str] = None	)->    Tuple[str]:
           lowerCamelCase					:					List[Any]        =	self._tokenizer.model.save(__a     ,		name=__a	)
           return tuple(__a	)










"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__lowerCamelCase      :Optional[Any]	    =		{
    'configuration_encodec': [
        'ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'EncodecConfig',
    ],
    'feature_extraction_encodec': ['EncodecFeatureExtractor'],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase      :Union[str, Any]	    =		[
        'ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST',
        'EncodecModel',
        'EncodecPreTrainedModel',
    ]

if TYPE_CHECKING:
    from .configuration_encodec import (
        ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP,
        EncodecConfig,
    )
    from .feature_extraction_encodec import EncodecFeatureExtractor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_encodec import (
            ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST,
            EncodecModel,
            EncodecPreTrainedModel,
        )

else:
    import sys

    __lowerCamelCase      :Dict	    =		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




