






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


UpperCAmelCase__						:Dict          =			logging.get_logger(__name__)

UpperCAmelCase__						:Optional[Any]          =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase__						:List[str]          =			{
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

UpperCAmelCase__						:Optional[Any]          =			{'mobilebert-uncased': 512}


UpperCAmelCase__						:str          =			{}







class 	SCREAMING_SNAKE_CASE	(      UpperCamelCase_  ):
       snake_case__    :     Union[str, Any]   		=			VOCAB_FILES_NAMES
       snake_case__    :     Optional[int]   		=			PRETRAINED_VOCAB_FILES_MAP
       snake_case__    :     Union[str, Any]   		=			PRETRAINED_INIT_CONFIGURATION
       snake_case__    :     Tuple   		=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       snake_case__    :     Optional[int]   		=			MobileBertTokenizer




       def __init__(					self      : int	,    A__      : List[Any]=None	,    A__      : Tuple=None	,    A__      : Any=True	,    A__      : Dict="[UNK]"	,    A__      : List[str]="[SEP]"	,    A__      : Any="[PAD]"	,    A__      : Optional[int]="[CLS]"	,    A__      : str="[MASK]"	,    A__      : Tuple=True	,    A__      : List[str]=None	,    **A__      : List[str]	,    ):


            """simple docstring"""







            super().__init__(
                A__	,    tokenizer_file=A__	,    do_lower_case=A__	,    unk_token=A__	,    sep_token=A__	,    pad_token=A__	,    cls_token=A__	,    mask_token=A__	,    tokenize_chinese_chars=A__	,    strip_accents=A__	,    **A__	,    )

            __lowerCamelCase						:       Optional[Any]				   =				json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
            if (
                normalizer_state.get("""lowercase"""	,    A__					) != do_lower_case
                or normalizer_state.get("""strip_accents"""	,    A__					) != strip_accents
                or normalizer_state.get("""handle_chinese_chars"""	,    A__					) != tokenize_chinese_chars
            ):
                 __lowerCamelCase						:       Union[str, Any]				   =				getattr(A__	,    normalizer_state.pop("""type"""					)					)
                 __lowerCamelCase						:       List[Any]				   =				do_lower_case
                 __lowerCamelCase						:       Any				   =				strip_accents
                 __lowerCamelCase						:       Tuple				   =				tokenize_chinese_chars
                 __lowerCamelCase						:       str				   =				normalizer_class(**A__					)

            __lowerCamelCase						:       Union[str, Any]				   =				do_lower_case




       def 					a_					(					self      : int	,    A__      : int	,    A__      : Optional[int]=None					):


            """simple docstring"""







            __lowerCamelCase						:       int				   =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

            if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

            return output




       def 					a_					(					self      : Any	,    A__      : Optional[Any]	,    A__      : List[str] = None					):


            """simple docstring"""







            __lowerCamelCase						:       Dict				   =				[self.sep_token_id]
            __lowerCamelCase						:       int				   =				[self.cls_token_id]
            if token_ids_a is None:
                 return len(cls + token_ids_a + sep					) * [0]
            return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]




       def 					a_					(					self      : Tuple	,    A__      : List[Any]	,    A__      : int = None					):


            """simple docstring"""







            __lowerCamelCase						:       Dict				   =				self._tokenizer.model.save(A__	,    name=A__					)
            return tuple(A__					)











'''simple docstring'''

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__						:Optional[int]          =			logging.get_logger(__name__)

UpperCAmelCase__						:List[Any]          =			{
    """asapp/sew-d-tiny-100k""": """https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json""",
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}







class 	SCREAMING_SNAKE_CASE	(      lowerCAmelCase_  ):
       snake_case__    :     Dict   		=			'sew-d'




       def __init__(					self      : Tuple	,    A__      : Optional[int]=32	,    A__      : Optional[int]=768	,    A__      : Any=12	,    A__      : List[str]=12	,    A__      : List[Any]=3072	,    A__      : str=2	,    A__      : Dict=512	,    A__      : Optional[Any]=256	,    A__      : Optional[Any]=True	,    A__      : Any=True	,    A__      : List[str]=("p2c", "c2p")	,    A__      : List[Any]="layer_norm"	,    A__      : Union[str, Any]="gelu_python"	,    A__      : int=0.1	,    A__      : Dict=0.1	,    A__      : int=0.1	,    A__      : Dict=0.0	,    A__      : Optional[Any]=0.1	,    A__      : Dict=0.02	,    A__      : Dict=1e-7	,    A__      : List[Any]=1e-5	,    A__      : Any="group"	,    A__      : Any="gelu"	,    A__      : Dict=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)	,    A__      : Dict=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)	,    A__      : str=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)	,    A__      : Optional[Any]=False	,    A__      : Any=128	,    A__      : Optional[Any]=16	,    A__      : Union[str, Any]=True	,    A__      : Any=0.05	,    A__      : List[str]=10	,    A__      : Union[str, Any]=2	,    A__      : Dict=0.0	,    A__      : str=10	,    A__      : Tuple=0	,    A__      : Any="mean"	,    A__      : Optional[int]=False	,    A__      : int=False	,    A__      : List[str]=256	,    A__      : Union[str, Any]=0	,    A__      : int=1	,    A__      : Optional[Any]=2	,    **A__      : List[Any]	,    ):


            """simple docstring"""







            super().__init__(**A__	,    pad_token_id=A__	,    bos_token_id=A__	,    eos_token_id=A__					)
            __lowerCamelCase						:       List[str]				   =				hidden_size
            __lowerCamelCase						:       Tuple				   =				feat_extract_norm
            __lowerCamelCase						:       Tuple				   =				feat_extract_activation
            __lowerCamelCase						:       List[Any]				   =				list(A__					)
            __lowerCamelCase						:       int				   =				list(A__					)
            __lowerCamelCase						:       Optional[Any]				   =				list(A__					)
            __lowerCamelCase						:       Tuple				   =				conv_bias
            __lowerCamelCase						:       List[str]				   =				num_conv_pos_embeddings
            __lowerCamelCase						:       Tuple				   =				num_conv_pos_embedding_groups
            __lowerCamelCase						:       int				   =				len(self.conv_dim					)
            __lowerCamelCase						:       Optional[int]				   =				num_hidden_layers
            __lowerCamelCase						:       Dict				   =				intermediate_size
            __lowerCamelCase						:       Optional[int]				   =				squeeze_factor
            __lowerCamelCase						:       Union[str, Any]				   =				max_position_embeddings
            __lowerCamelCase						:       int				   =				position_buckets
            __lowerCamelCase						:       Tuple				   =				share_att_key
            __lowerCamelCase						:       Any				   =				relative_attention
            __lowerCamelCase						:       Any				   =				norm_rel_ebd
            __lowerCamelCase						:       Dict				   =				list(A__					)
            __lowerCamelCase						:       Optional[Any]				   =				hidden_act
            __lowerCamelCase						:       List[str]				   =				num_attention_heads
            __lowerCamelCase						:       List[str]				   =				hidden_dropout
            __lowerCamelCase						:       Union[str, Any]				   =				attention_dropout
            __lowerCamelCase						:       Tuple				   =				activation_dropout
            __lowerCamelCase						:       Union[str, Any]				   =				feat_proj_dropout
            __lowerCamelCase						:       Union[str, Any]				   =				final_dropout
            __lowerCamelCase						:       List[str]				   =				layer_norm_eps
            __lowerCamelCase						:       Tuple				   =				feature_layer_norm_eps
            __lowerCamelCase						:       Union[str, Any]				   =				initializer_range
            __lowerCamelCase						:       List[str]				   =				vocab_size

            if (
                (len(self.conv_stride					) != self.num_feat_extract_layers)
                or (len(self.conv_kernel					) != self.num_feat_extract_layers)
                or (len(self.conv_dim					) != self.num_feat_extract_layers)
            ):
                 raise ValueError(
                     """Configuration for convolutional layers is incorrect."""
                     """It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"""
                     f"but is `len(config.conv_dim) = {len(self.conv_dim					)}`, `len(config.conv_stride)"
                     f"= {len(self.conv_stride					)}`, `len(config.conv_kernel) = {len(self.conv_kernel					)}`."					)

            # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
            __lowerCamelCase						:       Any				   =				apply_spec_augment
            __lowerCamelCase						:       str				   =				mask_time_prob
            __lowerCamelCase						:       Tuple				   =				mask_time_length
            __lowerCamelCase						:       Optional[int]				   =				mask_time_min_masks
            __lowerCamelCase						:       Dict				   =				mask_feature_prob
            __lowerCamelCase						:       Optional[Any]				   =				mask_feature_length
            __lowerCamelCase						:       str				   =				mask_feature_min_masks

            # ctc loss
            __lowerCamelCase						:       Any				   =				ctc_loss_reduction
            __lowerCamelCase						:       str				   =				ctc_zero_infinity

            # sequence classification
            __lowerCamelCase						:       Dict				   =				use_weighted_layer_sum
            __lowerCamelCase						:       List[Any]				   =				classifier_proj_size




       @property
       def 					a_					(					self      : Optional[int]					):


            """simple docstring"""







            return functools.reduce(operator.mul	,    self.conv_stride	,    1					)



