




"""simple docstring"""

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_			   =					logging.get_logger(__name__)

a_			   =					{
    'asapp/sew-d-tiny-100k': 'https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							str							       =		'sew-d'







						def __init__(self   ,		lowercase__=32   ,		lowercase__=7_68   ,		lowercase__=12   ,		lowercase__=12   ,		lowercase__=30_72   ,		lowercase__=2   ,		lowercase__=5_12   ,		lowercase__=2_56   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=("p2c", "c2p")   ,		lowercase__="layer_norm"   ,		lowercase__="gelu_python"   ,		lowercase__=0.1   ,		lowercase__=0.1   ,		lowercase__=0.1   ,		lowercase__=0.0   ,		lowercase__=0.1   ,		lowercase__=0.02   ,		lowercase__=1e-7   ,		lowercase__=1e-5   ,		lowercase__="group"   ,		lowercase__="gelu"   ,		lowercase__=(64, 1_28, 1_28, 1_28, 1_28, 2_56, 2_56, 2_56, 2_56, 5_12, 5_12, 5_12, 5_12)   ,		lowercase__=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)   ,		lowercase__=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)   ,		lowercase__=False   ,		lowercase__=1_28   ,		lowercase__=16   ,		lowercase__=True   ,		lowercase__=0.05   ,		lowercase__=10   ,		lowercase__=2   ,		lowercase__=0.0   ,		lowercase__=10   ,		lowercase__=0   ,		lowercase__="mean"   ,		lowercase__=False   ,		lowercase__=False   ,		lowercase__=2_56   ,		lowercase__=0   ,		lowercase__=1   ,		lowercase__=2   ,		**lowercase__   ,		):
											super().__init__(**UpperCAmelCase__   ,		pad_token_id=UpperCAmelCase__   ,		bos_token_id=UpperCAmelCase__   ,		eos_token_id=UpperCAmelCase__       )
											snake_case_					:		Dict 					=					hidden_size
											snake_case_					:		List[str] 					=					feat_extract_norm
											snake_case_					:		List[str] 					=					feat_extract_activation
											snake_case_					:		int 					=					list(UpperCAmelCase__       )
											snake_case_					:		Optional[Any] 					=					list(UpperCAmelCase__       )
											snake_case_					:		Any 					=					list(UpperCAmelCase__       )
											snake_case_					:		Union[str, Any] 					=					conv_bias
											snake_case_					:		Dict 					=					num_conv_pos_embeddings
											snake_case_					:		List[str] 					=					num_conv_pos_embedding_groups
											snake_case_					:		Any 					=					len(self.conv_dim       )
											snake_case_					:		Optional[int] 					=					num_hidden_layers
											snake_case_					:		Dict 					=					intermediate_size
											snake_case_					:		List[str] 					=					squeeze_factor
											snake_case_					:		Optional[Any] 					=					max_position_embeddings
											snake_case_					:		str 					=					position_buckets
											snake_case_					:		str 					=					share_att_key
											snake_case_					:		Optional[int] 					=					relative_attention
											snake_case_					:		Dict 					=					norm_rel_ebd
											snake_case_					:		str 					=					list(UpperCAmelCase__       )
											snake_case_					:		Optional[int] 					=					hidden_act
											snake_case_					:		Union[str, Any] 					=					num_attention_heads
											snake_case_					:		str 					=					hidden_dropout
											snake_case_					:		Optional[int] 					=					attention_dropout
											snake_case_					:		Any 					=					activation_dropout
											snake_case_					:		Optional[Any] 					=					feat_proj_dropout
											snake_case_					:		List[Any] 					=					final_dropout
											snake_case_					:		Any 					=					layer_norm_eps
											snake_case_					:		Optional[int] 					=					feature_layer_norm_eps
											snake_case_					:		str 					=					initializer_range
											snake_case_					:		Tuple 					=					vocab_size

											if (
											    (len(self.conv_stride       ) != self.num_feat_extract_layers)
											    or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
											    or (len(self.conv_dim       ) != self.num_feat_extract_layers)
											):
																raise ValueError(
																    """Configuration for convolutional layers is incorrect."""
																    """It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"""
																    f'but is `len(config.conv_dim) = {len(self.conv_dim       )}`, `len(config.conv_stride)'
																    f'= {len(self.conv_stride       )}`, `len(config.conv_kernel) = {len(self.conv_kernel       )}`.'       )

											# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
											snake_case_					:		List[str] 					=					apply_spec_augment
											snake_case_					:		Optional[int] 					=					mask_time_prob
											snake_case_					:		Union[str, Any] 					=					mask_time_length
											snake_case_					:		List[Any] 					=					mask_time_min_masks
											snake_case_					:		Dict 					=					mask_feature_prob
											snake_case_					:		Optional[Any] 					=					mask_feature_length
											snake_case_					:		int 					=					mask_feature_min_masks

											# ctc loss
											snake_case_					:		Optional[Any] 					=					ctc_loss_reduction
											snake_case_					:		int 					=					ctc_zero_infinity

											# sequence classification
											snake_case_					:		str 					=					use_weighted_layer_sum
											snake_case_					:		List[Any] 					=					classifier_proj_size







						@property
						def 						__UpperCamelCase							(self       ):
											return functools.reduce(operator.mul   ,		self.conv_stride   ,		1       )









"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			a_			   =					None


a_			   =					logging.get_logger(__name__)


a_			   =					{'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

a_			   =					{
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-en-ro''': '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json''',
        '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json''',
    },
}

a_			   =					{
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
a_			   =					['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							Dict							       =		VOCAB_FILES_NAMES
						_A		:							Union[str, Any]							       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_A		:							Optional[int]							       =		PRETRAINED_VOCAB_FILES_MAP
						_A		:							str							       =		["""input_ids""", """attention_mask"""]
						_A		:							Tuple							       =		MBartTokenizer

						_A		:							List[int]							       =		[]
						_A		:							List[int]							       =		[]







						def __init__(self   ,		lowercase__=None   ,		lowercase__=None   ,		lowercase__="<s>"   ,		lowercase__="</s>"   ,		lowercase__="</s>"   ,		lowercase__="<s>"   ,		lowercase__="<unk>"   ,		lowercase__="<pad>"   ,		lowercase__="<mask>"   ,		lowercase__=None   ,		lowercase__=None   ,		lowercase__=None   ,		**lowercase__   ,		):
											# Mask token behave like a normal word, i.e. include the space before it
											snake_case_					:		int 					=					AddedToken(lowercase__   ,		lstrip=lowercase__   ,		rstrip=lowercase__       ) if isinstance(lowercase__   ,		lowercase__       ) else mask_token

											super().__init__(
											    vocab_file=lowercase__   ,		tokenizer_file=lowercase__   ,		bos_token=lowercase__   ,		eos_token=lowercase__   ,		sep_token=lowercase__   ,		cls_token=lowercase__   ,		unk_token=lowercase__   ,		pad_token=lowercase__   ,		mask_token=lowercase__   ,		src_lang=lowercase__   ,		tgt_lang=lowercase__   ,		additional_special_tokens=lowercase__   ,		**lowercase__   ,		)

											snake_case_					:		Dict 					=					vocab_file
											snake_case_					:		Optional[int] 					=					False if not self.vocab_file else True

											snake_case_					:		Optional[Any] 					=					FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
																# Only add those special tokens if they are not already there.
																_additional_special_tokens.extend(
																    [t for t in additional_special_tokens if t not in _additional_special_tokens]       )

											self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}       )
											snake_case_					:		Any 					=					{
											    lang_code: self.convert_tokens_to_ids(lowercase__       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											snake_case_					:		Tuple 					=					src_lang if src_lang is not None else """en_XX"""
											snake_case_					:		Tuple 					=					self.convert_tokens_to_ids(self._src_lang       )
											snake_case_					:		Tuple 					=					tgt_lang
											self.set_src_lang_special_tokens(self._src_lang       )







						@property
						def 						__UpperCamelCase							(self       ):
											return self._src_lang







						@src_lang.setter
						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					:		Tuple 					=					new_src_lang
											self.set_src_lang_special_tokens(self._src_lang       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = None       ):
											if token_ids_a is None:
																return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = None       ):
											snake_case_					:		List[Any] 					=					[self.sep_token_id]
											snake_case_					:		Optional[int] 					=					[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		**lowercase__       ):
											if src_lang is None or tgt_lang is None:
																raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""       )
											snake_case_					:		int 					=					src_lang
											snake_case_					:		List[str] 					=					self(lowercase__   ,		add_special_tokens=lowercase__   ,		return_tensors=lowercase__   ,		**lowercase__       )
											snake_case_					:		List[str] 					=					self.convert_tokens_to_ids(lowercase__       )
											snake_case_					:		Union[str, Any] 					=					tgt_lang_id
											return inputs







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = "en_XX"   ,		lowercase__ = None   ,		lowercase__ = "ro_RO"   ,		**lowercase__   ,		):
											snake_case_					:		List[str] 					=					src_lang
											snake_case_					:		int 					=					tgt_lang
											return super().prepare_seqaseq_batch(lowercase__   ,		lowercase__   ,		**lowercase__       )







						def 						__UpperCamelCase							(self       ):
											return self.set_src_lang_special_tokens(self.src_lang       )







						def 						__UpperCamelCase							(self       ):
											return self.set_tgt_lang_special_tokens(self.tgt_lang       )







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					:		int 					=					self.convert_tokens_to_ids(lowercase__       )
											snake_case_					:		Tuple 					=					[]
											snake_case_					:		List[Any] 					=					[self.eos_token_id, self.cur_lang_code]

											snake_case_					:		List[Any] 					=					self.convert_ids_to_tokens(self.prefix_tokens       )
											snake_case_					:		Union[str, Any] 					=					self.convert_ids_to_tokens(self.suffix_tokens       )

											snake_case_					:		Optional[int] 					=					processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str   ,		pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str   ,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,		self.prefix_tokens + self.suffix_tokens       )       )   ,		)







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					:		Tuple 					=					self.convert_tokens_to_ids(lowercase__       )
											snake_case_					:		Optional[int] 					=					[]
											snake_case_					:		Optional[Any] 					=					[self.eos_token_id, self.cur_lang_code]

											snake_case_					:		Union[str, Any] 					=					self.convert_ids_to_tokens(self.prefix_tokens       )
											snake_case_					:		int 					=					self.convert_ids_to_tokens(self.suffix_tokens       )

											snake_case_					:		List[Any] 					=					processors.TemplateProcessing(
											    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str   ,		pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str   ,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,		self.prefix_tokens + self.suffix_tokens       )       )   ,		)







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = None       ):
											if not self.can_save_slow_tokenizer:
																raise ValueError(
																    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																    """tokenizer."""       )

											if not os.path.isdir(lowercase__       ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory.'       )
																return
											snake_case_					:		List[str] 					=					os.path.join(
											    lowercase__   ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

											if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowercase__       ):
																copyfile(self.vocab_file   ,		lowercase__       )

											return (out_vocab_file,)




