



import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_								=     logging.get_logger(__name__)

lowercase_								=     {
    """microsoft/unispeech-sat-base-100h-libri-ft""": (
        """https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"""
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}
class 				_snake_case			(					lowercase__):
				UpperCamelCase__				:					Union[str, Any]     		="""unispeech-sat"""




				def __init__(						self					:						Union[str, Any],					__lowercase					:						str=32,					__lowercase					:						Any=768,					__lowercase					:						Tuple=12,					__lowercase					:						List[str]=12,					__lowercase					:						int=3072,					__lowercase					:						Optional[Any]="gelu",					__lowercase					:						Tuple=0.1,					__lowercase					:						List[Any]=0.1,					__lowercase					:						Optional[int]=0.1,					__lowercase					:						Optional[Any]=0.0,					__lowercase					:						Optional[Any]=0.0,					__lowercase					:						List[str]=0.1,					__lowercase					:						str=0.1,					__lowercase					:						Optional[int]=0.02,					__lowercase					:						Optional[int]=1e-5,					__lowercase					:						Any="group",					__lowercase					:						Dict="gelu",					__lowercase					:						List[Any]=(512, 512, 512, 512, 512, 512, 512),					__lowercase					:						Dict=(5, 2, 2, 2, 2, 2, 2),					__lowercase					:						Union[str, Any]=(10, 3, 3, 3, 3, 2, 2),					__lowercase					:						List[Any]=False,					__lowercase					:						Union[str, Any]=128,					__lowercase					:						Optional[int]=16,					__lowercase					:						Union[str, Any]=False,					__lowercase					:						Optional[int]=True,					__lowercase					:						List[Any]=0.05,					__lowercase					:						Any=10,					__lowercase					:						Tuple=2,					__lowercase					:						Optional[Any]=0.0,					__lowercase					:						str=10,					__lowercase					:						List[Any]=0,					__lowercase					:						Dict=320,					__lowercase					:						str=2,					__lowercase					:						Optional[int]=0.1,					__lowercase					:						int=100,					__lowercase					:						Any=256,					__lowercase					:						str=256,					__lowercase					:						Tuple=0.1,					__lowercase					:						Dict="mean",					__lowercase					:						Optional[Any]=False,					__lowercase					:						Optional[Any]=False,					__lowercase					:						Optional[Any]=256,					__lowercase					:						Optional[int]=(512, 512, 512, 512, 1500),					__lowercase					:						Optional[Any]=(5, 3, 3, 1, 1),					__lowercase					:						Optional[int]=(1, 2, 3, 1, 1),					__lowercase					:						Tuple=512,					__lowercase					:						Tuple=0,					__lowercase					:						str=1,					__lowercase					:						Optional[Any]=2,					__lowercase					:						List[str]=504,					**__lowercase					:						Optional[int],					):
							super().__init__(**__lowercase,					pad_token_id=__lowercase,					bos_token_id=__lowercase,					eos_token_id=__lowercase						)
							lowercase__					  =				hidden_size
							lowercase__					  =				feat_extract_norm
							lowercase__					  =				feat_extract_activation
							lowercase__					  =				list(__lowercase						)
							lowercase__					  =				list(__lowercase						)
							lowercase__					  =				list(__lowercase						)
							lowercase__					  =				conv_bias
							lowercase__					  =				num_conv_pos_embeddings
							lowercase__					  =				num_conv_pos_embedding_groups
							lowercase__					  =				len(self.conv_dim						)
							lowercase__					  =				num_hidden_layers
							lowercase__					  =				intermediate_size
							lowercase__					  =				hidden_act
							lowercase__					  =				num_attention_heads
							lowercase__					  =				hidden_dropout
							lowercase__					  =				attention_dropout
							lowercase__					  =				activation_dropout
							lowercase__					  =				feat_proj_dropout
							lowercase__					  =				final_dropout
							lowercase__					  =				layerdrop
							lowercase__					  =				layer_norm_eps
							lowercase__					  =				initializer_range
							lowercase__					  =				vocab_size
							lowercase__					  =				num_clusters
							lowercase__					  =				do_stable_layer_norm
							lowercase__					  =				use_weighted_layer_sum

							if (
							    (len(self.conv_stride						) != self.num_feat_extract_layers)
							    or (len(self.conv_kernel						) != self.num_feat_extract_layers)
							    or (len(self.conv_dim						) != self.num_feat_extract_layers)
							):
										raise ValueError(
										    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
										    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
										    F''' {len(self.conv_dim						)}`, `len(config.conv_stride) = {len(self.conv_stride						)}`,'''
										    F''' `len(config.conv_kernel) = {len(self.conv_kernel						)}`.'''						)

							# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
							lowercase__					  =				apply_spec_augment
							lowercase__					  =				mask_time_prob
							lowercase__					  =				mask_time_length
							lowercase__					  =				mask_time_min_masks
							lowercase__					  =				mask_feature_prob
							lowercase__					  =				mask_feature_length
							lowercase__					  =				mask_feature_min_masks

							# parameters for pretraining with codevector quantized representations
							lowercase__					  =				num_codevectors_per_group
							lowercase__					  =				num_codevector_groups
							lowercase__					  =				contrastive_logits_temperature
							lowercase__					  =				feat_quantizer_dropout
							lowercase__					  =				num_negatives
							lowercase__					  =				codevector_dim
							lowercase__					  =				proj_codevector_dim
							lowercase__					  =				diversity_loss_weight

							# ctc loss
							lowercase__					  =				ctc_loss_reduction
							lowercase__					  =				ctc_zero_infinity

							# SequenceClassification-specific parameter. Feel free to ignore for other classes.
							lowercase__					  =				classifier_proj_size

							# XVector-specific parameters. Feel free to ignore for other classes.
							lowercase__					  =				list(__lowercase						)
							lowercase__					  =				list(__lowercase						)
							lowercase__					  =				list(__lowercase						)
							lowercase__					  =				xvector_output_dim




				@property
				def 							A__      (						self					:						List[str]						):
							return functools.reduce(operator.mul,					self.conv_stride,					1						)









import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class 				_snake_case			(					lowercase__    ,	unittest.TestCase):
				UpperCamelCase__				:					Dict     		=TransfoXLTokenizer
				UpperCamelCase__				:					List[Any]     		=False
				UpperCamelCase__				:					List[Any]     		=False




				def 							A__      (						self					:						Union[str, Any]						):
							super().setUp()

							lowercase__					  =				[
							    "<unk>",
							    "[CLS]",
							    "[SEP]",
							    "want",
							    "unwanted",
							    "wa",
							    "un",
							    "running",
							    ",",
							    "low",
							    "l",
							]
							lowercase__					  =				os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES["vocab_file"]						)
							with open(self.vocab_file,					"w",					encoding="utf-8"						) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)




				def 							A__      (						self					:						Union[str, Any],					**__lowercase					:						Any						):
							lowercase__					  =				True
							return TransfoXLTokenizer.from_pretrained(self.tmpdirname,					**__lowercase						)




				def 							A__      (						self					:						Tuple,					__lowercase					:						Optional[int]						):
							lowercase__					  =				"<unk> UNwanted , running"
							lowercase__					  =				"<unk> unwanted, running"
							return input_text, output_text




				def 							A__      (						self					:						str						):
							lowercase__					  =				TransfoXLTokenizer(vocab_file=self.vocab_file,					lower_case=__lowercase						)

							lowercase__					  =				tokenizer.tokenize("<unk> UNwanted , running"						)
							self.assertListEqual(__lowercase,					["<unk>", "unwanted", ",", "running"]						)

							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						),					[0, 4, 8, 7]						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				TransfoXLTokenizer(lower_case=__lowercase						)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "						),					["hello", "!", "how", "are", "you", "?"]						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				TransfoXLTokenizer(lower_case=__lowercase						)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "						),					["HeLLo", "!", "how", "Are", "yoU", "?"]						)




				def 							A__      (						self					:						str						):
							lowercase__					  =				TransfoXLTokenizer(lower_case=__lowercase						)
							lowercase__					  =				"Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"
							lowercase__					  =				[
							    "Hello",
							    "(",
							    "bracket",
							    ")",
							    "and",
							    "side",
							    "@-@",
							    "scrolled",
							    "[",
							    "and",
							    "]",
							    "Henry",
							    "'s",
							    "$",
							    "5",
							    "@,@",
							    "000",
							    "with",
							    "3",
							    "@.@",
							    "34",
							    "m",
							    ".",
							    "What",
							    "'s",
							    "up",
							    "!",
							    "?",
							]

							self.assertListEqual(tokenizer.tokenize(__lowercase						),					__lowercase						)

							self.assertEqual(tokenizer.convert_tokens_to_string(__lowercase						),					__lowercase						)




				def 							A__      (						self					:						List[str]						):
							lowercase__					  =				self.get_tokenizer()
							lowercase__					  =				len(__lowercase						)

							tokenizer.add_tokens(["new1", "new2"]						)
							tokenizer.move_added_token("new1",					1						)

							# Check that moved token is not copied (duplicate)
							self.assertEqual(len(__lowercase						),					original_len + 2						)
							# Check that token is moved to specified id
							self.assertEqual(tokenizer.encode("new1"						),					[1]						)
							self.assertEqual(tokenizer.decode([1]						),					"new1"						)




