


'''simple docstring'''

from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

_UpperCAmelCase				: List[str]							    =				_symbol_database.Default()


_UpperCAmelCase				: List[Any]							    =				_descriptor_pool.Default().AddSerializedFile(
    b"""\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03"""
)

_UpperCAmelCase				: Optional[Any]							    =				globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, """sentencepiece_model_pb2""", _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
 _UpperCAmelCase				: int							    =				None
 _UpperCAmelCase				: Tuple							    =				b"""H\003"""
 # (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
 # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
 # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
 # _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
 # _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
 _UpperCAmelCase				: Any							    =				4_5
 _UpperCAmelCase				: Tuple							    =				1_5_8_1
 _UpperCAmelCase				: Optional[int]							    =				1_5_1_7
 _UpperCAmelCase				: Optional[int]							    =				1_5_7_0
 _UpperCAmelCase				: Optional[int]							    =				1_5_8_4
 _UpperCAmelCase				: Optional[Any]							    =				1_7_9_3
 _UpperCAmelCase				: Union[str, Any]							    =				1_7_9_5
 _UpperCAmelCase				: Dict							    =				1_9_1_6
 _UpperCAmelCase				: List[Any]							    =				1_8_6_4
 _UpperCAmelCase				: Optional[Any]							    =				1_9_0_5
 _UpperCAmelCase				: Tuple							    =				1_9_1_9
 _UpperCAmelCase				: Dict							    =				2_4_2_9
 _UpperCAmelCase				: Optional[int]							    =				2_2_0_8
 _UpperCAmelCase				: Union[str, Any]							    =				2_4_1_8
 _UpperCAmelCase				: List[str]							    =				2_3_2_3
 _UpperCAmelCase				: Optional[Any]							    =				2_4_0_7
# @@protoc_insertion_point(module_scope)





'''simple docstring'''

import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


_UpperCAmelCase				: Tuple							    =				random.Random()

if is_torch_available():
	import torch

def 							__magic_name__(						lowerCamelCase,				lowerCamelCase=1.0,				lowerCamelCase=None,				lowerCamelCase=None):
						if rng is None:
												__lowerCAmelCase							       =				global_rng

						__lowerCAmelCase							       =				[]
						for batch_idx in range(shape[0]):
												values.append([])
												for _ in range(shape[1]):
																		values[-1].append(rng.random() * scale)

						return values


class       a__				( unittest.TestCase      ):







				"""simple docstring"""



				def __init__(self       ,			__lowercase       ,			__lowercase=7       ,			__lowercase=4_00       ,			__lowercase=20_00       ,			__lowercase=1       ,			__lowercase=0.0       ,			__lowercase=1_60_00       ,			__lowercase=True       ,			__lowercase=True       ,			):
										__lowerCAmelCase							       =				parent
										__lowerCAmelCase							       =				batch_size
										__lowerCAmelCase							       =				min_seq_length
										__lowerCAmelCase							       =				max_seq_length
										__lowerCAmelCase							       =				(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
										__lowerCAmelCase							       =				feature_size
										__lowerCAmelCase							       =				padding_value
										__lowerCAmelCase							       =				sampling_rate
										__lowerCAmelCase							       =				return_attention_mask
										__lowerCAmelCase							       =				do_normalize




				def 					_snake_case  (self		):
										return {
										    "feature_size": self.feature_size,
										    "padding_value": self.padding_value,
										    "sampling_rate": self.sampling_rate,
										    "return_attention_mask": self.return_attention_mask,
										    "do_normalize": self.do_normalize,
										}




				def 					_snake_case  (self       ,			__lowercase=False       ,			__lowercase=False		):
										def _flatten(__lowercase		):
																return list(itertools.chain(*__lowercase		)		)

										if equal_length:
																__lowerCAmelCase							       =				floats_list((self.batch_size, self.max_seq_length)		)
										else:
																# make sure that inputs increase in size
																__lowerCAmelCase							       =				[
																    _flatten(floats_list((x, self.feature_size)		)		)
																    for x in range(self.min_seq_length       ,			self.max_seq_length       ,			self.seq_length_diff		)
																]

										if numpify:
																__lowerCAmelCase							       =				[np.asarray(__lowercase		) for x in speech_inputs]

										return speech_inputs


@require_torch
@require_torchaudio
class       a__				( __A							,						unittest.TestCase      ):







				"""simple docstring"""



				__UpperCamelCase    :		Union[str, Any]	=	ASTFeatureExtractor




				def 					_snake_case  (self		):
										__lowerCAmelCase							       =				ASTFeatureExtractionTester(self		)




				def 					_snake_case  (self		):
										# Tests that all call wrap to encode_plus and batch_encode_plus
										__lowerCAmelCase							       =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
										# create three inputs of length 800, 1000, and 1200
										__lowerCAmelCase							       =				[floats_list((1, x)		)[0] for x in range(8_00       ,			14_00       ,			2_00		)]
										__lowerCAmelCase							       =				[np.asarray(__lowercase		) for speech_input in speech_inputs]

										# Test not batched input
										__lowerCAmelCase							       =				feat_extract(speech_inputs[0]       ,			return_tensors='''np'''		).input_values
										__lowerCAmelCase							       =				feat_extract(np_speech_inputs[0]       ,			return_tensors='''np'''		).input_values
										self.assertTrue(np.allclose(__lowercase       ,			__lowercase       ,			atol=1e-3		)		)

										# Test batched
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			padding=__lowercase       ,			return_tensors='''np'''		).input_values
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			padding=__lowercase       ,			return_tensors='''np'''		).input_values
										for enc_seq_a, enc_seq_a in zip(__lowercase       ,			__lowercase		):
																self.assertTrue(np.allclose(__lowercase       ,			__lowercase       ,			atol=1e-3		)		)

										# Test 2-D numpy arrays are batched.
										__lowerCAmelCase							       =				[floats_list((1, x)		)[0] for x in (8_00, 8_00, 8_00)]
										__lowerCAmelCase							       =				np.asarray(__lowercase		)
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			return_tensors='''np'''		).input_values
										__lowerCAmelCase							       =				feat_extract(__lowercase       ,			return_tensors='''np'''		).input_values
										for enc_seq_a, enc_seq_a in zip(__lowercase       ,			__lowercase		):
																self.assertTrue(np.allclose(__lowercase       ,			__lowercase       ,			atol=1e-3		)		)




				@require_torch
				def 					_snake_case  (self		):
										import torch

										__lowerCAmelCase							       =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
										__lowerCAmelCase							       =				np.random.rand(1_00		).astype(np.floataa		)
										__lowerCAmelCase							       =				np_speech_inputs.tolist()

										for inputs in [py_speech_inputs, np_speech_inputs]:
																__lowerCAmelCase							       =				feature_extractor.pad([{'''input_values''': inputs}]       ,			return_tensors='''np'''		)
																self.assertTrue(np_processed.input_values.dtype == np.floataa		)
																__lowerCAmelCase							       =				feature_extractor.pad([{'''input_values''': inputs}]       ,			return_tensors='''pt'''		)
																self.assertTrue(pt_processed.input_values.dtype == torch.floataa		)




				def 					_snake_case  (self       ,			__lowercase		):
										from datasets import load_dataset

										__lowerCAmelCase							       =				load_dataset('''hf-internal-testing/librispeech_asr_dummy'''       ,			'''clean'''       ,			split='''validation'''		)
										# automatic decoding with librispeech
										__lowerCAmelCase							       =				ds.sort('''id'''		).select(range(__lowercase		)		)[:num_samples]['''audio''']

										return [x["array"] for x in speech_samples]




				@require_torch
				def 					_snake_case  (self		):
										# fmt: off
										__lowerCAmelCase							       =				torch.tensor(
										    [-0.9_8_9_4, -1.2_7_7_6, -0.9_0_6_6, -1.2_7_7_6, -0.9_3_4_9, -1.2_6_0_9, -1.0_3_8_6, -1.2_7_7_6,
										     -1.1_5_6_1, -1.2_7_7_6, -1.2_0_5_2, -1.2_7_2_3, -1.2_1_9_0, -1.2_1_3_2, -1.2_7_7_6, -1.1_1_3_3,
										     -1.1_9_5_3, -1.1_3_4_3, -1.1_5_8_4, -1.2_2_0_3, -1.1_7_7_0, -1.2_4_7_4, -1.2_3_8_1, -1.1_9_3_6,
										     -0.9_2_7_0, -0.8_3_1_7, -0.8_0_4_9, -0.7_7_0_6, -0.7_5_6_5, -0.7_8_6_9]		)
										# fmt: on

										__lowerCAmelCase							       =				self._load_datasamples(1		)
										__lowerCAmelCase							       =				ASTFeatureExtractor()
										__lowerCAmelCase							       =				feature_extractor(__lowercase       ,			return_tensors='''pt'''		).input_values
										self.assertEquals(input_values.shape       ,			(1, 10_24, 1_28)		)
										self.assertTrue(torch.allclose(input_values[0, 0, :30]       ,			__lowercase       ,			atol=1e-4		)		)

