




from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

_snake_case            = _symbol_database.Default()


_snake_case            = _descriptor_pool.Default().AddSerializedFile(
    b"\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03"
)

_snake_case            = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "sentencepiece_model_pb2", _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
		_snake_case            = None
		_snake_case            = b"H\003"
		# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
		# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
		# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
		# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
		# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
		_snake_case            = 45
		_snake_case            = 1581
		_snake_case            = 1517
		_snake_case            = 1570
		_snake_case            = 1584
		_snake_case            = 1793
		_snake_case            = 1795
		_snake_case            = 1916
		_snake_case            = 1864
		_snake_case            = 1905
		_snake_case            = 1919
		_snake_case            = 2429
		_snake_case            = 2208
		_snake_case            = 2418
		_snake_case            = 2323
		_snake_case            = 2407
# @@protoc_insertion_point(module_scope)







import itertools
import os
import random
import tempfile
import unittest

import numpy as np

from transformers import TvltFeatureExtractor, is_datasets_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
  import torch

if is_datasets_available():
  from datasets import load_dataset

_snake_case            = random.Random()







def        lowerCAmelCase_			(  snake_case_,snake_case_=1.0,snake_case_=None,snake_case_=None  ):
      if rng is None:
            _A							:  str    =		global_rng

      _A							:  List[Any]    =		[]
      for batch_idx in range(shape[0]  ):
            values.append([]  )
            for _ in range(shape[1]  ):
                  values[-1].append(rng.random() * scale  )

      return values







class 		lowercase							(     unittest.TestCase	):

     def __init__(						self	,  _a	,  _a=7	,  _a=400	,  _a=2000	,  _a=2048	,  _a=128	,  _a=1	,  _a=512	,  _a=30	,  _a=4_4100	,  )		->							Tuple:
           _A							:  Any    =		parent
           _A							:  str    =		batch_size
           _A							:  Union[str, Any]    =		min_seq_length
           _A							:  int    =		max_seq_length
           _A							:  List[str]    =		(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
           _A							:  Tuple    =		spectrogram_length
           _A							:  int    =		feature_size
           _A							:  str    =		num_audio_channels
           _A							:  Tuple    =		hop_length
           _A							:  List[str]    =		chunk_length
           _A							:  Union[str, Any]    =		sampling_rate

     def       a__			(						self       )		->							Tuple:
           return {
               "spectrogram_length": self.spectrogram_length,
               "feature_size": self.feature_size,
               "num_audio_channels": self.num_audio_channels,
               "hop_length": self.hop_length,
               "chunk_length": self.chunk_length,
               "sampling_rate": self.sampling_rate,
           }



     def       a__			(						self	,  _a=False	,  _a=False       )		->							Optional[int]:
           def _flatten(_a       ):
                 return list(itertools.chain(*_a       )       )

           if equal_length:
                 _A							:  List[str]    =		[floats_list((self.max_seq_length, self.feature_size)       ) for _ in range(self.batch_size       )]
           else:
                 # make sure that inputs increase in size
                 _A							:  Union[str, Any]    =		[
                     floats_list((x, self.feature_size)       )
                     for x in range(self.min_seq_length	,  self.max_seq_length	,  self.seq_length_diff       )
                 ]
           if numpify:
                 _A							:  List[Any]    =		[np.asarray(_a       ) for x in speech_inputs]
           return speech_inputs







@require_torch
@require_torchaudio
class 		lowercase							(     UpperCamelCase__,unittest.TestCase	):
     _a											=		TvltFeatureExtractor

     def       a__			(						self       )		->							Any:
           _A							:  int    =		TvltFeatureExtractionTester(self       )

     def       a__			(						self       )		->							List[Any]:
           _A							:  Tuple    =		self.feature_extraction_class(**self.feat_extract_dict       )
           self.assertTrue(hasattr(_a	,  """spectrogram_length"""       )       )
           self.assertTrue(hasattr(_a	,  """feature_size"""       )       )
           self.assertTrue(hasattr(_a	,  """num_audio_channels"""       )       )
           self.assertTrue(hasattr(_a	,  """hop_length"""       )       )
           self.assertTrue(hasattr(_a	,  """chunk_length"""       )       )
           self.assertTrue(hasattr(_a	,  """sampling_rate"""       )       )

     def       a__			(						self       )		->							Optional[int]:
           _A							:  str    =		self.feature_extraction_class(**self.feat_extract_dict       )

           with tempfile.TemporaryDirectory() as tmpdirname:
                 _A							:  Tuple    =		feat_extract_first.save_pretrained(_a       )[0]
                 check_json_file_has_correct_format(_a       )
                 _A							:  Optional[int]    =		self.feature_extraction_class.from_pretrained(_a       )

           _A							:  Optional[Any]    =		feat_extract_first.to_dict()
           _A							:  int    =		feat_extract_second.to_dict()
           _A							:  int    =		dict_first.pop("""mel_filters"""       )
           _A							:  Optional[int]    =		dict_second.pop("""mel_filters"""       )
           self.assertTrue(np.allclose(_a	,  _a       )       )
           self.assertEqual(_a	,  _a       )

     def       a__			(						self       )		->							int:
           _A							:  str    =		self.feature_extraction_class(**self.feat_extract_dict       )

           with tempfile.TemporaryDirectory() as tmpdirname:
                 _A							:  List[str]    =		os.path.join(_a	,  """feat_extract.json"""       )
                 feat_extract_first.to_json_file(_a       )
                 _A							:  Union[str, Any]    =		self.feature_extraction_class.from_json_file(_a       )

           _A							:  Optional[Any]    =		feat_extract_first.to_dict()
           _A							:  Union[str, Any]    =		feat_extract_second.to_dict()
           _A							:  List[Any]    =		dict_first.pop("""mel_filters"""       )
           _A							:  Optional[Any]    =		dict_second.pop("""mel_filters"""       )
           self.assertTrue(np.allclose(_a	,  _a       )       )
           self.assertEqual(_a	,  _a       )

     def       a__			(						self       )		->							Optional[Any]:
           # Initialize feature_extractor
           _A							:  Tuple    =		self.feature_extraction_class(**self.feat_extract_dict       )

           # create three inputs of length 800, 1000, and 1200
           _A							:  Optional[int]    =		[floats_list((1, x)       )[0] for x in range(800	,  1400	,  200       )]
           _A							:  List[str]    =		[np.asarray(_a       ) for speech_input in speech_inputs]

           # Test not batched input
           _A							:  Optional[int]    =		feature_extractor(np_speech_inputs[0]	,  return_tensors="""np"""	,  sampling_rate=4_4100       ).audio_values

           self.assertTrue(encoded_audios.ndim == 4       )
           self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size       )
           self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length       )
           self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels       )

           # Test batched
           _A							:  str    =		feature_extractor(_a	,  return_tensors="""np"""	,  sampling_rate=4_4100       ).audio_values

           self.assertTrue(encoded_audios.ndim == 4       )
           self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size       )
           self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length       )
           self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels       )

           # Test audio masking
           _A							:  Optional[Any]    =		feature_extractor(
               _a	,  return_tensors="""np"""	,  sampling_rate=4_4100	,  mask_audio=_a       ).audio_values

           self.assertTrue(encoded_audios.ndim == 4       )
           self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size       )
           self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length       )
           self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels       )

           # Test 2-D numpy arrays are batched.
           _A							:  Dict    =		[floats_list((1, x)       )[0] for x in (800, 800, 800)]
           _A							:  Optional[int]    =		np.asarray(_a       )
           _A							:  Optional[Any]    =		feature_extractor(_a	,  return_tensors="""np"""	,  sampling_rate=4_4100       ).audio_values
           self.assertTrue(encoded_audios.ndim == 4       )
           self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size       )
           self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length       )
           self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels       )

     def       a__			(						self	,  _a       )		->							str:
           _A							:  Optional[Any]    =		load_dataset("""hf-internal-testing/librispeech_asr_dummy"""	,  """clean"""	,  split="""validation"""       )
           # automatic decoding with librispeech
           _A							:  Dict    =		ds.sort("""id"""       ).select(range(_a       )       )[:num_samples]["""audio"""]

           return [x["array"] for x in speech_samples]



     def       a__			(						self       )		->							Optional[Any]:
           _A							:  List[str]    =		self._load_datasamples(1       )
           _A							:  List[str]    =		TvltFeatureExtractor()
           _A							:  List[Any]    =		feature_extractor(_a	,  return_tensors="""pt"""       ).audio_values

           self.assertEquals(audio_values.shape	,  (1, 1, 192, 128)       )

           _A							:  int    =		torch.tensor([[-0.3032, -0.2708], [-0.4434, -0.4007]]       )
           self.assertTrue(torch.allclose(audio_values[0, 0, :2, :2]	,  _a	,  atol=1e-4       )       )

