




import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_snake_case            = datasets.utils.logging.get_logger(__name__)







@dataclass
class 		lowercase							(     datasets.BuilderConfig	):
     _a											=		1_0_0_0_0
     _a											=		None
     _a											=		None







class 		lowercase							(     datasets.ArrowBasedBuilder	):
     _a											=		ParquetConfig

     def       a__			(						self       )		->							Optional[int]:
           return datasets.DatasetInfo(features=self.config.features       )

     def       a__			(						self	,  _a       )		->							Any:
           if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''       )
           _A							:  str    =		dl_manager.download_and_extract(self.config.data_files       )
           if isinstance(_a	,  (str, list, tuple)       ):
                 _A							:  Optional[Any]    =		data_files
                 if isinstance(_a	,  _a       ):
                       _A							:  List[str]    =		[files]
                 # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                 _A							:  Optional[Any]    =		[dl_manager.iter_files(_a       ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,  gen_kwargs={"""files""": files}       )]
           _A							:  List[str]    =		[]
           for split_name, files in data_files.items():
                 if isinstance(_a	,  _a       ):
                       _A							:  int    =		[files]
                 # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                 _A							:  Dict    =		[dl_manager.iter_files(_a       ) for file in files]
                 # Infer features is they are stoed in the arrow schema
                 if self.info.features is None:
                       for file in itertools.chain.from_iterable(_a       ):
                             with open(_a	,  """rb"""       ) as f:
                                   _A							:  int    =		datasets.Features.from_arrow_schema(pq.read_schema(_a       )       )
                             break
                 splits.append(datasets.SplitGenerator(name=_a	,  gen_kwargs={"""files""": files}       )       )
           return splits

     def       a__			(						self	,  _a       )		->							pa.Table:
           if self.info.features is not None:
                 # more expensive cast to support nested features with keys in a different order
                 # allows str <-> int/float or str to Audio for example
                 _A							:  List[Any]    =		table_cast(_a	,  self.info.features.arrow_schema       )
           return pa_table



     def       a__			(						self	,  _a       )		->							Tuple:
           _A							:  Tuple    =		self.info.features.arrow_schema if self.info.features is not None else None
           if self.info.features is not None and self.config.columns is not None:
                 if sorted(field.name for field in schema       ) != sorted(self.config.columns       ):
                       raise ValueError(
                           F'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''       )
           for file_idx, file in enumerate(itertools.chain.from_iterable(_a       )       ):
                 with open(_a	,  """rb"""       ) as f:
                       _A							:  Union[str, Any]    =		pq.ParquetFile(_a       )
                       try:
                             for batch_idx, record_batch in enumerate(
                                 parquet_file.iter_batches(batch_size=self.config.batch_size	,  columns=self.config.columns       )       ):
                                   _A							:  Optional[int]    =		pa.Table.from_batches([record_batch]       )
                                   # Uncomment for debugging (will print the Arrow table size and elements)
                                   # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                   # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                   yield F'''{file_idx}_{batch_idx}''', self._cast_table(_a       )
                       except ValueError as e:
                             logger.error(F'''Failed to read file \'{file}\' with error {type(_a       )}: {e}'''       )
                             raise







import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
  from transformers import WhisperFeatureExtractor

if is_torch_available():
  import torch

_snake_case            = random.Random()







def        lowerCAmelCase_			(  snake_case_,snake_case_=1.0,snake_case_=None,snake_case_=None  ):
      if rng is None:
            _A							:  str    =		global_rng

      _A							:  List[Any]    =		[]
      for batch_idx in range(shape[0]  ):
            values.append([]  )
            for _ in range(shape[1]  ):
                  values[-1].append(rng.random() * scale  )

      return values







@require_torch
@require_torchaudio
class 		lowercase							(     unittest.TestCase	):

     def __init__(						self	,  _a	,  _a=7	,  _a=400	,  _a=2000	,  _a=10	,  _a=160	,  _a=8	,  _a=0.0	,  _a=4000	,  _a=False	,  _a=True	,  )		->							Optional[int]:
           _A							:  Any    =		parent
           _A							:  List[Any]    =		batch_size
           _A							:  List[Any]    =		min_seq_length
           _A							:  Dict    =		max_seq_length
           _A							:  Optional[Any]    =		(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
           _A							:  Tuple    =		padding_value
           _A							:  Tuple    =		sampling_rate
           _A							:  str    =		return_attention_mask
           _A							:  Any    =		do_normalize
           _A							:  Union[str, Any]    =		feature_size
           _A							:  List[Any]    =		chunk_length
           _A							:  List[Any]    =		hop_length

     def       a__			(						self       )		->							List[str]:
           return {
               "feature_size": self.feature_size,
               "hop_length": self.hop_length,
               "chunk_length": self.chunk_length,
               "padding_value": self.padding_value,
               "sampling_rate": self.sampling_rate,
               "return_attention_mask": self.return_attention_mask,
               "do_normalize": self.do_normalize,
           }



     def       a__			(						self	,  _a=False	,  _a=False       )		->							List[str]:
           def _flatten(_a       ):
                 return list(itertools.chain(*_a       )       )

           if equal_length:
                 _A							:  Union[str, Any]    =		[floats_list((self.max_seq_length, self.feature_size)       ) for _ in range(self.batch_size       )]
           else:
                 # make sure that inputs increase in size
                 _A							:  int    =		[
                     floats_list((x, self.feature_size)       )
                     for x in range(self.min_seq_length	,  self.max_seq_length	,  self.seq_length_diff       )
                 ]
           if numpify:
                 _A							:  Any    =		[np.asarray(_a       ) for x in speech_inputs]
           return speech_inputs







@require_torch
@require_torchaudio
class 		lowercase							(     UpperCamelCase__,unittest.TestCase	):
     _a											=		WhisperFeatureExtractor if is_speech_available() else None

     def       a__			(						self       )		->							Tuple:
           _A							:  Optional[int]    =		WhisperFeatureExtractionTester(self       )

     def       a__			(						self       )		->							Optional[Any]:
           _A							:  List[Any]    =		self.feature_extraction_class(**self.feat_extract_dict       )

           with tempfile.TemporaryDirectory() as tmpdirname:
                 _A							:  List[str]    =		feat_extract_first.save_pretrained(_a       )[0]
                 check_json_file_has_correct_format(_a       )
                 _A							:  Optional[int]    =		self.feature_extraction_class.from_pretrained(_a       )

           _A							:  Tuple    =		feat_extract_first.to_dict()
           _A							:  List[Any]    =		feat_extract_second.to_dict()
           _A							:  List[Any]    =		feat_extract_first.mel_filters
           _A							:  Union[str, Any]    =		feat_extract_second.mel_filters
           self.assertTrue(np.allclose(_a	,  _a       )       )
           self.assertEqual(_a	,  _a       )

     def       a__			(						self       )		->							Dict:
           _A							:  Any    =		self.feature_extraction_class(**self.feat_extract_dict       )

           with tempfile.TemporaryDirectory() as tmpdirname:
                 _A							:  Dict    =		os.path.join(_a	,  """feat_extract.json"""       )
                 feat_extract_first.to_json_file(_a       )
                 _A							:  Optional[int]    =		self.feature_extraction_class.from_json_file(_a       )

           _A							:  str    =		feat_extract_first.to_dict()
           _A							:  Any    =		feat_extract_second.to_dict()
           _A							:  Union[str, Any]    =		feat_extract_first.mel_filters
           _A							:  Union[str, Any]    =		feat_extract_second.mel_filters
           self.assertTrue(np.allclose(_a	,  _a       )       )
           self.assertEqual(_a	,  _a       )

     def       a__			(						self       )		->							Union[str, Any]:
           # Tests that all call wrap to encode_plus and batch_encode_plus
           _A							:  Optional[int]    =		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()       )
           # create three inputs of length 800, 1000, and 1200
           _A							:  Optional[Any]    =		[floats_list((1, x)       )[0] for x in range(800	,  1400	,  200       )]
           _A							:  Any    =		[np.asarray(_a       ) for speech_input in speech_inputs]

           # Test feature size
           _A							:  Dict    =		feature_extractor(_a	,  padding="""max_length"""	,  return_tensors="""np"""       ).input_features
           self.assertTrue(input_features.ndim == 3       )
           self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames       )
           self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size       )

           # Test not batched input
           _A							:  List[Any]    =		feature_extractor(speech_inputs[0]	,  return_tensors="""np"""       ).input_features
           _A							:  List[str]    =		feature_extractor(np_speech_inputs[0]	,  return_tensors="""np"""       ).input_features
           self.assertTrue(np.allclose(_a	,  _a	,  atol=1e-3       )       )

           # Test batched
           _A							:  Union[str, Any]    =		feature_extractor(_a	,  return_tensors="""np"""       ).input_features
           _A							:  Tuple    =		feature_extractor(_a	,  return_tensors="""np"""       ).input_features
           for enc_seq_a, enc_seq_a in zip(_a	,  _a       ):
                 self.assertTrue(np.allclose(_a	,  _a	,  atol=1e-3       )       )

           # Test 2-D numpy arrays are batched.
           _A							:  List[str]    =		[floats_list((1, x)       )[0] for x in (800, 800, 800)]
           _A							:  Any    =		np.asarray(_a       )
           _A							:  Union[str, Any]    =		feature_extractor(_a	,  return_tensors="""np"""       ).input_features
           _A							:  int    =		feature_extractor(_a	,  return_tensors="""np"""       ).input_features
           for enc_seq_a, enc_seq_a in zip(_a	,  _a       ):
                 self.assertTrue(np.allclose(_a	,  _a	,  atol=1e-3       )       )

           # Test truncation required
           _A							:  List[Any]    =		[floats_list((1, x)       )[0] for x in range(200	,  (feature_extractor.n_samples + 500)	,  200       )]
           _A							:  Union[str, Any]    =		[np.asarray(_a       ) for speech_input in speech_inputs]

           _A							:  Tuple    =		[x[: feature_extractor.n_samples] for x in speech_inputs]
           _A							:  Union[str, Any]    =		[np.asarray(_a       ) for speech_input in speech_inputs_truncated]

           _A							:  Optional[int]    =		feature_extractor(_a	,  return_tensors="""np"""       ).input_features
           _A							:  List[Any]    =		feature_extractor(_a	,  return_tensors="""np"""       ).input_features
           for enc_seq_a, enc_seq_a in zip(_a	,  _a       ):
                 self.assertTrue(np.allclose(_a	,  _a	,  atol=1e-3       )       )

     def       a__			(						self       )		->							Dict:
           import torch

           _A							:  Dict    =		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()       )
           _A							:  Optional[int]    =		np.random.rand(100	,  32       ).astype(np.floataa       )
           _A							:  str    =		np_speech_inputs.tolist()

           for inputs in [py_speech_inputs, np_speech_inputs]:
                 _A							:  Optional[Any]    =		feature_extractor.pad([{"""input_features""": inputs}]	,  return_tensors="""np"""       )
                 self.assertTrue(np_processed.input_features.dtype == np.floataa       )
                 _A							:  Optional[int]    =		feature_extractor.pad([{"""input_features""": inputs}]	,  return_tensors="""pt"""       )
                 self.assertTrue(pt_processed.input_features.dtype == torch.floataa       )

     def       a__			(						self	,  _a       )		->							Dict:
           _A							:  Optional[Any]    =		load_dataset("""hf-internal-testing/librispeech_asr_dummy"""	,  """clean"""	,  split="""validation"""       )
           # automatic decoding with librispeech
           _A							:  Optional[Any]    =		ds.sort("""id"""       ).select(range(_a       )       )[:num_samples]["""audio"""]

           return [x["array"] for x in speech_samples]

     def       a__			(						self       )		->							Tuple:
           # fmt: off
           _A							:  Dict    =		torch.tensor(
               [
                   0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
                   0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
                   0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
                   -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
               ]       )
           # fmt: on

           _A							:  Dict    =		self._load_datasamples(1       )
           _A							:  Optional[Any]    =		WhisperFeatureExtractor()
           _A							:  Optional[Any]    =		feature_extractor(_a	,  return_tensors="""pt"""       ).input_features
           self.assertEqual(input_features.shape	,  (1, 80, 3000)       )
           self.assertTrue(torch.allclose(input_features[0, 0, :30]	,  _a	,  atol=1e-4       )       )



     def       a__			(						self       )		->							str:
           _A							:  Dict    =		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()       )
           _A							:  str    =		self._load_datasamples(1       )[0]
           _A							:  Union[str, Any]    =		((audio - audio.min()) / (audio.max() - audio.min())) * 6_5535  # Rescale to [0, 65535] to show issue
           _A							:  List[Any]    =		feat_extract.zero_mean_unit_var_norm([audio]	,  attention_mask=_a       )[0]

           self.assertTrue(np.all(np.mean(_a       ) < 1e-3       )       )
           self.assertTrue(np.all(np.abs(np.var(_a       ) - 1       ) < 1e-3       )       )

