

import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam



class 					lowercase						(  datasets.BeamBasedBuilder ):


							def 						_snake_case							(			self)							->				List[Any]:
								return datasets.DatasetInfo(
								    features=datasets.Features({'content': datasets.Value('string')})    ,			supervised_keys=__snake_case    ,			)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case)							->				Optional[int]:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,			gen_kwargs={'examples': get_test_dummy_examples()})]





							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case)							->				Optional[int]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(__snake_case)



class 					lowercase						(  datasets.BeamBasedBuilder ):


							def 						_snake_case							(			self)							->				List[str]:
								return datasets.DatasetInfo(
								    features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string')})})    ,			supervised_keys=__snake_case    ,			)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case)							->				Optional[Any]:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN    ,			gen_kwargs={'examples': get_test_nested_examples()})
								]





							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case)							->				List[str]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(__snake_case)







def 						SCREAMING_SNAKE_CASE(  )    ->    Any:
	return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']  )]







def 						SCREAMING_SNAKE_CASE(  )    ->    Any:
	return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']  )]







class 					lowercase						(  __SCREAMING_SNAKE_CASE ):


							@require_beam
							def 						_snake_case							(			self)							->				Dict:
								UpperCAmelCase_		:      Any       	=					len(get_test_dummy_examples())
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									UpperCAmelCase_		:      List[str]       	=					DummyBeamDataset(cache_dir=__snake_case    ,			beam_runner='DirectRunner')
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(__snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			F"""{builder.name}-train.arrow""")))
									self.assertDictEqual(builder.info.features    ,			datasets.Features({'content': datasets.Value('string')}))
									UpperCAmelCase_		:      int       	=					builder.as_dataset()
									self.assertEqual(dset['train'].num_rows    ,			__snake_case)
									self.assertEqual(dset['train'].info.splits['train'].num_examples    ,			__snake_case)
									self.assertDictEqual(dset['train'][0]    ,			get_test_dummy_examples()[0][1])
									self.assertDictEqual(
									    dset['train'][expected_num_examples - 1]    ,			get_test_dummy_examples()[expected_num_examples - 1][1])
									self.assertTrue(
									    os.path.exists(os.path.join(__snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			'dataset_info.json')))
									del dset


							@require_beam
							def 						_snake_case							(			self)							->				Optional[Any]:
								import apache_beam as beam

								UpperCAmelCase_		:      Optional[Any]       	=					beam.io.parquetio.WriteToParquet

								UpperCAmelCase_		:      Dict       	=					len(get_test_dummy_examples())
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									UpperCAmelCase_		:      Tuple       	=					DummyBeamDataset(cache_dir=__snake_case    ,			beam_runner='DirectRunner')
									with patch('apache_beam.io.parquetio.WriteToParquet') as write_parquet_mock:
										UpperCAmelCase_		:      Optional[Any]       	=					partial(__snake_case    ,			num_shards=2)
										builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            __snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			F"""{builder.name}-train-00000-of-00002.arrow""")))
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            __snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			F"""{builder.name}-train-00000-of-00002.arrow""")))
									self.assertDictEqual(builder.info.features    ,			datasets.Features({'content': datasets.Value('string')}))
									UpperCAmelCase_		:      int       	=					builder.as_dataset()
									self.assertEqual(dset['train'].num_rows    ,			__snake_case)
									self.assertEqual(dset['train'].info.splits['train'].num_examples    ,			__snake_case)
									# Order is not preserved when sharding, so we just check that all the elements are there
									self.assertListEqual(sorted(dset['train']['content'])    ,			sorted(['foo', 'bar', 'foobar']))
									self.assertTrue(
									    os.path.exists(os.path.join(__snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			'dataset_info.json')))
									del dset


							@require_beam
							def 						_snake_case							(			self)							->				int:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									UpperCAmelCase_		:      Any       	=					DummyBeamDataset(cache_dir=__snake_case)
									self.assertRaises(datasets.builder.MissingBeamOptions    ,			builder.download_and_prepare)





							@require_beam
							def 						_snake_case							(			self)							->				int:
								UpperCAmelCase_		:      int       	=					len(get_test_nested_examples())
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									UpperCAmelCase_		:      Tuple       	=					NestedBeamDataset(cache_dir=__snake_case    ,			beam_runner='DirectRunner')
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(__snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			F"""{builder.name}-train.arrow""")))
									self.assertDictEqual(
									    builder.info.features    ,			datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string')})}))
									UpperCAmelCase_		:      Tuple       	=					builder.as_dataset()
									self.assertEqual(dset['train'].num_rows    ,			__snake_case)
									self.assertEqual(dset['train'].info.splits['train'].num_examples    ,			__snake_case)
									self.assertDictEqual(dset['train'][0]    ,			get_test_nested_examples()[0][1])
									self.assertDictEqual(
									    dset['train'][expected_num_examples - 1]    ,			get_test_nested_examples()[expected_num_examples - 1][1])
									self.assertTrue(
									    os.path.exists(os.path.join(__snake_case    ,			builder.name    ,			'default'    ,			'0.0.0'    ,			'dataset_info.json')))
									del dset









'''simple docstring'''


import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer



class 					lowercase						(  a_ ):
							_lowerCamelCase		:      List[str]=   "Wav2Vec2FeatureExtractor"
							_lowerCamelCase		:      Any=   "AutoTokenizer"


							def __init__(			self    ,			_snake_case    ,			_snake_case)							->				List[str]:
								super().__init__(_snake_case    ,			_snake_case)
								UpperCAmelCase_		:      int       	=					self.feature_extractor
								UpperCAmelCase_		:      str       	=					False


							@classmethod
							def 						_snake_case							(			cls    ,			_snake_case    ,			**_snake_case)							->				List[Any]:
								try:
									return super().from_pretrained(_snake_case    ,			**_snake_case)
								except OSError:
									warnings.warn(
									    F"""Loading a tokenizer inside {cls.__name__} from a config that does not"""
									    ' include a `tokenizer_class` attribute is deprecated and will be '
									    'removed in v5. Please add `\'tokenizer_class\': \'Wav2Vec2CTCTokenizer\'`'
									    ' attribute to either your `config.json` or `tokenizer_config.json` '
									    'file to suppress this warning: '    ,			_snake_case    ,			)

									UpperCAmelCase_		:      Any       	=					WavaVecaFeatureExtractor.from_pretrained(_snake_case    ,			**_snake_case)
									UpperCAmelCase_		:      List[str]       	=					WavaVecaCTCTokenizer.from_pretrained(_snake_case    ,			**_snake_case)

									return cls(feature_extractor=_snake_case    ,			tokenizer=_snake_case)


							def __call__(			self    ,			*_snake_case    ,			**_snake_case)							->				Optional[Any]:
								# For backward compatibility
								if self._in_target_context_manager:
									return self.current_processor(*_snake_case    ,			**_snake_case)

								if "raw_speech" in kwargs:
									warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.')
									UpperCAmelCase_		:      List[str]       	=					kwargs.pop('raw_speech')
								else:
									UpperCAmelCase_		:      Dict       	=					kwargs.pop('audio'    ,			_snake_case)
								UpperCAmelCase_		:      str       	=					kwargs.pop('sampling_rate'    ,			_snake_case)
								UpperCAmelCase_		:      List[str]       	=					kwargs.pop('text'    ,			_snake_case)
								if len(_snake_case) > 0:
									UpperCAmelCase_		:      List[str]       	=					args[0]
									UpperCAmelCase_		:      Tuple       	=					args[1:]

								if audio is None and text is None:
									raise ValueError('You need to specify either an `audio` or `text` input to process.')

								if audio is not None:
									UpperCAmelCase_		:      Union[str, Any]       	=					self.feature_extractor(_snake_case    ,			*_snake_case    ,			sampling_rate=_snake_case    ,			**_snake_case)
								if text is not None:
									UpperCAmelCase_		:      Optional[int]       	=					self.tokenizer(_snake_case    ,			**_snake_case)

								if text is None:
									return inputs
								elif audio is None:
									return encodings
								else:
									UpperCAmelCase_		:      List[str]       	=					encodings['input_ids']
									return inputs


							def 						_snake_case							(			self    ,			*_snake_case    ,			**_snake_case)							->				str:
								# For backward compatibility
								if self._in_target_context_manager:
									return self.current_processor.pad(*_snake_case    ,			**_snake_case)

								UpperCAmelCase_		:      Union[str, Any]       	=					kwargs.pop('input_features'    ,			_snake_case)
								UpperCAmelCase_		:      Dict       	=					kwargs.pop('labels'    ,			_snake_case)
								if len(_snake_case) > 0:
									UpperCAmelCase_		:      Optional[Any]       	=					args[0]
									UpperCAmelCase_		:      int       	=					args[1:]

								if input_features is not None:
									UpperCAmelCase_		:      Dict       	=					self.feature_extractor.pad(_snake_case    ,			*_snake_case    ,			**_snake_case)
								if labels is not None:
									UpperCAmelCase_		:      Union[str, Any]       	=					self.tokenizer.pad(_snake_case    ,			**_snake_case)

								if labels is None:
									return input_features
								elif input_features is None:
									return labels
								else:
									UpperCAmelCase_		:      Union[str, Any]       	=					labels['input_ids']
									return input_features


							def 						_snake_case							(			self    ,			*_snake_case    ,			**_snake_case)							->				Dict:
								return self.tokenizer.batch_decode(*_snake_case    ,			**_snake_case)


							def 						_snake_case							(			self    ,			*_snake_case    ,			**_snake_case)							->				Tuple:
								return self.tokenizer.decode(*_snake_case    ,			**_snake_case)





							@contextmanager
							def 						_snake_case							(			self)							->				Tuple:
								warnings.warn(
								    '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
								    'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
								    'your audio inputs, or in a separate call.')
								UpperCAmelCase_		:      Optional[Any]       	=					True
								UpperCAmelCase_		:      Tuple       	=					self.tokenizer
								yield
								UpperCAmelCase_		:      Tuple       	=					self.feature_extractor
								UpperCAmelCase_		:      Tuple       	=					False






