# flake8: noqa
# Lint as: python3

from typing import Dict, List, Optional, Type

from .. import config
from ..utils import logging
from .formatting import (
    ArrowFormatter,
    CustomFormatter,
    Formatter,
    PandasFormatter,
    PythonFormatter,
    TensorFormatter,
    format_table,
    query_table,
)
from .np_formatter import NumpyFormatter


A_      =      logging.get_logger(__name__)

A_      =      {}
A_      =      {}
A_      =      {}






def        __UpperCAmelCase    (       UpperCAmelCase,		UpperCAmelCase,		UpperCAmelCase = None,		)->       List[Any]:



			"""simple docstring"""







			lowercase							=		aliases if aliases is not None else []
			if format_type in _FORMAT_TYPES:
						logger.warning(
						    f'Overwriting format type \'{format_type}\' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})'       )
			lowercase							=		formatter_cls
			for alias in set(aliases + [format_type]       ):
						if alias in _FORMAT_TYPES_ALIASES:
									logger.warning(
									    f'Overwriting format type alias \'{alias}\' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})'       )
						lowercase							=		format_type






def        __UpperCAmelCase    (       UpperCAmelCase,		UpperCAmelCase,		UpperCAmelCase = None       )->       Tuple:



			"""simple docstring"""







			lowercase							=		aliases if aliases is not None else []
			for alias in set(aliases + [format_type]       ):
						lowercase							=		unavailable_error


# Here we define all the available formatting functions that can be used by `Dataset.set_format`
_register_formatter(PythonFormatter, None, aliases=["python"])
_register_formatter(ArrowFormatter, "arrow", aliases=["pa", "pyarrow"])
_register_formatter(NumpyFormatter, "numpy", aliases=["np"])
_register_formatter(PandasFormatter, "pandas", aliases=["pd"])
_register_formatter(CustomFormatter, "custom")

if config.TORCH_AVAILABLE:
						from .torch_formatter import TorchFormatter

						_register_formatter(TorchFormatter, "torch", aliases=["pt", "pytorch"])
else:
						A_      =      ValueError("PyTorch needs to be installed to be able to return PyTorch tensors.")
						_register_unavailable_formatter(_torch_error, "torch", aliases=["pt", "pytorch"])

if config.TF_AVAILABLE:
						from .tf_formatter import TFFormatter

						_register_formatter(TFFormatter, "tensorflow", aliases=["tf"])
else:
						A_      =      ValueError("Tensorflow needs to be installed to be able to return Tensorflow tensors.")
						_register_unavailable_formatter(_tf_error, "tensorflow", aliases=["tf"])

if config.JAX_AVAILABLE:
						from .jax_formatter import JaxFormatter

						_register_formatter(JaxFormatter, "jax", aliases=[])
else:
						A_      =      ValueError("JAX needs to be installed to be able to return JAX arrays.")
						_register_unavailable_formatter(_jax_error, "jax", aliases=[])






def        __UpperCAmelCase    (       UpperCAmelCase       )->       Optional[str]:



			"""simple docstring"""







			if format_type in _FORMAT_TYPES_ALIASES:
						return _FORMAT_TYPES_ALIASES[format_type]
			else:
						return format_type






def        __UpperCAmelCase    (       UpperCAmelCase,		**UpperCAmelCase       )->       Formatter:



			"""simple docstring"""







			lowercase							=		get_format_type_from_alias(UpperCAmelCase       )
			if format_type in _FORMAT_TYPES:
						return _FORMAT_TYPES[format_type](**UpperCAmelCase       )
			if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
						raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
			else:
						raise ValueError(
						    f'Return type should be None or selected in {list(type for type in _FORMAT_TYPES.keys() if type != None       )}, but got \'{format_type}\''       )






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class      __lowercase					(    datasets.BeamBasedBuilder    ):

						def        __a		(	self				:      Tuple   )							->		Tuple:


									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'''content''': datasets.Value('''string'''   )}   )							,							supervised_keys=__lowerCamelCase							,							)

						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      Tuple   )							->		int:


									'''simple docstring'''

									return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'''examples''': get_test_dummy_examples()}   )]



						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Union[str, Any]   )							->		Optional[int]:


									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(__lowerCamelCase   )






class      __lowercase					(    datasets.BeamBasedBuilder    ):

						def        __a		(	self				:      int   )							->		Any:


									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''   )}   )}   )							,							supervised_keys=__lowerCamelCase							,							)

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Optional[int]   )							->		int:


									'''simple docstring'''

									return [
									    datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'''examples''': get_test_nested_examples()}   )
									]



						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      Union[str, Any]   )							->		Tuple:


									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(__lowerCamelCase   )






def        __UpperCAmelCase    (       )->       Dict:



			"""simple docstring"""







			return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']       )]






def        __UpperCAmelCase    (       )->       List[str]:



			"""simple docstring"""







			return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']       )]






class      __lowercase					(    _A    ):

						@require_beam
						def        __a		(	self				:      Optional[int]   )							->		Optional[Any]:


									'''simple docstring'''

									lowercase							=		len(get_test_dummy_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train.arrow'   )   )   )
												self.assertDictEqual(builder.info.features							,							datasets.Features({'''content''': datasets.Value('''string'''   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												self.assertDictEqual(dset['''train'''][0]							,							get_test_dummy_examples()[0][1]   )
												self.assertDictEqual(
												    dset['''train'''][expected_num_examples - 1]							,							get_test_dummy_examples()[expected_num_examples - 1][1]   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset

						@require_beam
						def        __a		(	self				:      Union[str, Any]   )							->		List[Any]:


									'''simple docstring'''

									import apache_beam as beam

									lowercase							=		beam.io.parquetio.WriteToParquet

									lowercase							=		len(get_test_dummy_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												with patch('''apache_beam.io.parquetio.WriteToParquet'''   ) as write_parquet_mock:
															lowercase							=		partial(__lowerCamelCase							,							num_shards=2   )
															builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(
												            __lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train-00000-of-00002.arrow'   )   )   )
												self.assertTrue(
												    os.path.exists(
												        os.path.join(
												            __lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train-00000-of-00002.arrow'   )   )   )
												self.assertDictEqual(builder.info.features							,							datasets.Features({'''content''': datasets.Value('''string'''   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												# Order is not preserved when sharding, so we just check that all the elements are there
												self.assertListEqual(sorted(dset['''train''']['''content''']   )							,							sorted(['''foo''', '''bar''', '''foobar''']   )   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset

						@require_beam
						def        __a		(	self				:      int   )							->		int:


									'''simple docstring'''

									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase   )
												self.assertRaises(datasets.builder.MissingBeamOptions							,							builder.download_and_prepare   )



						@require_beam
						def        __a		(	self				:      str   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		len(get_test_nested_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		NestedBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train.arrow'   )   )   )
												self.assertDictEqual(
												    builder.info.features							,							datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''   )}   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												self.assertDictEqual(dset['''train'''][0]							,							get_test_nested_examples()[0][1]   )
												self.assertDictEqual(
												    dset['''train'''][expected_num_examples - 1]							,							get_test_nested_examples()[expected_num_examples - 1][1]   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset





