


import pytest

from datasets.splits import SplitDict, SplitInfo
from datasets.utils.py_utils import asdict

@pytest.mark.parametrize(
    """split_dict"""					,   [
        SplitDict(),
        SplitDict({"""train""": SplitInfo(name="""train"""					,   num_bytes=1337					,   num_examples=42					,   dataset_name="""my_dataset"""      )}      ),
        SplitDict({"""train""": SplitInfo(name="""train"""					,   num_bytes=1337					,   num_examples=42      )}      ),
        SplitDict({"""train""": SplitInfo()}      ),
    ]					,   )
def 		UpperCamelCase      (lowercase_:				SplitDict      )  ->   Tuple:
	A__							:					Tuple       =	split_dict._to_yaml_list()
	assert len(UpperCamelCase__      ) == len(UpperCamelCase__      )
	A__							:					Optional[Any]       =	SplitDict._from_yaml_list(UpperCamelCase__      )
	for split_name, split_info in split_dict.items():
		# dataset_name field is deprecated, and is therefore not part of the YAML dump
		A__							:					Dict       =	None
		# the split name of split_dict takes over the name of the split info object
		A__							:					Dict       =	split_name
	assert split_dict == reloaded


@pytest.mark.parametrize(
    """split_info"""					,   [SplitInfo(), SplitInfo(dataset_name=UpperCamelCase__      ), SplitInfo(dataset_name="""my_dataset"""      )]      )
def 		UpperCamelCase      (lowercase_:				Optional[int]      )  ->   Tuple:
	# For backward compatibility, we need asdict(split_dict) to return split info dictrionaries with the "dataset_name"
	# field even if it's deprecated. This way old versionso of `datasets` can still reload dataset_infos.json files
	A__							:					Union[str, Any]       =	asdict(SplitDict({"""train""": split_info}      )      )
	assert "dataset_name" in split_dict_asdict["train"]
	assert split_dict_asdict["train"]["dataset_name"] == split_info.dataset_name






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam



class 					_a		(datasets.BeamBasedBuilder       ):


					'''simple docstring'''






					def 						__A    (				self    ):
						return datasets.DatasetInfo(
						    features=datasets.Features({"""content""": datasets.Value("""string"""    )}    )					, supervised_keys=A__					, )






					def 						__A    (				self					, A__					, A__    ):
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN					, gen_kwargs={"""examples""": get_test_dummy_examples()}    )]






					def 						__A    (				self					, A__					, A__    ):
						import apache_beam as beam

						return pipeline | "Load Examples" >> beam.Create(A__    )



class 					_a		(datasets.BeamBasedBuilder       ):


					'''simple docstring'''






					def 						__A    (				self    ):
						return datasets.DatasetInfo(
						    features=datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""    )}    )}    )					, supervised_keys=A__					, )






					def 						__A    (				self					, A__					, A__    ):
						return [
						    datasets.SplitGenerator(name=datasets.Split.TRAIN					, gen_kwargs={"""examples""": get_test_nested_examples()}    )
						]






					def 						__A    (				self					, A__					, A__    ):
						import apache_beam as beam

						return pipeline | "Load Examples" >> beam.Create(A__    )



def 		UpperCamelCase      ()  ->   Dict:
	return [(i, {"content": content}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]      )]

def 		UpperCamelCase      ()  ->   Tuple:
	return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]      )]



class 					_a		(__magic_name__       ):


					'''simple docstring'''






					@require_beam
					def 						__A    (				self    ):
						A__							:					Dict       =	len(get_test_dummy_examples()    )
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					int       =	DummyBeamDataset(cache_dir=A__					, beam_runner="""DirectRunner"""    )
							builder.download_and_prepare()
							self.assertTrue(
							    os.path.exists(
							        os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train.arrow"""    )    )    )
							self.assertDictEqual(builder.info.features					, datasets.Features({"""content""": datasets.Value("""string"""    )}    )    )
							A__							:					int       =	builder.as_dataset()
							self.assertEqual(dset["""train"""].num_rows					, A__    )
							self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					, A__    )
							self.assertDictEqual(dset["""train"""][0]					, get_test_dummy_examples()[0][1]    )
							self.assertDictEqual(
							    dset["""train"""][expected_num_examples - 1]					, get_test_dummy_examples()[expected_num_examples - 1][1]    )
							self.assertTrue(
							    os.path.exists(os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, """dataset_info.json"""    )    )    )
							del dset






					@require_beam
					def 						__A    (				self    ):
						import apache_beam as beam

						A__							:					int       =	beam.io.parquetio.WriteToParquet

						A__							:					List[str]       =	len(get_test_dummy_examples()    )
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					str       =	DummyBeamDataset(cache_dir=A__					, beam_runner="""DirectRunner"""    )
							with patch("""apache_beam.io.parquetio.WriteToParquet"""    ) as write_parquet_mock:
								A__							:					Optional[Any]       =	partial(A__					, num_shards=2    )
								builder.download_and_prepare()
							self.assertTrue(
							    os.path.exists(
							        os.path.join(
							            A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train-00000-of-00002.arrow"""    )    )    )
							self.assertTrue(
							    os.path.exists(
							        os.path.join(
							            A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train-00000-of-00002.arrow"""    )    )    )
							self.assertDictEqual(builder.info.features					, datasets.Features({"""content""": datasets.Value("""string"""    )}    )    )
							A__							:					Optional[int]       =	builder.as_dataset()
							self.assertEqual(dset["""train"""].num_rows					, A__    )
							self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					, A__    )
							# Order is not preserved when sharding, so we just check that all the elements are there
							self.assertListEqual(sorted(dset["""train"""]["""content"""]    )					, sorted(["""foo""", """bar""", """foobar"""]    )    )
							self.assertTrue(
							    os.path.exists(os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, """dataset_info.json"""    )    )    )
							del dset






					@require_beam
					def 						__A    (				self    ):
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					int       =	DummyBeamDataset(cache_dir=A__    )
							self.assertRaises(datasets.builder.MissingBeamOptions					, builder.download_and_prepare    )






					@require_beam
					def 						__A    (				self    ):
						A__							:					List[Any]       =	len(get_test_nested_examples()    )
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					Optional[int]       =	NestedBeamDataset(cache_dir=A__					, beam_runner="""DirectRunner"""    )
							builder.download_and_prepare()
							self.assertTrue(
							    os.path.exists(
							        os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train.arrow"""    )    )    )
							self.assertDictEqual(
							    builder.info.features					, datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""    )}    )}    )    )
							A__							:					Optional[int]       =	builder.as_dataset()
							self.assertEqual(dset["""train"""].num_rows					, A__    )
							self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					, A__    )
							self.assertDictEqual(dset["""train"""][0]					, get_test_nested_examples()[0][1]    )
							self.assertDictEqual(
							    dset["""train"""][expected_num_examples - 1]					, get_test_nested_examples()[expected_num_examples - 1][1]    )
							self.assertTrue(
							    os.path.exists(os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, """dataset_info.json"""    )    )    )
							del dset



