import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class      __lowercase					(    datasets.BeamBasedBuilder    ):

						def        __a		(	self				:      Tuple   )							->		Tuple:


									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'''content''': datasets.Value('''string'''   )}   )							,							supervised_keys=__lowerCamelCase							,							)

						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Union[str, Any]							,							__lowerCamelCase				:      Tuple   )							->		int:


									'''simple docstring'''

									return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'''examples''': get_test_dummy_examples()}   )]



						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      Any							,							__lowerCamelCase				:      Union[str, Any]   )							->		Optional[int]:


									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(__lowerCamelCase   )






class      __lowercase					(    datasets.BeamBasedBuilder    ):

						def        __a		(	self				:      int   )							->		Any:


									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''   )}   )}   )							,							supervised_keys=__lowerCamelCase							,							)

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Optional[int]   )							->		int:


									'''simple docstring'''

									return [
									    datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'''examples''': get_test_nested_examples()}   )
									]



						def        __a		(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Tuple							,							__lowerCamelCase				:      Union[str, Any]   )							->		Tuple:


									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(__lowerCamelCase   )






def        __UpperCAmelCase    (       )->       Dict:



			"""simple docstring"""







			return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']       )]






def        __UpperCAmelCase    (       )->       List[str]:



			"""simple docstring"""







			return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']       )]






class      __lowercase					(    _A    ):

						@require_beam
						def        __a		(	self				:      Optional[int]   )							->		Optional[Any]:


									'''simple docstring'''

									lowercase							=		len(get_test_dummy_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train.arrow'   )   )   )
												self.assertDictEqual(builder.info.features							,							datasets.Features({'''content''': datasets.Value('''string'''   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												self.assertDictEqual(dset['''train'''][0]							,							get_test_dummy_examples()[0][1]   )
												self.assertDictEqual(
												    dset['''train'''][expected_num_examples - 1]							,							get_test_dummy_examples()[expected_num_examples - 1][1]   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset

						@require_beam
						def        __a		(	self				:      Union[str, Any]   )							->		List[Any]:


									'''simple docstring'''

									import apache_beam as beam

									lowercase							=		beam.io.parquetio.WriteToParquet

									lowercase							=		len(get_test_dummy_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												with patch('''apache_beam.io.parquetio.WriteToParquet'''   ) as write_parquet_mock:
															lowercase							=		partial(__lowerCamelCase							,							num_shards=2   )
															builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(
												            __lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train-00000-of-00002.arrow'   )   )   )
												self.assertTrue(
												    os.path.exists(
												        os.path.join(
												            __lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train-00000-of-00002.arrow'   )   )   )
												self.assertDictEqual(builder.info.features							,							datasets.Features({'''content''': datasets.Value('''string'''   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												# Order is not preserved when sharding, so we just check that all the elements are there
												self.assertListEqual(sorted(dset['''train''']['''content''']   )							,							sorted(['''foo''', '''bar''', '''foobar''']   )   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset

						@require_beam
						def        __a		(	self				:      int   )							->		int:


									'''simple docstring'''

									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		DummyBeamDataset(cache_dir=__lowerCamelCase   )
												self.assertRaises(datasets.builder.MissingBeamOptions							,							builder.download_and_prepare   )



						@require_beam
						def        __a		(	self				:      str   )							->		List[Any]:


									'''simple docstring'''

									lowercase							=		len(get_test_nested_examples()   )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
												lowercase							=		NestedBeamDataset(cache_dir=__lowerCamelCase							,							beam_runner='''DirectRunner'''   )
												builder.download_and_prepare()
												self.assertTrue(
												    os.path.exists(
												        os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							f'{builder.name}-train.arrow'   )   )   )
												self.assertDictEqual(
												    builder.info.features							,							datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''   )}   )}   )   )
												lowercase							=		builder.as_dataset()
												self.assertEqual(dset['''train'''].num_rows							,							__lowerCamelCase   )
												self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples							,							__lowerCamelCase   )
												self.assertDictEqual(dset['''train'''][0]							,							get_test_nested_examples()[0][1]   )
												self.assertDictEqual(
												    dset['''train'''][expected_num_examples - 1]							,							get_test_nested_examples()[expected_num_examples - 1][1]   )
												self.assertTrue(
												    os.path.exists(os.path.join(__lowerCamelCase							,							builder.name							,							'''default'''							,							'''0.0.0'''							,							'''dataset_info.json'''   )   )   )
												del dset






from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

A_      =      TypeVar("KEY")
A_      =      TypeVar("VAL")






@dataclass(frozen=_A       ,     slots=_A    )
class      __lowercase					(    Generic[KEY, VAL]    ):
						lowercase					 =     42
						lowercase					 =     42






class      __lowercase					(    _Item    ):

						def __init__(	self				:      Optional[int]   )							->		None:


									'''simple docstring'''

									super().__init__(__lowerCamelCase							,							__lowerCamelCase   )



						def __bool__(	self				:      str   )							->		bool:


									'''simple docstring'''

									return False





A_      =      _DeletedItem()






class      __lowercase					(    MutableMapping[KEY, VAL]    ):

						def __init__(	self				:      Any							,							__lowerCamelCase				:      int = 8							,							__lowerCamelCase				:      float = 0.75   )							->		None:


									'''simple docstring'''

									lowercase							=		initial_block_size
									lowercase							=		[None] * initial_block_size
									assert 0.0 < capacity_factor < 1.0
									lowercase							=		capacity_factor
									lowercase							=		0

						def        __a		(	self				:      Optional[Any]							,							__lowerCamelCase				:      KEY   )							->		int:


									'''simple docstring'''

									return hash(__lowerCamelCase   ) % len(self._buckets   )

						def        __a		(	self				:      List[Any]							,							__lowerCamelCase				:      int   )							->		int:


									'''simple docstring'''

									return (ind + 1) % len(self._buckets   )

						def        __a		(	self				:      Optional[Any]							,							__lowerCamelCase				:      int							,							__lowerCamelCase				:      KEY							,							__lowerCamelCase				:      VAL   )							->		bool:


									'''simple docstring'''

									lowercase							=		self._buckets[ind]
									if not stored:
												lowercase							=		_Item(__lowerCamelCase							,							__lowerCamelCase   )
												self._len += 1
												return True
									elif stored.key == key:
												lowercase							=		_Item(__lowerCamelCase							,							__lowerCamelCase   )
												return True
									else:
												return False

						def        __a		(	self				:      Optional[Any]   )							->		bool:


									'''simple docstring'''

									lowercase							=		len(self._buckets   ) * self._capacity_factor
									return len(self   ) >= int(__lowerCamelCase   )

						def        __a		(	self				:      Tuple   )							->		bool:


									'''simple docstring'''

									if len(self._buckets   ) <= self._initial_block_size:
												return False
									lowercase							=		len(self._buckets   ) * self._capacity_factor / 2
									return len(self   ) < limit

						def        __a		(	self				:      Optional[Any]							,							__lowerCamelCase				:      int   )							->		None:


									'''simple docstring'''

									lowercase							=		self._buckets
									lowercase							=		[None] * new_size
									lowercase							=		0
									for item in old_buckets:
												if item:
															self._add_item(item.key							,							item.val   )

						def        __a		(	self				:      List[str]   )							->		None:


									'''simple docstring'''

									self._resize(len(self._buckets   ) * 2   )

						def        __a		(	self				:      Any   )							->		None:


									'''simple docstring'''

									self._resize(len(self._buckets   ) // 2   )

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      KEY   )							->		Iterator[int]:


									'''simple docstring'''

									lowercase							=		self._get_bucket_index(__lowerCamelCase   )
									for _ in range(len(self._buckets   )   ):
												yield ind
												lowercase							=		self._get_next_ind(__lowerCamelCase   )

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      KEY							,							__lowerCamelCase				:      VAL   )							->		None:


									'''simple docstring'''

									for ind in self._iterate_buckets(__lowerCamelCase   ):
												if self._try_set(__lowerCamelCase							,							__lowerCamelCase							,							__lowerCamelCase   ):
															break

						def __setitem__(	self				:      Optional[int]							,							__lowerCamelCase				:      KEY							,							__lowerCamelCase				:      VAL   )							->		None:


									'''simple docstring'''

									if self._is_full():
												self._size_up()

									self._add_item(__lowerCamelCase							,							__lowerCamelCase   )

						def __delitem__(	self				:      List[str]							,							__lowerCamelCase				:      KEY   )							->		None:


									'''simple docstring'''

									for ind in self._iterate_buckets(__lowerCamelCase   ):
												lowercase							=		self._buckets[ind]
												if item is None:
															raise KeyError(__lowerCamelCase   )
												if item is _deleted:
															continue
												if item.key == key:
															lowercase							=		_deleted
															self._len -= 1
															break
									if self._is_sparse():
												self._size_down()

						def __getitem__(	self				:      Any							,							__lowerCamelCase				:      KEY   )							->		VAL:


									'''simple docstring'''

									for ind in self._iterate_buckets(__lowerCamelCase   ):
												lowercase							=		self._buckets[ind]
												if item is None:
															break
												if item is _deleted:
															continue
												if item.key == key:
															return item.val
									raise KeyError(__lowerCamelCase   )

						def __len__(	self				:      str   )							->		int:


									'''simple docstring'''

									return self._len

						def __iter__(	self				:      Union[str, Any]   )							->		Iterator[KEY]:


									'''simple docstring'''

									yield from (item.key for item in self._buckets if item)



						def __repr__(	self				:      Any   )							->		str:


									'''simple docstring'''

									lowercase							=		''' ,'''.join(
									    f'{item.key}: {item.val}' for item in self._buckets if item   )
									return f'HashMap({val_string})'





