






'''simple docstring'''

# flake8: noqa
# Lint as: python3

from typing import Dict, List, Optional, Type

from .. import config
from ..utils import logging
from .formatting import (
    ArrowFormatter,
    CustomFormatter,
    Formatter,
    PandasFormatter,
    PythonFormatter,
    TensorFormatter,
    format_table,
    query_table,
)
from .np_formatter import NumpyFormatter


UpperCAmelCase__						:Optional[int]          =			logging.get_logger(__name__)

UpperCAmelCase__						:Dict[Optional[str], Type[Formatter]]          =			{}
UpperCAmelCase__						:Dict[Optional[str], str]          =			{}
UpperCAmelCase__						:Dict[Optional[str], Exception]          =			{}
def     __lowercase  (_lowercase,						_lowercase,						_lowercase = None,						)   ->     List[Any]:


     """simple docstring"""
     __lowerCamelCase						:       Optional[Any]				   =				aliases if aliases is not None else []
     if format_type in _FORMAT_TYPES:
          logger.warning(
              f"Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})"     )
     __lowerCamelCase						:       int				   =				formatter_cls
     for alias in set(aliases + [format_type]     ):
          if alias in _FORMAT_TYPES_ALIASES:
               logger.warning(
                   f"Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})"     )
          __lowerCamelCase						:       Union[str, Any]				   =				format_type
def     __lowercase  (_lowercase,						_lowercase,						_lowercase = None     )   ->     List[Any]:


     """simple docstring"""
     __lowerCamelCase						:       str				   =				aliases if aliases is not None else []
     for alias in set(aliases + [format_type]     ):
          __lowerCamelCase						:       int				   =				unavailable_error


# Here we define all the available formatting functions that can be used by `Dataset.set_format`
_register_formatter(PythonFormatter, None, aliases=["""python"""])
_register_formatter(ArrowFormatter, """arrow""", aliases=["""pa""", """pyarrow"""])
_register_formatter(NumpyFormatter, """numpy""", aliases=["""np"""])
_register_formatter(PandasFormatter, """pandas""", aliases=["""pd"""])
_register_formatter(CustomFormatter, """custom""")

if config.TORCH_AVAILABLE:
   from .torch_formatter import TorchFormatter

   _register_formatter(TorchFormatter, """torch""", aliases=["""pt""", """pytorch"""])
else:
   UpperCAmelCase__						:List[str]          =			ValueError("""PyTorch needs to be installed to be able to return PyTorch tensors.""")
   _register_unavailable_formatter(_torch_error, """torch""", aliases=["""pt""", """pytorch"""])

if config.TF_AVAILABLE:
   from .tf_formatter import TFFormatter

   _register_formatter(TFFormatter, """tensorflow""", aliases=["""tf"""])
else:
   UpperCAmelCase__						:Optional[Any]          =			ValueError("""Tensorflow needs to be installed to be able to return Tensorflow tensors.""")
   _register_unavailable_formatter(_tf_error, """tensorflow""", aliases=["""tf"""])

if config.JAX_AVAILABLE:
   from .jax_formatter import JaxFormatter

   _register_formatter(JaxFormatter, """jax""", aliases=[])
else:
   UpperCAmelCase__						:Optional[int]          =			ValueError("""JAX needs to be installed to be able to return JAX arrays.""")
   _register_unavailable_formatter(_jax_error, """jax""", aliases=[])
def     __lowercase  (_lowercase     )   ->     Optional[str]:


     """simple docstring"""
     if format_type in _FORMAT_TYPES_ALIASES:
          return _FORMAT_TYPES_ALIASES[format_type]
     else:
          return format_type
def     __lowercase  (_lowercase,						**_lowercase     )   ->     Formatter:


     """simple docstring"""
     __lowerCamelCase						:       Any				   =				get_format_type_from_alias(_lowercase     )
     if format_type in _FORMAT_TYPES:
          return _FORMAT_TYPES[format_type](**_lowercase     )
     if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
          raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
     else:
          raise ValueError(
              f"Return type should be None or selected in {list(type for type in _FORMAT_TYPES.keys() if type != None     )}, but got '{format_type}'"     )











'''simple docstring'''

from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)
def     __lowercase  (_lowercase,						_lowercase     )   ->     List[str]:


     """simple docstring"""
     __lowerCamelCase						:       Union[str, Any]				   =				[]
     for part_id in partition_order:
          __lowerCamelCase						:       Union[str, Any]				   =				df.where(f"SPARK_PARTITION_ID() = {part_id}"     ).collect()
          for row_idx, row in enumerate(_lowercase     ):
               expected_row_ids_and_row_dicts.append((f"{part_id}_{row_idx}", row.asDict())     )
     return expected_row_ids_and_row_dicts
@require_not_windows
@require_dill_gt_0_3_2
def     __lowercase  ()   ->     Optional[int]:


     """simple docstring"""
     __lowerCamelCase						:       int				   =				pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
     __lowerCamelCase						:       List[str]				   =				spark.range(100     ).repartition(1     )
     __lowerCamelCase						:       str				   =				Spark(_lowercase     )
     # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
     # that each partition can hold 2 rows.
     spark_builder._repartition_df_if_needed(max_shard_size=16     )
     # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
     assert spark_builder.df.rdd.getNumPartitions() == 50
@require_not_windows
@require_dill_gt_0_3_2
def     __lowercase  ()   ->     Optional[Any]:


     """simple docstring"""
     __lowerCamelCase						:       Any				   =				pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
     __lowerCamelCase						:       Dict				   =				spark.range(10     ).repartition(2     )
     __lowerCamelCase						:       List[Any]				   =				[1, 0]
     __lowerCamelCase						:       Any				   =				_generate_iterable_examples(_lowercase,						_lowercase     )  # Reverse the partitions.
     __lowerCamelCase						:       int				   =				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase,						_lowercase     )

     for i, (row_id, row_dict) in enumerate(generate_fn()     ):
          __lowerCamelCase   ,      __lowerCamelCase						:       str				   =				expected_row_ids_and_row_dicts[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def     __lowercase  ()   ->     int:


     """simple docstring"""
     __lowerCamelCase						:       Dict				   =				pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
     __lowerCamelCase						:       Union[str, Any]				   =				spark.range(10     ).repartition(1     )
     __lowerCamelCase						:       Dict				   =				SparkExamplesIterable(_lowercase     )
     assert it.n_shards == 1
     for i, (row_id, row_dict) in enumerate(_lowercase     ):
          assert row_id == f"0_{i}"
          assert row_dict == {"id": i}
@require_not_windows
@require_dill_gt_0_3_2
def     __lowercase  ()   ->     Tuple:


     """simple docstring"""
     __lowerCamelCase						:       int				   =				pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
     __lowerCamelCase						:       Optional[Any]				   =				spark.range(30     ).repartition(3     )
     # Mock the generator so that shuffle reverses the partition indices.
     with patch("""numpy.random.Generator"""     ) as generator_mock:
          __lowerCamelCase						:       Dict				   =				lambda _lowercase     : x.reverse()
          __lowerCamelCase						:       Union[str, Any]				   =				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase,						[2, 1, 0]     )

          __lowerCamelCase						:       Tuple				   =				SparkExamplesIterable(_lowercase     ).shuffle_data_sources(_lowercase     )
          assert shuffled_it.n_shards == 3
          for i, (row_id, row_dict) in enumerate(_lowercase     ):
               __lowerCamelCase   ,      __lowerCamelCase						:       Union[str, Any]				   =				expected_row_ids_and_row_dicts[i]
               assert row_id == expected_row_id
               assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def     __lowercase  ()   ->     str:


     """simple docstring"""
     __lowerCamelCase						:       Union[str, Any]				   =				pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
     __lowerCamelCase						:       List[str]				   =				spark.range(20     ).repartition(4     )

     # Partitions 0 and 2
     __lowerCamelCase						:       Union[str, Any]				   =				SparkExamplesIterable(_lowercase     ).shard_data_sources(worker_id=0,						num_workers=2     )
     assert shard_it_a.n_shards == 2
     __lowerCamelCase						:       Optional[int]				   =				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase,						[0, 2]     )
     for i, (row_id, row_dict) in enumerate(_lowercase     ):
          __lowerCamelCase   ,      __lowerCamelCase						:       int				   =				expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict

     # Partitions 1 and 3
     __lowerCamelCase						:       List[str]				   =				SparkExamplesIterable(_lowercase     ).shard_data_sources(worker_id=1,						num_workers=2     )
     assert shard_it_a.n_shards == 2
     __lowerCamelCase						:       int				   =				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase,						[1, 3]     )
     for i, (row_id, row_dict) in enumerate(_lowercase     ):
          __lowerCamelCase   ,      __lowerCamelCase						:       int				   =				expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def     __lowercase  ()   ->     int:


     """simple docstring"""
     __lowerCamelCase						:       Union[str, Any]				   =				pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
     __lowerCamelCase						:       Dict				   =				spark.range(100     ).repartition(1     )
     __lowerCamelCase						:       Any				   =				Spark(_lowercase     )
     # Choose a small max_shard_size for maximum partitioning.
     spark_builder._repartition_df_if_needed(max_shard_size=1     )
     # The new number of partitions should not be greater than the number of rows.
     assert spark_builder.df.rdd.getNumPartitions() == 100



