




from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching


def       UpperCAmelCase__ ():
  """simple docstring"""

  import os as original_os
  from os import path as original_path
  from os import rename as original_rename
  from os.path import dirname as original_dirname
  from os.path import join as original_join

  assert _test_patching.os is original_os
  assert _test_patching.path is original_path
  assert _test_patching.join is original_join

  assert _test_patching.renamed_os is original_os
  assert _test_patching.renamed_path is original_path
  assert _test_patching.renamed_join is original_join

  _snake_case							:	Dict			=						"""__test_patch_submodule_mock__"""
  with patch_submodule(_test_patching	,     """os.path.join"""	,     snake_case__					):
    # Every way to access os.path.join must be patched, and the rest must stay untouched

    # check os.path.join
    assert isinstance(_test_patching.os	,     _PatchedModuleObj					)
    assert isinstance(_test_patching.os.path	,     _PatchedModuleObj					)
    assert _test_patching.os.path.join is mock

    # check path.join
    assert isinstance(_test_patching.path	,     _PatchedModuleObj					)
    assert _test_patching.path.join is mock

    # check join
    assert _test_patching.join is mock

    # check that the other attributes are untouched
    assert _test_patching.os.rename is original_rename
    assert _test_patching.path.dirname is original_dirname
    assert _test_patching.os.path.dirname is original_dirname

    # Even renamed modules or objects must be patched

    # check renamed_os.path.join
    assert isinstance(_test_patching.renamed_os	,     _PatchedModuleObj					)
    assert isinstance(_test_patching.renamed_os.path	,     _PatchedModuleObj					)
    assert _test_patching.renamed_os.path.join is mock

    # check renamed_path.join
    assert isinstance(_test_patching.renamed_path	,     _PatchedModuleObj					)
    assert _test_patching.renamed_path.join is mock

    # check renamed_join
    assert _test_patching.renamed_join is mock

    # check that the other attributes are untouched
    assert _test_patching.renamed_os.rename is original_rename
    assert _test_patching.renamed_path.dirname is original_dirname
    assert _test_patching.renamed_os.path.dirname is original_dirname

  # check that everthing is back to normal when the patch is over

  assert _test_patching.os is original_os
  assert _test_patching.path is original_path
  assert _test_patching.join is original_join

  assert _test_patching.renamed_os is original_os
  assert _test_patching.renamed_path is original_path
  assert _test_patching.renamed_join is original_join


def       UpperCAmelCase__ ():
  """simple docstring"""

  assert _test_patching.open is open

  _snake_case							:	Any			=						"""__test_patch_submodule_builtin_mock__"""
  # _test_patching has "open" in its globals
  assert _test_patching.open is open
  with patch_submodule(_test_patching	,     """open"""	,     snake_case__					):
    assert _test_patching.open is mock

  # check that everthing is back to normal when the patch is over

  assert _test_patching.open is open


def       UpperCAmelCase__ ():
  """simple docstring"""

  _snake_case							:	Union[str, Any]			=						"""__test_patch_submodule_missing_mock__"""
  with patch_submodule(_test_patching	,     """pandas.read_csv"""	,     snake_case__					):
    pass


def       UpperCAmelCase__ ():
  """simple docstring"""

  _snake_case							:	Any			=						"""__test_patch_submodule_missing_builtin_mock__"""
  # _test_patching doesn't have "len" in its globals
  assert getattr(_test_patching	,     """len"""	,     snake_case__					) is None
  with patch_submodule(_test_patching	,     """len"""	,     snake_case__					):
    assert _test_patching.len is mock
  assert _test_patching.len is len


def       UpperCAmelCase__ ():
  """simple docstring"""

  _snake_case							:	Optional[int]			=						"""__test_patch_submodule_start_and_stop_mock__"""
  _snake_case							:	Dict			=						patch_submodule(_test_patching	,     """open"""	,     snake_case__					)
  assert _test_patching.open is open
  patch.start()
  assert _test_patching.open is mock
  patch.stop()
  assert _test_patching.open is open


def       UpperCAmelCase__ ():
  """simple docstring"""

  from os import rename as original_rename
  from os.path import dirname as original_dirname
  from os.path import join as original_join

  _snake_case							:	Any			=						"""__test_patch_submodule_successive_join__"""
  _snake_case							:	Union[str, Any]			=						"""__test_patch_submodule_successive_dirname__"""
  _snake_case							:	Tuple			=						"""__test_patch_submodule_successive_rename__"""
  assert _test_patching.os.path.join is original_join
  assert _test_patching.os.path.dirname is original_dirname
  assert _test_patching.os.rename is original_rename

  with patch_submodule(_test_patching	,     """os.path.join"""	,     snake_case__					):
    with patch_submodule(_test_patching	,     """os.rename"""	,     snake_case__					):
      with patch_submodule(_test_patching	,     """os.path.dirname"""	,     snake_case__					):
        assert _test_patching.os.path.join is mock_join
        assert _test_patching.os.path.dirname is mock_dirname
        assert _test_patching.os.rename is mock_rename

    # try another order
  with patch_submodule(_test_patching	,     """os.rename"""	,     snake_case__					):
    with patch_submodule(_test_patching	,     """os.path.join"""	,     snake_case__					):
      with patch_submodule(_test_patching	,     """os.path.dirname"""	,     snake_case__					):
        assert _test_patching.os.path.join is mock_join
        assert _test_patching.os.path.dirname is mock_dirname
        assert _test_patching.os.rename is mock_rename

  assert _test_patching.os.path.join is original_join
  assert _test_patching.os.path.dirname is original_dirname
  assert _test_patching.os.rename is original_rename






def       UpperCAmelCase__ ():
  """simple docstring"""

  _snake_case							:	str			=						"""__test_patch_submodule_doesnt_exist_mock__"""
  with patch_submodule(_test_patching	,     """__module_that_doesn_exist__.__attribute_that_doesn_exist__"""	,     snake_case__					):
    pass
  with patch_submodule(_test_patching	,     """os.__attribute_that_doesn_exist__"""	,     snake_case__					):
    pass








"""simple docstring"""


import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def       UpperCAmelCase__ (snake_case__							:      Optional[Any]	,     snake_case__							:      Dict					):
  """simple docstring"""

  assert isinstance(snake_case__	,     snake_case__					)
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""	,     [False, True]					)
def       UpperCAmelCase__ (snake_case__							:      str	,     snake_case__							:      Optional[int]	,     snake_case__							:      Dict					):
  """simple docstring"""

  _snake_case							:	str			=						tmp_path / """cache"""
  _snake_case							:	int			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    _snake_case							:	str			=						ParquetDatasetReader(snake_case__	,     cache_dir=snake_case__	,     keep_in_memory=snake_case__					).read()
  _check_parquet_dataset(snake_case__	,     snake_case__					)


@pytest.mark.parametrize(
    """features"""	,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]	,     )
def       UpperCAmelCase__ (snake_case__							:      Dict	,     snake_case__							:      int	,     snake_case__							:      List[Any]					):
  """simple docstring"""

  _snake_case							:	str			=						tmp_path / """cache"""
  _snake_case							:	List[Any]			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  _snake_case							:	List[Any]			=						features.copy() if features else default_expected_features
  _snake_case							:	List[Any]			=						(
      Features({feature: Value(snake_case__					) for feature, dtype in features.items()}					) if features is not None else None
  )
  _snake_case							:	Optional[Any]			=						ParquetDatasetReader(snake_case__	,     features=snake_case__	,     cache_dir=snake_case__					).read()
  _check_parquet_dataset(snake_case__	,     snake_case__					)


@pytest.mark.parametrize("""split"""	,     [None, NamedSplit("""train"""					), """train""", """test"""]					)
def       UpperCAmelCase__ (snake_case__							:      Optional[int]	,     snake_case__							:      int	,     snake_case__							:      int					):
  """simple docstring"""

  _snake_case							:	List[str]			=						tmp_path / """cache"""
  _snake_case							:	Any			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  _snake_case							:	int			=						ParquetDatasetReader(snake_case__	,     cache_dir=snake_case__	,     split=snake_case__					).read()
  _check_parquet_dataset(snake_case__	,     snake_case__					)
  assert dataset.split == split if split else "train"


@pytest.mark.parametrize("""path_type"""	,     [str, list]					)
def       UpperCAmelCase__ (snake_case__							:      Dict	,     snake_case__							:      str	,     snake_case__							:      str					):
  """simple docstring"""

  if issubclass(snake_case__	,     snake_case__					):
    _snake_case							:	Optional[Any]			=						parquet_path
  elif issubclass(snake_case__	,     snake_case__					):
    _snake_case							:	int			=						[parquet_path]
  _snake_case							:	Union[str, Any]			=						tmp_path / """cache"""
  _snake_case							:	Tuple			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  _snake_case							:	List[str]			=						ParquetDatasetReader(snake_case__	,     cache_dir=snake_case__					).read()
  _check_parquet_dataset(snake_case__	,     snake_case__					)


def       UpperCAmelCase__ (snake_case__							:      str	,     snake_case__							:      Optional[int]	,     snake_case__							:      str=("train",)					):
  """simple docstring"""

  assert isinstance(snake_case__	,     snake_case__					)
  for split in splits:
    _snake_case							:	int			=						dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""	,     [False, True]					)
def       UpperCAmelCase__ (snake_case__							:      str	,     snake_case__							:      str	,     snake_case__							:      List[Any]					):
  """simple docstring"""

  _snake_case							:	Tuple			=						tmp_path / """cache"""
  _snake_case							:	Optional[int]			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    _snake_case							:	Tuple			=						ParquetDatasetReader(
        {"""train""": parquet_path}	,     cache_dir=snake_case__	,     keep_in_memory=snake_case__					).read()
  _check_parquet_datasetdict(snake_case__	,     snake_case__					)


@pytest.mark.parametrize(
    """features"""	,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]	,     )
def       UpperCAmelCase__ (snake_case__							:      List[str]	,     snake_case__							:      Tuple	,     snake_case__							:      List[Any]					):
  """simple docstring"""

  _snake_case							:	Optional[int]			=						tmp_path / """cache"""
  _snake_case							:	Dict			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  _snake_case							:	Optional[Any]			=						features.copy() if features else default_expected_features
  _snake_case							:	Dict			=						(
      Features({feature: Value(snake_case__					) for feature, dtype in features.items()}					) if features is not None else None
  )
  _snake_case							:	Optional[int]			=						ParquetDatasetReader({"""train""": parquet_path}	,     features=snake_case__	,     cache_dir=snake_case__					).read()
  _check_parquet_datasetdict(snake_case__	,     snake_case__					)


@pytest.mark.parametrize("""split"""	,     [None, NamedSplit("""train"""					), """train""", """test"""]					)
def       UpperCAmelCase__ (snake_case__							:      List[str]	,     snake_case__							:      Optional[Any]	,     snake_case__							:      Tuple					):
  """simple docstring"""

  if split:
    _snake_case							:	int			=						{split: parquet_path}
  else:
    _snake_case							:	Optional[Any]			=						"""train"""
    _snake_case							:	int			=						{"""train""": parquet_path, """test""": parquet_path}
  _snake_case							:	Dict			=						tmp_path / """cache"""
  _snake_case							:	Any			=						{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
  _snake_case							:	Union[str, Any]			=						ParquetDatasetReader(snake_case__	,     cache_dir=snake_case__					).read()
  _check_parquet_datasetdict(snake_case__	,     snake_case__	,     splits=list(path.keys()					)					)
  assert all(dataset[split].split == split for split in path.keys()					)


def       UpperCAmelCase__ (snake_case__							:      Tuple	,     snake_case__							:      Tuple					):
  """simple docstring"""

  _snake_case							:	List[Any]			=						ParquetDatasetWriter(snake_case__	,     tmp_path / """foo.parquet"""					)
  assert writer.write() > 0
  _snake_case							:	str			=						pq.ParquetFile(tmp_path / """foo.parquet"""					)
  _snake_case							:	int			=						pf.read()
  assert dataset.data.table == output_table


def       UpperCAmelCase__ (snake_case__							:      int	,     snake_case__							:      int					):
  """simple docstring"""

  _snake_case							:	Optional[Any]			=						str(shared_datadir / """test_image_rgb.jpg"""					)
  _snake_case							:	Tuple			=						{"""image""": [image_path]}
  _snake_case							:	Optional[int]			=						Features({"""image""": Image()}					)
  _snake_case							:	int			=						Dataset.from_dict(snake_case__	,     features=snake_case__					)
  _snake_case							:	Optional[Any]			=						ParquetDatasetWriter(snake_case__	,     tmp_path / """foo.parquet"""					)
  assert writer.write() > 0

  _snake_case							:	List[str]			=						Dataset.from_parquet(str(tmp_path / """foo.parquet"""					)					)
  assert dataset.features == reloaded_dataset.features

  _snake_case							:	Optional[Any]			=						ParquetDatasetReader(str(tmp_path / """foo.parquet"""					)	,     streaming=snake_case__					).read()
  assert dataset.features == reloaded_iterable_dataset.features






@pytest.mark.parametrize(
    """feature, expected"""	,     [
        (Features({"""foo""": Value("""int32"""					)}					), None),
        (Features({"""image""": Image(), """foo""": Value("""int32"""					)}					), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"""nested""": Sequence(Audio()					)}					), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]	,     )
def       UpperCAmelCase__ (snake_case__							:      List[Any]	,     snake_case__							:      str					):
  """simple docstring"""

  assert get_writer_batch_size(snake_case__					) == expected


