


'''simple docstring'''



import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def    lowerCamelCase__   ( A     :    Optional[Any]   ,     A     :    int  ):




						'''simple docstring'''






						assert isinstance(A   ,     A  )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,     [False, True]  )
def    lowerCamelCase__   ( A     :    Any   ,     A     :    Any   ,     A     :    Optional[int]  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A   ,     keep_in_memory=A  ).read()
						_check_parquet_dataset(A   ,     A  )


@pytest.mark.parametrize(
    '''features'''   ,     [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,     )
def    lowerCamelCase__   ( A     :    Optional[Any]   ,     A     :    Optional[Any]   ,     A     :    Tuple  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			features.copy() if features else default_expected_features
						UpperCAmelCase         =			(
						    Features({feature: Value(A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     features=A   ,     cache_dir=A  ).read()
						_check_parquet_dataset(A   ,     A  )


@pytest.mark.parametrize('''split'''   ,     [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def    lowerCamelCase__   ( A     :    str   ,     A     :    str   ,     A     :    int  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A   ,     split=A  ).read()
						_check_parquet_dataset(A   ,     A  )
						assert dataset.split == split if split else "train"


@pytest.mark.parametrize('''path_type'''   ,     [str, list]  )
def    lowerCamelCase__   ( A     :    str   ,     A     :    Optional[Any]   ,     A     :    int  ):




						'''simple docstring'''






						if issubclass(A   ,     A  ):
												UpperCAmelCase         =			parquet_path
						elif issubclass(A   ,     A  ):
												UpperCAmelCase         =			[parquet_path]
						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A  ).read()
						_check_parquet_dataset(A   ,     A  )


def    lowerCamelCase__   ( A     :    List[Any]   ,     A     :    str   ,     A     :    Dict=("train",)  ):




						'''simple docstring'''






						assert isinstance(A   ,     A  )
						for split in splits:
												UpperCAmelCase         =			dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,     [False, True]  )
def    lowerCamelCase__   ( A     :    str   ,     A     :    Optional[int]   ,     A     :    Union[str, Any]  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												UpperCAmelCase         =			ParquetDatasetReader(
												    {'''train''': parquet_path}   ,     cache_dir=A   ,     keep_in_memory=A  ).read()
						_check_parquet_datasetdict(A   ,     A  )


@pytest.mark.parametrize(
    '''features'''   ,     [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,     )
def    lowerCamelCase__   ( A     :    str   ,     A     :    Any   ,     A     :    int  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			features.copy() if features else default_expected_features
						UpperCAmelCase         =			(
						    Features({feature: Value(A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						UpperCAmelCase         =			ParquetDatasetReader({'''train''': parquet_path}   ,     features=A   ,     cache_dir=A  ).read()
						_check_parquet_datasetdict(A   ,     A  )


@pytest.mark.parametrize('''split'''   ,     [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def    lowerCamelCase__   ( A     :    Optional[Any]   ,     A     :    Tuple   ,     A     :    List[Any]  ):




						'''simple docstring'''






						if split:
												UpperCAmelCase         =			{split: parquet_path}
						else:
												UpperCAmelCase         =			'''train'''
												UpperCAmelCase         =			{'''train''': parquet_path, '''test''': parquet_path}
						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A  ).read()
						_check_parquet_datasetdict(A   ,     A   ,     splits=list(path.keys()  )  )
						assert all(dataset[split].split == split for split in path.keys()  )


def    lowerCamelCase__   ( A     :    Any   ,     A     :    Dict  ):




						'''simple docstring'''






						UpperCAmelCase         =			ParquetDatasetWriter(A   ,     tmp_path / '''foo.parquet'''  )
						assert writer.write() > 0
						UpperCAmelCase         =			pq.ParquetFile(tmp_path / '''foo.parquet'''  )
						UpperCAmelCase         =			pf.read()
						assert dataset.data.table == output_table


def    lowerCamelCase__   ( A     :    Dict   ,     A     :    Optional[int]  ):




						'''simple docstring'''






						UpperCAmelCase         =			str(shared_datadir / '''test_image_rgb.jpg'''  )
						UpperCAmelCase         =			{'''image''': [image_path]}
						UpperCAmelCase         =			Features({'''image''': Image()}  )
						UpperCAmelCase         =			Dataset.from_dict(A   ,     features=A  )
						UpperCAmelCase         =			ParquetDatasetWriter(A   ,     tmp_path / '''foo.parquet'''  )
						assert writer.write() > 0

						UpperCAmelCase         =			Dataset.from_parquet(str(tmp_path / '''foo.parquet'''  )  )
						assert dataset.features == reloaded_dataset.features

						UpperCAmelCase         =			ParquetDatasetReader(str(tmp_path / '''foo.parquet'''  )   ,     streaming=A  ).read()
						assert dataset.features == reloaded_iterable_dataset.features




@pytest.mark.parametrize(
    '''feature, expected'''   ,     [
        (Features({'''foo''': Value('''int32'''  )}  ), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''  )}  ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()  )}  ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]   ,     )
def    lowerCamelCase__   ( A     :    str   ,     A     :    int  ):




						'''simple docstring'''






						assert get_writer_batch_size(A  ) == expected









'''simple docstring'''



# Copyright 2022 The HuggingFace Team and The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_lowercase    :    Any			     =  {
    """configuration_cpmant""": ["""CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CpmAntConfig"""],
    """tokenization_cpmant""": ["""CpmAntTokenizer"""],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowercase    :    int			     =  [
	    """CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """CpmAntForCausalLM""",
	    """CpmAntModel""",
	    """CpmAntPreTrainedModel""",
	]


if TYPE_CHECKING:
	from .configuration_cpmant import CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP, CpmAntConfig
	from .tokenization_cpmant import CpmAntTokenizer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_cpmant import (
		    CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    CpmAntForCausalLM,
		    CpmAntModel,
		    CpmAntPreTrainedModel,
		)


else:
	import sys

	_lowercase    :    Dict			     =  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





