



'''simple docstring'''

import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases







def 						__UpperCamelCase				(							_lowercase,				_lowercase					) ->			Tuple:
	assert isinstance(_lowercase,				_lowercase					)
	assert dataset.num_rows == 4
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_1", "col_2", "col_3"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype







@pytest.mark.parametrize('keep_in_memory',				[False, True]					)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			Tuple:
	_lowercase					:       Optional[int]       =		tmp_path / 'cache'
	_lowercase					:       Tuple       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		_lowercase					:       Tuple       =		ParquetDatasetReader(_lowercase,				cache_dir=_lowercase,				keep_in_memory=_lowercase					).read()
	_check_parquet_dataset(_lowercase,				_lowercase					)







@pytest.mark.parametrize(
    'features',				[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ],				)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			str:
	_lowercase					:       int       =		tmp_path / 'cache'
	_lowercase					:       Any       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	_lowercase					:       List[Any]       =		features.copy() if features else default_expected_features
	_lowercase					:       str       =		(
	    Features({feature: Value(_lowercase					) for feature, dtype in features.items()}					) if features is not None else None
	)
	_lowercase					:       Any       =		ParquetDatasetReader(_lowercase,				features=_lowercase,				cache_dir=_lowercase					).read()
	_check_parquet_dataset(_lowercase,				_lowercase					)







@pytest.mark.parametrize('split',				[None, NamedSplit('train'					), 'train', 'test']					)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			Union[str, Any]:
	_lowercase					:       Tuple       =		tmp_path / 'cache'
	_lowercase					:       Optional[Any]       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	_lowercase					:       List[str]       =		ParquetDatasetReader(_lowercase,				cache_dir=_lowercase,				split=_lowercase					).read()
	_check_parquet_dataset(_lowercase,				_lowercase					)
	assert dataset.split == split if split else "train"







@pytest.mark.parametrize('path_type',				[str, list]					)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			List[str]:







	if issubclass(_lowercase,				_lowercase					):
		_lowercase					:       int       =		parquet_path
	elif issubclass(_lowercase,				_lowercase					):
		_lowercase					:       Any       =		[parquet_path]
	_lowercase					:       Optional[Any]       =		tmp_path / 'cache'
	_lowercase					:       List[str]       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	_lowercase					:       Any       =		ParquetDatasetReader(_lowercase,				cache_dir=_lowercase					).read()
	_check_parquet_dataset(_lowercase,				_lowercase					)







def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase=("train",)					) ->			int:
	assert isinstance(_lowercase,				_lowercase					)
	for split in splits:
		_lowercase					:       int       =		dataset_dict[split]
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
			assert dataset.features[feature].dtype == expected_dtype







@pytest.mark.parametrize('keep_in_memory',				[False, True]					)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			Union[str, Any]:
	_lowercase					:       Union[str, Any]       =		tmp_path / 'cache'
	_lowercase					:       int       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		_lowercase					:       Any       =		ParquetDatasetReader(
		    {'train': parquet_path},				cache_dir=_lowercase,				keep_in_memory=_lowercase					).read()
	_check_parquet_datasetdict(_lowercase,				_lowercase					)







@pytest.mark.parametrize(
    'features',				[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ],				)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			Tuple:
	_lowercase					:       List[str]       =		tmp_path / 'cache'
	_lowercase					:       Any       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	_lowercase					:       str       =		features.copy() if features else default_expected_features
	_lowercase					:       Dict       =		(
	    Features({feature: Value(_lowercase					) for feature, dtype in features.items()}					) if features is not None else None
	)
	_lowercase					:       List[str]       =		ParquetDatasetReader({'train': parquet_path},				features=_lowercase,				cache_dir=_lowercase					).read()
	_check_parquet_datasetdict(_lowercase,				_lowercase					)







@pytest.mark.parametrize('split',				[None, NamedSplit('train'					), 'train', 'test']					)
def 						__UpperCamelCase				(							_lowercase,				_lowercase,				_lowercase					) ->			List[str]:







	if split:
		_lowercase					:       Tuple       =		{split: parquet_path}
	else:
		_lowercase					:       List[Any]       =		'train'
		_lowercase					:       List[str]       =		{'train': parquet_path, 'test': parquet_path}
	_lowercase					:       Optional[Any]       =		tmp_path / 'cache'
	_lowercase					:       Optional[Any]       =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
	_lowercase					:       Tuple       =		ParquetDatasetReader(_lowercase,				cache_dir=_lowercase					).read()
	_check_parquet_datasetdict(_lowercase,				_lowercase,				splits=list(path.keys()					)					)
	assert all(dataset[split].split == split for split in path.keys()					)







def 						__UpperCamelCase				(							_lowercase,				_lowercase					) ->			str:
	_lowercase					:       Optional[int]       =		ParquetDatasetWriter(_lowercase,				tmp_path / 'foo.parquet'					)
	assert writer.write() > 0
	_lowercase					:       Any       =		pq.ParquetFile(tmp_path / 'foo.parquet'					)
	_lowercase					:       int       =		pf.read()
	assert dataset.data.table == output_table







def 						__UpperCamelCase				(							_lowercase,				_lowercase					) ->			List[Any]:
	_lowercase					:       List[Any]       =		str(shared_datadir / 'test_image_rgb.jpg'					)
	_lowercase					:       List[Any]       =		{'image': [image_path]}
	_lowercase					:       Dict       =		Features({'image': Image()}					)
	_lowercase					:       str       =		Dataset.from_dict(_lowercase,				features=_lowercase					)
	_lowercase					:       int       =		ParquetDatasetWriter(_lowercase,				tmp_path / 'foo.parquet'					)
	assert writer.write() > 0

	_lowercase					:       Tuple       =		Dataset.from_parquet(str(tmp_path / 'foo.parquet'					)					)
	assert dataset.features == reloaded_dataset.features

	_lowercase					:       List[str]       =		ParquetDatasetReader(str(tmp_path / 'foo.parquet'					),				streaming=_lowercase					).read()
	assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    'feature, expected',				[
        (Features({'foo': Value('int32'					)}					), None),
        (Features({'image': Image(), 'foo': Value('int32'					)}					), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'nested': Sequence(Audio()					)}					), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ],				)
def 						__UpperCamelCase				(							_lowercase,				_lowercase					) ->			int:
	assert get_writer_batch_size(_lowercase					) == expected





'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A				:    Optional[Any]     						={'''configuration_xlnet''': ['''XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLNetConfig''']}

try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    Tuple     						=['''XLNetTokenizer''']

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    str     						=['''XLNetTokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    Any     						=[
		    '''XLNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''XLNetForMultipleChoice''',
		    '''XLNetForQuestionAnswering''',
		    '''XLNetForQuestionAnsweringSimple''',
		    '''XLNetForSequenceClassification''',
		    '''XLNetForTokenClassification''',
		    '''XLNetLMHeadModel''',
		    '''XLNetModel''',
		    '''XLNetPreTrainedModel''',
		    '''load_tf_weights_in_xlnet''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    str     						=[
		    '''TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFXLNetForMultipleChoice''',
		    '''TFXLNetForQuestionAnsweringSimple''',
		    '''TFXLNetForSequenceClassification''',
		    '''TFXLNetForTokenClassification''',
		    '''TFXLNetLMHeadModel''',
		    '''TFXLNetMainLayer''',
		    '''TFXLNetModel''',
		    '''TFXLNetPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_xlnet import XLNetTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_xlnet_fast import XLNetTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_xlnet import (
				    XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
				    XLNetForMultipleChoice,
				    XLNetForQuestionAnswering,
				    XLNetForQuestionAnsweringSimple,
				    XLNetForSequenceClassification,
				    XLNetForTokenClassification,
				    XLNetLMHeadModel,
				    XLNetModel,
				    XLNetPreTrainedModel,
				    load_tf_weights_in_xlnet,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_xlnet import (
				    TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFXLNetForMultipleChoice,
				    TFXLNetForQuestionAnsweringSimple,
				    TFXLNetForSequenceClassification,
				    TFXLNetForTokenClassification,
				    TFXLNetLMHeadModel,
				    TFXLNetMainLayer,
				    TFXLNetModel,
				    TFXLNetPreTrainedModel,
				)

else:
		import sys

		_A				:    str     						=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
