
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_UpperCAmelCase      :				Any          = {
    "configuration_funnel": ["FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP", "FunnelConfig"],
    "convert_funnel_original_tf_checkpoint_to_pytorch": [],
    "tokenization_funnel": ["FunnelTokenizer"],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase      :				Union[str, Any]          = ["FunnelTokenizerFast"]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase      :				Optional[int]          = [
					    "FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "FunnelBaseModel",
					    "FunnelForMaskedLM",
					    "FunnelForMultipleChoice",
					    "FunnelForPreTraining",
					    "FunnelForQuestionAnswering",
					    "FunnelForSequenceClassification",
					    "FunnelForTokenClassification",
					    "FunnelModel",
					    "FunnelPreTrainedModel",
					    "load_tf_weights_in_funnel",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase      :				int          = [
					    "TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TFFunnelBaseModel",
					    "TFFunnelForMaskedLM",
					    "TFFunnelForMultipleChoice",
					    "TFFunnelForPreTraining",
					    "TFFunnelForQuestionAnswering",
					    "TFFunnelForSequenceClassification",
					    "TFFunnelForTokenClassification",
					    "TFFunnelModel",
					    "TFFunnelPreTrainedModel",
					]


if TYPE_CHECKING:
					from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
					from .tokenization_funnel import FunnelTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_funnel_fast import FunnelTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_funnel import (
										    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    FunnelBaseModel,
										    FunnelForMaskedLM,
										    FunnelForMultipleChoice,
										    FunnelForPreTraining,
										    FunnelForQuestionAnswering,
										    FunnelForSequenceClassification,
										    FunnelForTokenClassification,
										    FunnelModel,
										    FunnelPreTrainedModel,
										    load_tf_weights_in_funnel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_funnel import (
										    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFFunnelBaseModel,
										    TFFunnelForMaskedLM,
										    TFFunnelForMultipleChoice,
										    TFFunnelForPreTraining,
										    TFFunnelForQuestionAnswering,
										    TFFunnelForSequenceClassification,
										    TFFunnelForTokenClassification,
										    TFFunnelModel,
										    TFFunnelPreTrainedModel,
										)

else:
					import sys

					_UpperCAmelCase      :				Optional[Any]          = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def    A (      lowercase					, lowercase				)     ->						Optional[int]:
		'''simple docstring'''

		assert isinstance(lowercase					, lowercase				)
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('keep_in_memory'					, [False, True]				)
def    A (      lowercase					, lowercase					, lowercase				)     ->						Tuple:
		'''simple docstring'''

		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				UpperCamelCase						=  ParquetDatasetReader(lowercase					, cache_dir=lowercase					, keep_in_memory=lowercase				).read()
		_check_parquet_dataset(lowercase					, lowercase				)


@pytest.mark.parametrize(
    'features'					, [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]					, )
def    A (      lowercase					, lowercase					, lowercase				)     ->						Tuple:
		'''simple docstring'''

		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		UpperCamelCase						=  features.copy() if features else default_expected_features
		UpperCamelCase						=  (
		    Features({feature: Value(lowercase				) for feature, dtype in features.items()}				) if features is not None else None
		)
		UpperCamelCase						=  ParquetDatasetReader(lowercase					, features=lowercase					, cache_dir=lowercase				).read()
		_check_parquet_dataset(lowercase					, lowercase				)


@pytest.mark.parametrize('split'					, [None, NamedSplit('train'				), 'train', 'test']				)
def    A (      lowercase					, lowercase					, lowercase				)     ->						Optional[int]:
		'''simple docstring'''

		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		UpperCamelCase						=  ParquetDatasetReader(lowercase					, cache_dir=lowercase					, split=lowercase				).read()
		_check_parquet_dataset(lowercase					, lowercase				)
		assert dataset.split == split if split else "train"


@pytest.mark.parametrize('path_type'					, [str, list]				)
def    A (      lowercase					, lowercase					, lowercase				)     ->						Union[str, Any]:
		'''simple docstring'''

		if issubclass(lowercase					, lowercase				):
				UpperCamelCase						=  parquet_path
		elif issubclass(lowercase					, lowercase				):
				UpperCamelCase						=  [parquet_path]
		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		UpperCamelCase						=  ParquetDatasetReader(lowercase					, cache_dir=lowercase				).read()
		_check_parquet_dataset(lowercase					, lowercase				)


def    A (      lowercase					, lowercase					, lowercase=("train",)				)     ->						Tuple:
		'''simple docstring'''

		assert isinstance(lowercase					, lowercase				)
		for split in splits:
				UpperCamelCase						=  dataset_dict[split]
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('keep_in_memory'					, [False, True]				)
def    A (      lowercase					, lowercase					, lowercase				)     ->						Optional[Any]:
		'''simple docstring'''

		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				UpperCamelCase						=  ParquetDatasetReader(
				    {'train': parquet_path}					, cache_dir=lowercase					, keep_in_memory=lowercase				).read()
		_check_parquet_datasetdict(lowercase					, lowercase				)


@pytest.mark.parametrize(
    'features'					, [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]					, )
def    A (      lowercase					, lowercase					, lowercase				)     ->						List[Any]:
		'''simple docstring'''

		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		UpperCamelCase						=  features.copy() if features else default_expected_features
		UpperCamelCase						=  (
		    Features({feature: Value(lowercase				) for feature, dtype in features.items()}				) if features is not None else None
		)
		UpperCamelCase						=  ParquetDatasetReader({'train': parquet_path}					, features=lowercase					, cache_dir=lowercase				).read()
		_check_parquet_datasetdict(lowercase					, lowercase				)


@pytest.mark.parametrize('split'					, [None, NamedSplit('train'				), 'train', 'test']				)
def    A (      lowercase					, lowercase					, lowercase				)     ->						Union[str, Any]:
		'''simple docstring'''

		if split:
				UpperCamelCase						=  {split: parquet_path}
		else:
				UpperCamelCase						=  'train'
				UpperCamelCase						=  {'train': parquet_path, 'test': parquet_path}
		UpperCamelCase						=  tmp_path / 'cache'
		UpperCamelCase						=  {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		UpperCamelCase						=  ParquetDatasetReader(lowercase					, cache_dir=lowercase				).read()
		_check_parquet_datasetdict(lowercase					, lowercase					, splits=list(path.keys()				)				)
		assert all(dataset[split].split == split for split in path.keys()				)


def    A (      lowercase					, lowercase				)     ->						List[Any]:
		'''simple docstring'''

		UpperCamelCase						=  ParquetDatasetWriter(lowercase					, tmp_path / 'foo.parquet'				)
		assert writer.write() > 0
		UpperCamelCase						=  pq.ParquetFile(tmp_path / 'foo.parquet'				)
		UpperCamelCase						=  pf.read()
		assert dataset.data.table == output_table


def    A (      lowercase					, lowercase				)     ->						Optional[int]:
		'''simple docstring'''

		UpperCamelCase						=  str(shared_datadir / 'test_image_rgb.jpg'				)
		UpperCamelCase						=  {'image': [image_path]}
		UpperCamelCase						=  Features({'image': Image()}				)
		UpperCamelCase						=  Dataset.from_dict(lowercase					, features=lowercase				)
		UpperCamelCase						=  ParquetDatasetWriter(lowercase					, tmp_path / 'foo.parquet'				)
		assert writer.write() > 0

		UpperCamelCase						=  Dataset.from_parquet(str(tmp_path / 'foo.parquet'				)				)
		assert dataset.features == reloaded_dataset.features

		UpperCamelCase						=  ParquetDatasetReader(str(tmp_path / 'foo.parquet'				)					, streaming=lowercase				).read()
		assert dataset.features == reloaded_iterable_dataset.features


@pytest.mark.parametrize(
    'feature, expected'					, [
        (Features({'foo': Value('int32'				)}				), None),
        (Features({'image': Image(), 'foo': Value('int32'				)}				), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'nested': Sequence(Audio()				)}				), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]					, )
def    A (      lowercase					, lowercase				)     ->						Union[str, Any]:
		'''simple docstring'''

		assert get_writer_batch_size(lowercase				) == expected
