


'''simple docstring'''



import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def    lowerCamelCase__   ( A     :    Optional[Any]   ,     A     :    int  ):




						'''simple docstring'''






						assert isinstance(A   ,     A  )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,     [False, True]  )
def    lowerCamelCase__   ( A     :    Any   ,     A     :    Any   ,     A     :    Optional[int]  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A   ,     keep_in_memory=A  ).read()
						_check_parquet_dataset(A   ,     A  )


@pytest.mark.parametrize(
    '''features'''   ,     [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,     )
def    lowerCamelCase__   ( A     :    Optional[Any]   ,     A     :    Optional[Any]   ,     A     :    Tuple  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			features.copy() if features else default_expected_features
						UpperCAmelCase         =			(
						    Features({feature: Value(A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     features=A   ,     cache_dir=A  ).read()
						_check_parquet_dataset(A   ,     A  )


@pytest.mark.parametrize('''split'''   ,     [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def    lowerCamelCase__   ( A     :    str   ,     A     :    str   ,     A     :    int  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A   ,     split=A  ).read()
						_check_parquet_dataset(A   ,     A  )
						assert dataset.split == split if split else "train"


@pytest.mark.parametrize('''path_type'''   ,     [str, list]  )
def    lowerCamelCase__   ( A     :    str   ,     A     :    Optional[Any]   ,     A     :    int  ):




						'''simple docstring'''






						if issubclass(A   ,     A  ):
												UpperCAmelCase         =			parquet_path
						elif issubclass(A   ,     A  ):
												UpperCAmelCase         =			[parquet_path]
						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A  ).read()
						_check_parquet_dataset(A   ,     A  )


def    lowerCamelCase__   ( A     :    List[Any]   ,     A     :    str   ,     A     :    Dict=("train",)  ):




						'''simple docstring'''






						assert isinstance(A   ,     A  )
						for split in splits:
												UpperCAmelCase         =			dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,     [False, True]  )
def    lowerCamelCase__   ( A     :    str   ,     A     :    Optional[int]   ,     A     :    Union[str, Any]  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												UpperCAmelCase         =			ParquetDatasetReader(
												    {'''train''': parquet_path}   ,     cache_dir=A   ,     keep_in_memory=A  ).read()
						_check_parquet_datasetdict(A   ,     A  )


@pytest.mark.parametrize(
    '''features'''   ,     [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,     )
def    lowerCamelCase__   ( A     :    str   ,     A     :    Any   ,     A     :    int  ):




						'''simple docstring'''






						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			features.copy() if features else default_expected_features
						UpperCAmelCase         =			(
						    Features({feature: Value(A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						UpperCAmelCase         =			ParquetDatasetReader({'''train''': parquet_path}   ,     features=A   ,     cache_dir=A  ).read()
						_check_parquet_datasetdict(A   ,     A  )


@pytest.mark.parametrize('''split'''   ,     [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def    lowerCamelCase__   ( A     :    Optional[Any]   ,     A     :    Tuple   ,     A     :    List[Any]  ):




						'''simple docstring'''






						if split:
												UpperCAmelCase         =			{split: parquet_path}
						else:
												UpperCAmelCase         =			'''train'''
												UpperCAmelCase         =			{'''train''': parquet_path, '''test''': parquet_path}
						UpperCAmelCase         =			tmp_path / '''cache'''
						UpperCAmelCase         =			{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						UpperCAmelCase         =			ParquetDatasetReader(A   ,     cache_dir=A  ).read()
						_check_parquet_datasetdict(A   ,     A   ,     splits=list(path.keys()  )  )
						assert all(dataset[split].split == split for split in path.keys()  )


def    lowerCamelCase__   ( A     :    Any   ,     A     :    Dict  ):




						'''simple docstring'''






						UpperCAmelCase         =			ParquetDatasetWriter(A   ,     tmp_path / '''foo.parquet'''  )
						assert writer.write() > 0
						UpperCAmelCase         =			pq.ParquetFile(tmp_path / '''foo.parquet'''  )
						UpperCAmelCase         =			pf.read()
						assert dataset.data.table == output_table


def    lowerCamelCase__   ( A     :    Dict   ,     A     :    Optional[int]  ):




						'''simple docstring'''






						UpperCAmelCase         =			str(shared_datadir / '''test_image_rgb.jpg'''  )
						UpperCAmelCase         =			{'''image''': [image_path]}
						UpperCAmelCase         =			Features({'''image''': Image()}  )
						UpperCAmelCase         =			Dataset.from_dict(A   ,     features=A  )
						UpperCAmelCase         =			ParquetDatasetWriter(A   ,     tmp_path / '''foo.parquet'''  )
						assert writer.write() > 0

						UpperCAmelCase         =			Dataset.from_parquet(str(tmp_path / '''foo.parquet'''  )  )
						assert dataset.features == reloaded_dataset.features

						UpperCAmelCase         =			ParquetDatasetReader(str(tmp_path / '''foo.parquet'''  )   ,     streaming=A  ).read()
						assert dataset.features == reloaded_iterable_dataset.features




@pytest.mark.parametrize(
    '''feature, expected'''   ,     [
        (Features({'''foo''': Value('''int32'''  )}  ), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''  )}  ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()  )}  ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]   ,     )
def    lowerCamelCase__   ( A     :    str   ,     A     :    int  ):




						'''simple docstring'''






						assert get_writer_batch_size(A  ) == expected









'''simple docstring'''



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
	import torch

	from transformers.generation import DisjunctiveConstraint






@require_torch
class  UpperCamelCase__(     unittest.TestCase ):




				def   a__(	self    :				Optional[int]   )->  str:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2, 4], [1, 2, 3, 4]]
										UpperCAmelCase         =			DisjunctiveConstraint(lowerCAmelCase   )
										self.assertTrue(isinstance(dc.token_ids   ,					lowerCAmelCase   )   )

										with self.assertRaises(lowerCAmelCase   ):
																DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]   )   )

										with self.assertRaises(lowerCAmelCase   ):
																DisjunctiveConstraint([torch.LongTensor([1, 2, 4]   ), torch.LongTensor([1, 2, 3, 4, 5]   )]   )




				def   a__(	self    :				Union[str, Any]   )->  str:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2], [1, 2, 3, 4]]

										with self.assertRaises(lowerCAmelCase   ):
																DisjunctiveConstraint(lowerCAmelCase   )  # fails here




				def   a__(	self    :				Any   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2, 3], [1, 2, 4]]

										UpperCAmelCase         =			DisjunctiveConstraint(lowerCAmelCase   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(1   )
										UpperCAmelCase         =			stepped is True and completed is False and reset is False
										self.assertTrue(lowerCAmelCase   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(2   )
										UpperCAmelCase         =			stepped is True and completed is False and reset is False
										self.assertTrue(lowerCAmelCase   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1, 2]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(3   )
										UpperCAmelCase         =			stepped is True and completed is True and reset is False
										self.assertTrue(lowerCAmelCase   )
										self.assertTrue(dc.completed   )  # Completed!
										self.assertTrue(dc.current_seq == [1, 2, 3]   )





				def   a__(	self    :				int   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

										UpperCAmelCase         =			DisjunctiveConstraint(lowerCAmelCase   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(1   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(2   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1, 2]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(4   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1, 2, 4]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(5   )
										self.assertTrue(dc.completed   )  # Completed!
										self.assertTrue(dc.current_seq == [1, 2, 4, 5]   )

										dc.reset()

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(1   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.remaining() == 3   )
										self.assertTrue(dc.current_seq == [1]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(2   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.remaining() == 2   )
										self.assertTrue(dc.current_seq == [1, 2]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(5   )
										self.assertTrue(dc.completed   )  # Completed!
										self.assertTrue(dc.remaining() == 0   )
										self.assertTrue(dc.current_seq == [1, 2, 5]   )





