



"""simple docstring"""





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def 	UpperCAmelCase     (		A	:						Optional[int]     ,				A	:						Union[str, Any]       ):

		'''simple docstring'''


		assert isinstance(A     ,				A       )
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('keep_in_memory'     ,				[False, True]       )
def 	UpperCAmelCase     (		A	:						Any     ,				A	:						Optional[int]     ,				A	:						Optional[int]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				_UpperCAmelCase							 =		JsonDatasetReader(A     ,				cache_dir=A     ,				keep_in_memory=A       ).read()
		_check_json_dataset(A     ,				A       )




@pytest.mark.parametrize(
    'features'     ,				[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]     ,				)
def 	UpperCAmelCase     (		A	:						Dict     ,				A	:						str     ,				A	:						List[str]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		_UpperCAmelCase							 =		features.copy() if features else default_expected_features
		_UpperCAmelCase							 =		(
		    Features({feature: Value(A       ) for feature, dtype in features.items()}       ) if features is not None else None
		)
		_UpperCAmelCase							 =		JsonDatasetReader(A     ,				features=A     ,				cache_dir=A       ).read()
		_check_json_dataset(A     ,				A       )




@pytest.mark.parametrize(
    'features'     ,				[
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ]     ,				)
def 	UpperCAmelCase     (		A	:						Union[str, Any]     ,				A	:						str     ,				A	:						List[Any]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
		_UpperCAmelCase							 =		features.copy() if features else default_expected_features
		_UpperCAmelCase							 =		(
		    Features({feature: Value(A       ) for feature, dtype in features.items()}       ) if features is not None else None
		)
		_UpperCAmelCase							 =		JsonDatasetReader(A     ,				features=A     ,				cache_dir=A       ).read()
		assert isinstance(A     ,				A       )
		assert dataset.num_rows == 2
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_3", "col_1", "col_2"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype




def 	UpperCAmelCase     (		A	:						Union[str, Any]     ,				A	:						Optional[int]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		{'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
		_UpperCAmelCase							 =		features.copy()
		_UpperCAmelCase							 =		(
		    Features({feature: Value(A       ) for feature, dtype in features.items()}       ) if features is not None else None
		)
		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		JsonDatasetReader(A     ,				features=A     ,				cache_dir=A       ).read()
		assert isinstance(A     ,				A       )
		assert dataset.num_rows == 2
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_2", "col_3", "col_1"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('split'     ,				[None, NamedSplit('train'       ), 'train', 'test']       )
def 	UpperCAmelCase     (		A	:						Union[str, Any]     ,				A	:						Any     ,				A	:						Optional[Any]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		_UpperCAmelCase							 =		JsonDatasetReader(A     ,				cache_dir=A     ,				split=A       ).read()
		_check_json_dataset(A     ,				A       )
		assert dataset.split == split if split else "train"




@pytest.mark.parametrize('path_type'     ,				[str, list]       )
def 	UpperCAmelCase     (		A	:						int     ,				A	:						int     ,				A	:						Dict       ):

		'''simple docstring'''


		if issubclass(A     ,				A       ):
				_UpperCAmelCase							 =		jsonl_path
		elif issubclass(A     ,				A       ):
				_UpperCAmelCase							 =		[jsonl_path]
		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		_UpperCAmelCase							 =		JsonDatasetReader(A     ,				cache_dir=A       ).read()
		_check_json_dataset(A     ,				A       )




def 	UpperCAmelCase     (		A	:						Tuple     ,				A	:						Optional[Any]     ,				A	:						Optional[int]=("train",)       ):

		'''simple docstring'''


		assert isinstance(A     ,				A       )
		for split in splits:
				_UpperCAmelCase							 =		dataset_dict[split]
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('keep_in_memory'     ,				[False, True]       )
def 	UpperCAmelCase     (		A	:						Optional[Any]     ,				A	:						Dict     ,				A	:						Union[str, Any]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				_UpperCAmelCase							 =		JsonDatasetReader({'train': jsonl_path}     ,				cache_dir=A     ,				keep_in_memory=A       ).read()
		_check_json_datasetdict(A     ,				A       )




@pytest.mark.parametrize(
    'features'     ,				[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]     ,				)
def 	UpperCAmelCase     (		A	:						Dict     ,				A	:						List[Any]     ,				A	:						Union[str, Any]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		_UpperCAmelCase							 =		features.copy() if features else default_expected_features
		_UpperCAmelCase							 =		(
		    Features({feature: Value(A       ) for feature, dtype in features.items()}       ) if features is not None else None
		)
		_UpperCAmelCase							 =		JsonDatasetReader({'train': jsonl_path}     ,				features=A     ,				cache_dir=A       ).read()
		_check_json_datasetdict(A     ,				A       )




@pytest.mark.parametrize('split'     ,				[None, NamedSplit('train'       ), 'train', 'test']       )
def 	UpperCAmelCase     (		A	:						List[Any]     ,				A	:						str     ,				A	:						int       ):

		'''simple docstring'''


		if split:
				_UpperCAmelCase							 =		{split: jsonl_path}
		else:
				_UpperCAmelCase							 =		'train'
				_UpperCAmelCase							 =		{'train': jsonl_path, 'test': jsonl_path}
		_UpperCAmelCase							 =		tmp_path / 'cache'
		_UpperCAmelCase							 =		{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
		_UpperCAmelCase							 =		JsonDatasetReader(A     ,				cache_dir=A       ).read()
		_check_json_datasetdict(A     ,				A     ,				splits=list(path.keys()       )       )
		assert all(dataset[split].split == split for split in path.keys()       )




def 	UpperCAmelCase     (		A	:						Dict       ):

		'''simple docstring'''


		return json.load(A       )




def 	UpperCAmelCase     (		A	:						Optional[Any]       ):

		'''simple docstring'''


		return [json.loads(A       ) for line in buffer]






class 			lowercase__			:
							'''simple docstring'''







							@pytest.mark.parametrize('lines, load_json_function'	,       [(True, load_json_lines), (False, load_json)]						)
							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case						)			->							Optional[Any]:
									with io.BytesIO() as buffer:
											JsonDatasetWriter(snake_case	,       snake_case	,       lines=snake_case						).write()
											buffer.seek(0						)
											_UpperCAmelCase							 =		load_json_function(snake_case						)
									assert isinstance(snake_case	,       snake_case						)
									assert isinstance(exported_content[0]	,       snake_case						)
									assert len(snake_case						) == 10






							@pytest.mark.parametrize(
							    'orient, container, keys, len_at'	,       [
							        ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
							        ('split', dict, {'columns', 'data'}, 'data'),
							        ('index', dict, set('0123456789'						), None),
							        ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
							        ('values', list, None, None),
							        ('table', dict, {'schema', 'data'}, 'data'),
							    ]	,       )
							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case	,       snake_case	,       snake_case						)			->							str:
									with io.BytesIO() as buffer:
											JsonDatasetWriter(snake_case	,       snake_case	,       lines=snake_case	,       orient=snake_case						).write()
											buffer.seek(0						)
											_UpperCAmelCase							 =		load_json(snake_case						)
									assert isinstance(snake_case	,       snake_case						)
									if keys:
											if container is dict:
													assert exported_content.keys() == keys
											else:
													assert exported_content[0].keys() == keys
									else:
											assert not hasattr(snake_case	,       'keys'						) and not hasattr(exported_content[0]	,       'keys'						)
									if len_at:
											assert len(exported_content[len_at]						) == 10
									else:
											assert len(snake_case						) == 10






							@pytest.mark.parametrize('lines, load_json_function'	,       [(True, load_json_lines), (False, load_json)]						)
							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case						)			->							str:
									with io.BytesIO() as buffer:
											JsonDatasetWriter(snake_case	,       snake_case	,       lines=snake_case	,       num_proc=2						).write()
											buffer.seek(0						)
											_UpperCAmelCase							 =		load_json_function(snake_case						)
									assert isinstance(snake_case	,       snake_case						)
									assert isinstance(exported_content[0]	,       snake_case						)
									assert len(snake_case						) == 10






							@pytest.mark.parametrize(
							    'orient, container, keys, len_at'	,       [
							        ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
							        ('split', dict, {'columns', 'data'}, 'data'),
							        ('index', dict, set('0123456789'						), None),
							        ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
							        ('values', list, None, None),
							        ('table', dict, {'schema', 'data'}, 'data'),
							    ]	,       )
							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case	,       snake_case	,       snake_case						)			->							Tuple:
									with io.BytesIO() as buffer:
											JsonDatasetWriter(snake_case	,       snake_case	,       lines=snake_case	,       orient=snake_case	,       num_proc=2						).write()
											buffer.seek(0						)
											_UpperCAmelCase							 =		load_json(snake_case						)
									assert isinstance(snake_case	,       snake_case						)
									if keys:
											if container is dict:
													assert exported_content.keys() == keys
											else:
													assert exported_content[0].keys() == keys
									else:
											assert not hasattr(snake_case	,       'keys'						) and not hasattr(exported_content[0]	,       'keys'						)
									if len_at:
											assert len(exported_content[len_at]						) == 10
									else:
											assert len(snake_case						) == 10






							def        lowerCamelCase_   (  self	,       snake_case						)			->							int:
									with pytest.raises(snake_case						):
											with io.BytesIO() as buffer:
													JsonDatasetWriter(snake_case	,       snake_case	,       num_proc=0						)






							@pytest.mark.parametrize('compression, extension'	,       [('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]						)
							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case	,       snake_case	,       snake_case						)			->							Tuple:
									_UpperCAmelCase							 =		tmp_path_factory.mktemp('data'						) / f'test.json.{extension}'
									_UpperCAmelCase							 =		str(shared_datadir / f'test_file.json.{extension}'						)
									JsonDatasetWriter(snake_case	,       snake_case	,       compression=snake_case						).write()

									with fsspec.open(snake_case	,       'rb'	,       compression='infer'						) as f:
											_UpperCAmelCase							 =		f.read()
									with fsspec.open(snake_case	,       'rb'	,       compression='infer'						) as f:
											_UpperCAmelCase							 =		f.read()
									assert exported_content == original_content







"""simple docstring"""





import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
					import torch




@is_pipeline_test
class 			lowercase__			(      unittest.TestCase    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
							_UpperCAmelCase      				=					TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING






							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case						)			->							Dict:
									_UpperCAmelCase							 =		TextaTextGenerationPipeline(model=snake_case	,       tokenizer=snake_case						)
									return generator, ["Something to write", "Something else"]






							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case						)			->							Dict:
									_UpperCAmelCase							 =		generator('Something there'						)
									self.assertEqual(snake_case	,       [{'generated_text': ANY(snake_case						)}]						)
									# These are encoder decoder, they don't just append to incoming string
									self.assertFalse(outputs[0]['generated_text'].startswith('Something there'						)						)

									_UpperCAmelCase							 =		generator(['This is great !', 'Something else']	,       num_return_sequences=2	,       do_sample=snake_case						)
									self.assertEqual(
									    snake_case	,       [
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									    ]	,       )

									_UpperCAmelCase							 =		generator(
									    ['This is great !', 'Something else']	,       num_return_sequences=2	,       batch_size=2	,       do_sample=snake_case						)
									self.assertEqual(
									    snake_case	,       [
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									    ]	,       )

									with self.assertRaises(snake_case						):
											generator(4						)






							@require_torch
							def        lowerCamelCase_   (  self						)			->							Dict:
									_UpperCAmelCase							 =		pipeline('text2text-generation'	,       model='patrickvonplaten/t5-tiny-random'	,       framework='pt'						)
									# do_sample=False necessary for reproducibility
									_UpperCAmelCase							 =		generator('Something there'	,       do_sample=snake_case						)
									self.assertEqual(snake_case	,       [{'generated_text': ''}]						)

									_UpperCAmelCase							 =		3
									_UpperCAmelCase							 =		generator(
									    'Something there'	,       num_return_sequences=snake_case	,       num_beams=snake_case	,       )
									_UpperCAmelCase							 =		[
									    {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide Beide'},
									    {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide'},
									    {'generated_text': ''},
									]
									self.assertEqual(snake_case	,       snake_case						)

									_UpperCAmelCase							 =		generator('This is a test'	,       do_sample=snake_case	,       num_return_sequences=2	,       return_tensors=snake_case						)
									self.assertEqual(
									    snake_case	,       [
									        {'generated_token_ids': ANY(torch.Tensor						)},
									        {'generated_token_ids': ANY(torch.Tensor						)},
									    ]	,       )
									_UpperCAmelCase							 =		generator.model.config.eos_token_id
									_UpperCAmelCase							 =		'<pad>'
									_UpperCAmelCase							 =		generator(
									    ['This is a test', 'This is a second test']	,       do_sample=snake_case	,       num_return_sequences=2	,       batch_size=2	,       return_tensors=snake_case	,       )
									self.assertEqual(
									    snake_case	,       [
									        [
									            {'generated_token_ids': ANY(torch.Tensor						)},
									            {'generated_token_ids': ANY(torch.Tensor						)},
									        ],
									        [
									            {'generated_token_ids': ANY(torch.Tensor						)},
									            {'generated_token_ids': ANY(torch.Tensor						)},
									        ],
									    ]	,       )






							@require_tf
							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		pipeline('text2text-generation'	,       model='patrickvonplaten/t5-tiny-random'	,       framework='tf'						)
									# do_sample=False necessary for reproducibility
									_UpperCAmelCase							 =		generator('Something there'	,       do_sample=snake_case						)
									self.assertEqual(snake_case	,       [{'generated_text': ''}]						)


