

import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 		UpperCamelCase      (lowercase_:				Dict					,   lowercase_:				Union[str, Any]      )  ->   Union[str, Any]:
	assert isinstance(lowercase_					,   lowercase_      )
	assert dataset.num_rows == 4
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_1", "col_2", "col_3"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory"""					,   [False, True]      )
def 		UpperCamelCase      (lowercase_:				Dict					,   lowercase_:				Optional[int]					,   lowercase_:				Tuple      )  ->   Optional[int]:
	A__							:					Dict       =	tmp_path / """cache"""
	A__							:					str       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		A__							:					Union[str, Any]       =	JsonDatasetReader(lowercase_					,   cache_dir=lowercase_					,   keep_in_memory=lowercase_      ).read()
	_check_json_dataset(lowercase_					,   lowercase_      )

@pytest.mark.parametrize(
    """features"""					,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,   )
def 		UpperCamelCase      (lowercase_:				int					,   lowercase_:				Optional[int]					,   lowercase_:				List[str]      )  ->   int:
	A__							:					Union[str, Any]       =	tmp_path / """cache"""
	A__							:					Union[str, Any]       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:					Any       =	features.copy() if features else default_expected_features
	A__							:					List[Any]       =	(
	    Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
	)
	A__							:					List[Any]       =	JsonDatasetReader(lowercase_					,   features=lowercase_					,   cache_dir=lowercase_      ).read()
	_check_json_dataset(lowercase_					,   lowercase_      )

@pytest.mark.parametrize(
    """features"""					,   [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]					,   )
def 		UpperCamelCase      (lowercase_:				str					,   lowercase_:				List[Any]					,   lowercase_:				List[str]      )  ->   Dict:
	A__							:					Optional[int]       =	tmp_path / """cache"""
	A__							:					Optional[int]       =	{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
	A__							:					Optional[int]       =	features.copy() if features else default_expected_features
	A__							:					int       =	(
	    Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
	)
	A__							:					Union[str, Any]       =	JsonDatasetReader(lowercase_					,   features=lowercase_					,   cache_dir=lowercase_      ).read()
	assert isinstance(lowercase_					,   lowercase_      )
	assert dataset.num_rows == 2
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_3", "col_1", "col_2"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype

def 		UpperCamelCase      (lowercase_:				int					,   lowercase_:				Tuple      )  ->   int:
	# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
	A__							:					str       =	{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
	A__							:					Optional[int]       =	features.copy()
	A__							:					Optional[int]       =	(
	    Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
	)
	A__							:					Union[str, Any]       =	tmp_path / """cache"""
	A__							:					Dict       =	JsonDatasetReader(lowercase_					,   features=lowercase_					,   cache_dir=lowercase_      ).read()
	assert isinstance(lowercase_					,   lowercase_      )
	assert dataset.num_rows == 2
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_2", "col_3", "col_1"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""split"""					,   [None, NamedSplit("""train"""      ), """train""", """test"""]      )
def 		UpperCamelCase      (lowercase_:				Dict					,   lowercase_:				Dict					,   lowercase_:				Optional[Any]      )  ->   Optional[Any]:
	A__							:					Union[str, Any]       =	tmp_path / """cache"""
	A__							:					Tuple       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:					List[str]       =	JsonDatasetReader(lowercase_					,   cache_dir=lowercase_					,   split=lowercase_      ).read()
	_check_json_dataset(lowercase_					,   lowercase_      )
	assert dataset.split == split if split else "train"

@pytest.mark.parametrize("""path_type"""					,   [str, list]      )
def 		UpperCamelCase      (lowercase_:				str					,   lowercase_:				Union[str, Any]					,   lowercase_:				List[str]      )  ->   List[Any]:
	if issubclass(lowercase_					,   lowercase_      ):
		A__							:					Optional[int]       =	jsonl_path
	elif issubclass(lowercase_					,   lowercase_      ):
		A__							:					Tuple       =	[jsonl_path]
	A__							:					int       =	tmp_path / """cache"""
	A__							:					Tuple       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:					Tuple       =	JsonDatasetReader(lowercase_					,   cache_dir=lowercase_      ).read()
	_check_json_dataset(lowercase_					,   lowercase_      )

def 		UpperCamelCase      (lowercase_:				int					,   lowercase_:				Dict					,   lowercase_:				Any=("train",)      )  ->   Optional[int]:
	assert isinstance(lowercase_					,   lowercase_      )
	for split in splits:
		A__							:					Optional[Any]       =	dataset_dict[split]
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
			assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory"""					,   [False, True]      )
def 		UpperCamelCase      (lowercase_:				Optional[Any]					,   lowercase_:				Any					,   lowercase_:				int      )  ->   Dict:
	A__							:					int       =	tmp_path / """cache"""
	A__							:					Optional[int]       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		A__							:					List[str]       =	JsonDatasetReader({"""train""": jsonl_path}					,   cache_dir=lowercase_					,   keep_in_memory=lowercase_      ).read()
	_check_json_datasetdict(lowercase_					,   lowercase_      )

@pytest.mark.parametrize(
    """features"""					,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,   )
def 		UpperCamelCase      (lowercase_:				str					,   lowercase_:				Dict					,   lowercase_:				Optional[int]      )  ->   int:
	A__							:					List[Any]       =	tmp_path / """cache"""
	A__							:					List[str]       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:					str       =	features.copy() if features else default_expected_features
	A__							:					Optional[int]       =	(
	    Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
	)
	A__							:					str       =	JsonDatasetReader({"""train""": jsonl_path}					,   features=lowercase_					,   cache_dir=lowercase_      ).read()
	_check_json_datasetdict(lowercase_					,   lowercase_      )

@pytest.mark.parametrize("""split"""					,   [None, NamedSplit("""train"""      ), """train""", """test"""]      )
def 		UpperCamelCase      (lowercase_:				Union[str, Any]					,   lowercase_:				Any					,   lowercase_:				int      )  ->   List[str]:
	if split:
		A__							:					List[Any]       =	{split: jsonl_path}
	else:
		A__							:					List[str]       =	"""train"""
		A__							:					Optional[Any]       =	{"""train""": jsonl_path, """test""": jsonl_path}
	A__							:					Any       =	tmp_path / """cache"""
	A__							:					Any       =	{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	A__							:					List[Any]       =	JsonDatasetReader(lowercase_					,   cache_dir=lowercase_      ).read()
	_check_json_datasetdict(lowercase_					,   lowercase_					,   splits=list(path.keys()      )      )
	assert all(dataset[split].split == split for split in path.keys()      )

def 		UpperCamelCase      (lowercase_:				Optional[int]      )  ->   Optional[Any]:
	return json.load(lowercase_      )

def 		UpperCamelCase      (lowercase_:				Union[str, Any]      )  ->   Optional[int]:
	return [json.loads(lowercase_      ) for line in buffer]



class 					_a		:


					'''simple docstring'''






					@pytest.mark.parametrize("""lines, load_json_function"""					, [(True, load_json_lines), (False, load_json)]    )
					def 						__A    (				self					, A__					, A__					, A__    ):
						with io.BytesIO() as buffer:
							JsonDatasetWriter(A__					, A__					, lines=A__    ).write()
							buffer.seek(0    )
							A__							:					Union[str, Any]       =	load_json_function(A__    )
						assert isinstance(A__					, A__    )
						assert isinstance(exported_content[0]					, A__    )
						assert len(A__    ) == 10






					@pytest.mark.parametrize(
					    """orient, container, keys, len_at"""					, [
					        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
					        ("""split""", dict, {"""columns""", """data"""}, """data"""),
					        ("""index""", dict, set("""0123456789"""    ), None),
					        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
					        ("""values""", list, None, None),
					        ("""table""", dict, {"""schema""", """data"""}, """data"""),
					    ]					, )
					def 						__A    (				self					, A__					, A__					, A__					, A__					, A__    ):
						with io.BytesIO() as buffer:
							JsonDatasetWriter(A__					, A__					, lines=A__					, orient=A__    ).write()
							buffer.seek(0    )
							A__							:					Any       =	load_json(A__    )
						assert isinstance(A__					, A__    )
						if keys:
							if container is dict:
								assert exported_content.keys() == keys
							else:
								assert exported_content[0].keys() == keys
						else:
							assert not hasattr(A__					, """keys"""    ) and not hasattr(exported_content[0]					, """keys"""    )
						if len_at:
							assert len(exported_content[len_at]    ) == 10
						else:
							assert len(A__    ) == 10






					@pytest.mark.parametrize("""lines, load_json_function"""					, [(True, load_json_lines), (False, load_json)]    )
					def 						__A    (				self					, A__					, A__					, A__    ):
						with io.BytesIO() as buffer:
							JsonDatasetWriter(A__					, A__					, lines=A__					, num_proc=2    ).write()
							buffer.seek(0    )
							A__							:					List[str]       =	load_json_function(A__    )
						assert isinstance(A__					, A__    )
						assert isinstance(exported_content[0]					, A__    )
						assert len(A__    ) == 10






					@pytest.mark.parametrize(
					    """orient, container, keys, len_at"""					, [
					        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
					        ("""split""", dict, {"""columns""", """data"""}, """data"""),
					        ("""index""", dict, set("""0123456789"""    ), None),
					        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
					        ("""values""", list, None, None),
					        ("""table""", dict, {"""schema""", """data"""}, """data"""),
					    ]					, )
					def 						__A    (				self					, A__					, A__					, A__					, A__					, A__    ):
						with io.BytesIO() as buffer:
							JsonDatasetWriter(A__					, A__					, lines=A__					, orient=A__					, num_proc=2    ).write()
							buffer.seek(0    )
							A__							:					List[Any]       =	load_json(A__    )
						assert isinstance(A__					, A__    )
						if keys:
							if container is dict:
								assert exported_content.keys() == keys
							else:
								assert exported_content[0].keys() == keys
						else:
							assert not hasattr(A__					, """keys"""    ) and not hasattr(exported_content[0]					, """keys"""    )
						if len_at:
							assert len(exported_content[len_at]    ) == 10
						else:
							assert len(A__    ) == 10






					def 						__A    (				self					, A__    ):
						with pytest.raises(A__    ):
							with io.BytesIO() as buffer:
								JsonDatasetWriter(A__					, A__					, num_proc=0    )






					@pytest.mark.parametrize("""compression, extension"""					, [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]    )
					def 						__A    (				self					, A__					, A__					, A__					, A__					, A__    ):
						A__							:					Dict       =	tmp_path_factory.mktemp("""data"""    ) / F"""test.json.{extension}"""
						A__							:					Optional[int]       =	str(shared_datadir / F"""test_file.json.{extension}"""    )
						JsonDatasetWriter(A__					, A__					, compression=A__    ).write()

						with fsspec.open(A__					, """rb"""					, compression="""infer"""    ) as f:
							A__							:					Dict       =	f.read()
						with fsspec.open(A__					, """rb"""					, compression="""infer"""    ) as f:
							A__							:					Optional[Any]       =	f.read()
						assert exported_content == original_content






# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
A_ :				Optional[int]						 =    abspath(join(dirname(__file__), 'src'))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='ignore', category=FutureWarning)

def 		UpperCamelCase      (lowercase_:				List[str]      )  ->   Any:
	config.addinivalue_line(
	    """markers"""					,   """is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested"""      )
	config.addinivalue_line(
	    """markers"""					,   """is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested"""      )
	config.addinivalue_line("""markers"""					,   """is_pipeline_test: mark test to run only when pipelines are tested"""      )
	config.addinivalue_line("""markers"""					,   """is_staging_test: mark test to run only in the staging environment"""      )
	config.addinivalue_line("""markers"""					,   """accelerate_tests: mark test that require accelerate"""      )
	config.addinivalue_line("""markers"""					,   """tool_tests: mark the tool tests that are run on their specific schedule"""      )

def 		UpperCamelCase      (lowercase_:				Optional[int]      )  ->   Optional[Any]:
	from transformers.testing_utils import pytest_addoption_shared

	pytest_addoption_shared(lowercase_      )

def 		UpperCamelCase      (lowercase_:				List[str]      )  ->   Optional[Any]:
	from transformers.testing_utils import pytest_terminal_summary_main

	A__							:					List[Any]       =	terminalreporter.config.getoption("""--make-reports"""      )
	if make_reports:
		pytest_terminal_summary_main(lowercase_					,   id=lowercase_      )

def 		UpperCamelCase      (lowercase_:				Union[str, Any]					,   lowercase_:				int      )  ->   List[str]:
	# If no tests are collected, pytest exists with code 5, which makes the CI fail.
	if exitstatus == 5:
		A__							:					Tuple       =	0


# Doctest custom flag to ignore output.
A_ :				Tuple						 =    doctest.register_optionflag('IGNORE_RESULT')

A_ :				Dict						 =    doctest.OutputChecker



class 					_a		(__magic_name__       ):


					'''simple docstring'''






					def 						__A    (				self					, A__					, A__					, A__    ):
						if IGNORE_RESULT & optionflags:
							return True
						return OutputChecker.check_output(self					, A__					, A__					, A__    )



A_ :				str						 =    CustomOutputChecker
A_ :				Dict						 =    HfDoctestModule


A_ :				Optional[int]						 =    HfDocTestParser



