



'''simple docstring'''





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase    ):
			assert isinstance(_lowercase ,    _lowercase    )
			assert dataset.num_rows == 4
			assert dataset.num_columns == 3
			assert dataset.column_names == ["col_1", "col_2", "col_3"]
			for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("keep_in_memory" ,    [False, True]    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):
			A							:					Union[str, Any]              =						tmp_path / "cache"
			A							:					Union[str, Any]              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
						A							:					Any              =						JsonDatasetReader(_lowercase ,    cache_dir=_lowercase ,    keep_in_memory=_lowercase    ).read()
			_check_json_dataset(_lowercase ,    _lowercase    )




@pytest.mark.parametrize(
    "features" ,    [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ] ,    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):
			A							:					Optional[Any]              =						tmp_path / "cache"
			A							:					List[str]              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			A							:					List[str]              =						features.copy() if features else default_expected_features
			A							:					int              =						(
			    Features({feature: Value(_lowercase    ) for feature, dtype in features.items()}    ) if features is not None else None
			)
			A							:					Optional[Any]              =						JsonDatasetReader(_lowercase ,    features=_lowercase ,    cache_dir=_lowercase    ).read()
			_check_json_dataset(_lowercase ,    _lowercase    )




@pytest.mark.parametrize(
    "features" ,    [
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ] ,    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):
			A							:					str              =						tmp_path / "cache"
			A							:					List[Any]              =						{"col_3": "float64", "col_1": "string", "col_2": "int64"}
			A							:					Tuple              =						features.copy() if features else default_expected_features
			A							:					Union[str, Any]              =						(
			    Features({feature: Value(_lowercase    ) for feature, dtype in features.items()}    ) if features is not None else None
			)
			A							:					Optional[int]              =						JsonDatasetReader(_lowercase ,    features=_lowercase ,    cache_dir=_lowercase    ).read()
			assert isinstance(_lowercase ,    _lowercase    )
			assert dataset.num_rows == 2
			assert dataset.num_columns == 3
			assert dataset.column_names == ["col_3", "col_1", "col_2"]
			for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype




def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase    ):
			A							:					str              =						{"col_2": "int64", "col_3": "float64", "col_1": "string"}
			A							:					List[str]              =						features.copy()
			A							:					int              =						(
			    Features({feature: Value(_lowercase    ) for feature, dtype in features.items()}    ) if features is not None else None
			)
			A							:					Union[str, Any]              =						tmp_path / "cache"
			A							:					Any              =						JsonDatasetReader(_lowercase ,    features=_lowercase ,    cache_dir=_lowercase    ).read()
			assert isinstance(_lowercase ,    _lowercase    )
			assert dataset.num_rows == 2
			assert dataset.num_columns == 3
			assert dataset.column_names == ["col_2", "col_3", "col_1"]
			for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("split" ,    [None, NamedSplit("train"    ), "train", "test"]    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):
			A							:					List[str]              =						tmp_path / "cache"
			A							:					Tuple              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			A							:					Any              =						JsonDatasetReader(_lowercase ,    cache_dir=_lowercase ,    split=_lowercase    ).read()
			_check_json_dataset(_lowercase ,    _lowercase    )
			assert dataset.split == split if split else "train"




@pytest.mark.parametrize("path_type" ,    [str, list]    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):







			if issubclass(_lowercase ,    _lowercase    ):
						A							:					Optional[Any]              =						jsonl_path
			elif issubclass(_lowercase ,    _lowercase    ):
						A							:					List[str]              =						[jsonl_path]
			A							:					int              =						tmp_path / "cache"
			A							:					List[str]              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			A							:					Union[str, Any]              =						JsonDatasetReader(_lowercase ,    cache_dir=_lowercase    ).read()
			_check_json_dataset(_lowercase ,    _lowercase    )




def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase=("train",)    ):
			assert isinstance(_lowercase ,    _lowercase    )
			for split in splits:
						A							:					str              =						dataset_dict[split]
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
									assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("keep_in_memory" ,    [False, True]    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):
			A							:					Union[str, Any]              =						tmp_path / "cache"
			A							:					Tuple              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
						A							:					Union[str, Any]              =						JsonDatasetReader({"train": jsonl_path} ,    cache_dir=_lowercase ,    keep_in_memory=_lowercase    ).read()
			_check_json_datasetdict(_lowercase ,    _lowercase    )




@pytest.mark.parametrize(
    "features" ,    [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ] ,    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):
			A							:					Tuple              =						tmp_path / "cache"
			A							:					List[str]              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			A							:					Any              =						features.copy() if features else default_expected_features
			A							:					Any              =						(
			    Features({feature: Value(_lowercase    ) for feature, dtype in features.items()}    ) if features is not None else None
			)
			A							:					List[Any]              =						JsonDatasetReader({"train": jsonl_path} ,    features=_lowercase ,    cache_dir=_lowercase    ).read()
			_check_json_datasetdict(_lowercase ,    _lowercase    )




@pytest.mark.parametrize("split" ,    [None, NamedSplit("train"    ), "train", "test"]    )
def 	UpperCAmelCase   (						_lowerCamelCase ,    _lowerCamelCase ,    _lowerCamelCase    ):







			if split:
						A							:					Union[str, Any]              =						{split: jsonl_path}
			else:
						A							:					str              =						"train"
						A							:					List[Any]              =						{"train": jsonl_path, "test": jsonl_path}
			A							:					Union[str, Any]              =						tmp_path / "cache"
			A							:					Dict              =						{"col_1": "string", "col_2": "int64", "col_3": "float64"}
			A							:					Optional[Any]              =						JsonDatasetReader(_lowercase ,    cache_dir=_lowercase    ).read()
			_check_json_datasetdict(_lowercase ,    _lowercase ,    splits=list(path.keys()    )    )
			assert all(dataset[split].split == split for split in path.keys()    )




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			return json.load(_lowercase    )




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			return [json.loads(_lowercase    ) for line in buffer]






class 	lowerCamelCase_    :






						'''simple docstring'''







						@pytest.mark.parametrize("lines, load_json_function"	,	[(True, load_json_lines), (False, load_json)] )
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[Any]	,	__lowerCamelCase      :			Tuple	,	__lowerCamelCase      :			Any	,	__lowerCamelCase      :			Optional[int] )      ->				List[str]:
									with io.BytesIO() as buffer:
												JsonDatasetWriter(__A	,	__A	,	lines=__A ).write()
												buffer.seek(0 )
												A							:					List[str]              =						load_json_function(__A )
									assert isinstance(__A	,	__A )
									assert isinstance(exported_content[0]	,	__A )
									assert len(__A ) == 10







						@pytest.mark.parametrize(
						    "orient, container, keys, len_at"	,	[
						        ("records", list, {"tokens", "labels", "answers", "id"}, None),
						        ("split", dict, {"columns", "data"}, "data"),
						        ("index", dict, set("0123456789" ), None),
						        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
						        ("values", list, None, None),
						        ("table", dict, {"schema", "data"}, "data"),
						    ]	,	)
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[Any]	,	__lowerCamelCase      :			Union[str, Any]	,	__lowerCamelCase      :			List[Any]	,	__lowerCamelCase      :			Union[str, Any]	,	__lowerCamelCase      :			Optional[int]	,	__lowerCamelCase      :			List[Any] )      ->				Union[str, Any]:
									with io.BytesIO() as buffer:
												JsonDatasetWriter(__A	,	__A	,	lines=__A	,	orient=__A ).write()
												buffer.seek(0 )
												A							:					Tuple              =						load_json(__A )
									assert isinstance(__A	,	__A )
									if keys:
												if container is dict:
															assert exported_content.keys() == keys
												else:
															assert exported_content[0].keys() == keys
									else:
												assert not hasattr(__A	,	"keys" ) and not hasattr(exported_content[0]	,	"keys" )
									if len_at:
												assert len(exported_content[len_at] ) == 10
									else:
												assert len(__A ) == 10







						@pytest.mark.parametrize("lines, load_json_function"	,	[(True, load_json_lines), (False, load_json)] )
						def 			SCREAMING_SNAKE_CASE__		(	self      :			int	,	__lowerCamelCase      :			Union[str, Any]	,	__lowerCamelCase      :			Union[str, Any]	,	__lowerCamelCase      :			Optional[Any] )      ->				Tuple:
									with io.BytesIO() as buffer:
												JsonDatasetWriter(__A	,	__A	,	lines=__A	,	num_proc=2 ).write()
												buffer.seek(0 )
												A							:					Any              =						load_json_function(__A )
									assert isinstance(__A	,	__A )
									assert isinstance(exported_content[0]	,	__A )
									assert len(__A ) == 10







						@pytest.mark.parametrize(
						    "orient, container, keys, len_at"	,	[
						        ("records", list, {"tokens", "labels", "answers", "id"}, None),
						        ("split", dict, {"columns", "data"}, "data"),
						        ("index", dict, set("0123456789" ), None),
						        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
						        ("values", list, None, None),
						        ("table", dict, {"schema", "data"}, "data"),
						    ]	,	)
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Tuple	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			List[str]	,	__lowerCamelCase      :			List[Any]	,	__lowerCamelCase      :			List[Any]	,	__lowerCamelCase      :			List[Any] )      ->				Union[str, Any]:
									with io.BytesIO() as buffer:
												JsonDatasetWriter(__A	,	__A	,	lines=__A	,	orient=__A	,	num_proc=2 ).write()
												buffer.seek(0 )
												A							:					Optional[int]              =						load_json(__A )
									assert isinstance(__A	,	__A )
									if keys:
												if container is dict:
															assert exported_content.keys() == keys
												else:
															assert exported_content[0].keys() == keys
									else:
												assert not hasattr(__A	,	"keys" ) and not hasattr(exported_content[0]	,	"keys" )
									if len_at:
												assert len(exported_content[len_at] ) == 10
									else:
												assert len(__A ) == 10







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Tuple	,	__lowerCamelCase      :			Optional[int] )      ->				Optional[int]:
									with pytest.raises(__A ):
												with io.BytesIO() as buffer:
															JsonDatasetWriter(__A	,	__A	,	num_proc=0 )







						@pytest.mark.parametrize("compression, extension"	,	[("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")] )
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Dict	,	__lowerCamelCase      :			Tuple	,	__lowerCamelCase      :			Dict	,	__lowerCamelCase      :			Tuple	,	__lowerCamelCase      :			Optional[Any]	,	__lowerCamelCase      :			int )      ->				Optional[Any]:
									A							:					int              =						tmp_path_factory.mktemp("data" ) / F"""test.json.{extension}"""
									A							:					Tuple              =						str(shared_datadir / F"""test_file.json.{extension}""" )
									JsonDatasetWriter(__A	,	__A	,	compression=__A ).write()

									with fsspec.open(__A	,	"rb"	,	compression="infer" ) as f:
												A							:					Dict              =						f.read()
									with fsspec.open(__A	,	"rb"	,	compression="infer" ) as f:
												A							:					List[Any]              =						f.read()
									assert exported_content == original_content




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
							import torch

							from transformers import XLMRobertaModel

@require_sentencepiece
@require_tokenizers
@require_torch
class 	lowerCamelCase_    ( unittest.TestCase						):






						'''simple docstring'''







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Any )      ->				Union[str, Any]:
									A							:					Union[str, Any]              =						XLMRobertaModel.from_pretrained("xlm-roberta-base" )
									A							:					Tuple              =						torch.tensor([[0, 5_81, 1_02_69, 83, 9_99_42, 1_36, 6_07_42, 23, 70, 8_05_83, 1_82_76, 2]] )
									# The dog is cute and lives in the garden house

									A							:					Tuple              =						torch.Size((1, 12, 7_68) )  # batch_size, sequence_length, embedding_vector_dim
									A							:					List[str]              =						torch.tensor(
									    [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]] )
									#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
									#  xlmr.eval()
									#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
									with torch.no_grad():
												A							:					Tuple              =						model(__lowerCamelCase )["last_hidden_state"].detach()
									self.assertEqual(output.shape	,	__lowerCamelCase )
									# compare the actual values for a slice of last dim
									self.assertTrue(torch.allclose(output[:, :, -1]	,	__lowerCamelCase	,	atol=1e-3 ) )







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int] )      ->				int:
									A							:					str              =						XLMRobertaModel.from_pretrained("xlm-roberta-large" )
									A							:					List[Any]              =						torch.tensor([[0, 5_81, 1_02_69, 83, 9_99_42, 1_36, 6_07_42, 23, 70, 8_05_83, 1_82_76, 2]] )
									# The dog is cute and lives in the garden house

									A							:					Optional[Any]              =						torch.Size((1, 12, 10_24) )  # batch_size, sequence_length, embedding_vector_dim
									A							:					List[Any]              =						torch.tensor(
									    [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]] )
									#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
									#  xlmr.eval()
									#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
									with torch.no_grad():
												A							:					Optional[int]              =						model(__lowerCamelCase )["last_hidden_state"].detach()
									self.assertEqual(output.shape	,	__lowerCamelCase )
									# compare the actual values for a slice of last dim
									self.assertTrue(torch.allclose(output[:, :, -1]	,	__lowerCamelCase	,	atol=1e-3 ) )