





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases







def       UpperCAmelCase							(     _lowerCamelCase      :  List[Any]			,   _lowerCamelCase      :  Optional[int]     ):





						'''simple docstring'''


						assert isinstance(_lowerCamelCase			,   _lowerCamelCase     )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype







@pytest.mark.parametrize("keep_in_memory"			,   [False, True]     )
def       UpperCAmelCase							(     _lowerCamelCase      :  Dict			,   _lowerCamelCase      :  Optional[int]			,   _lowerCamelCase      :  str     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Tuple   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Any   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												SCREAMING_SNAKE_CASE__     :				List[Any]   =					JsonDatasetReader(_lowerCamelCase			,   cache_dir=_lowerCamelCase			,   keep_in_memory=_lowerCamelCase     ).read()
						_check_json_dataset(_lowerCamelCase			,   _lowerCamelCase     )







@pytest.mark.parametrize(
    "features"			,   [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]			,   )
def       UpperCAmelCase							(     _lowerCamelCase      :  Tuple			,   _lowerCamelCase      :  Union[str, Any]			,   _lowerCamelCase      :  Union[str, Any]     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Dict   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				List[str]   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					features.copy() if features else default_expected_features
						SCREAMING_SNAKE_CASE__     :				int   =					(
						    Features({feature: Value(_lowerCamelCase     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						SCREAMING_SNAKE_CASE__     :				Any   =					JsonDatasetReader(_lowerCamelCase			,   features=_lowerCamelCase			,   cache_dir=_lowerCamelCase     ).read()
						_check_json_dataset(_lowerCamelCase			,   _lowerCamelCase     )







@pytest.mark.parametrize(
    "features"			,   [
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]			,   )
def       UpperCAmelCase							(     _lowerCamelCase      :  int			,   _lowerCamelCase      :  str			,   _lowerCamelCase      :  List[Any]     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Dict   =					{"col_3": "float64", "col_1": "string", "col_2": "int64"}
						SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					features.copy() if features else default_expected_features
						SCREAMING_SNAKE_CASE__     :				str   =					(
						    Features({feature: Value(_lowerCamelCase     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						SCREAMING_SNAKE_CASE__     :				Any   =					JsonDatasetReader(_lowerCamelCase			,   features=_lowerCamelCase			,   cache_dir=_lowerCamelCase     ).read()
						assert isinstance(_lowerCamelCase			,   _lowerCamelCase     )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_3", "col_1", "col_2"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype







def       UpperCAmelCase							(     _lowerCamelCase      :  Optional[Any]			,   _lowerCamelCase      :  str     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				List[str]   =					{"col_2": "int64", "col_3": "float64", "col_1": "string"}
						SCREAMING_SNAKE_CASE__     :				int   =					features.copy()
						SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					(
						    Features({feature: Value(_lowerCamelCase     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Dict   =					JsonDatasetReader(_lowerCamelCase			,   features=_lowerCamelCase			,   cache_dir=_lowerCamelCase     ).read()
						assert isinstance(_lowerCamelCase			,   _lowerCamelCase     )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_2", "col_3", "col_1"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype







@pytest.mark.parametrize("split"			,   [None, NamedSplit("train"     ), "train", "test"]     )
def       UpperCAmelCase							(     _lowerCamelCase      :  str			,   _lowerCamelCase      :  Optional[int]			,   _lowerCamelCase      :  Optional[int]     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Dict   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					JsonDatasetReader(_lowerCamelCase			,   cache_dir=_lowerCamelCase			,   split=_lowerCamelCase     ).read()
						_check_json_dataset(_lowerCamelCase			,   _lowerCamelCase     )
						assert dataset.split == split if split else "train"







@pytest.mark.parametrize("path_type"			,   [str, list]     )
def       UpperCAmelCase							(     _lowerCamelCase      :  Any			,   _lowerCamelCase      :  Tuple			,   _lowerCamelCase      :  List[Any]     ):





						'''simple docstring'''


						if issubclass(_lowerCamelCase			,   _lowerCamelCase     ):
												SCREAMING_SNAKE_CASE__     :				List[Any]   =					jsonl_path
						elif issubclass(_lowerCamelCase			,   _lowerCamelCase     ):
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					[jsonl_path]
						SCREAMING_SNAKE_CASE__     :				Tuple   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Tuple   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					JsonDatasetReader(_lowerCamelCase			,   cache_dir=_lowerCamelCase     ).read()
						_check_json_dataset(_lowerCamelCase			,   _lowerCamelCase     )







def       UpperCAmelCase							(     _lowerCamelCase      :  Dict			,   _lowerCamelCase      :  Tuple			,   _lowerCamelCase      :  Union[str, Any]=("train",)     ):





						'''simple docstring'''


						assert isinstance(_lowerCamelCase			,   _lowerCamelCase     )
						for split in splits:
												SCREAMING_SNAKE_CASE__     :				List[str]   =					dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype







@pytest.mark.parametrize("keep_in_memory"			,   [False, True]     )
def       UpperCAmelCase							(     _lowerCamelCase      :  Optional[Any]			,   _lowerCamelCase      :  Dict			,   _lowerCamelCase      :  int     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				List[Any]   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Optional[int]   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					JsonDatasetReader({"train": jsonl_path}			,   cache_dir=_lowerCamelCase			,   keep_in_memory=_lowerCamelCase     ).read()
						_check_json_datasetdict(_lowerCamelCase			,   _lowerCamelCase     )







@pytest.mark.parametrize(
    "features"			,   [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]			,   )
def       UpperCAmelCase							(     _lowerCamelCase      :  str			,   _lowerCamelCase      :  Any			,   _lowerCamelCase      :  Tuple     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				str   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				List[Any]   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						SCREAMING_SNAKE_CASE__     :				Any   =					features.copy() if features else default_expected_features
						SCREAMING_SNAKE_CASE__     :				List[str]   =					(
						    Features({feature: Value(_lowerCamelCase     ) for feature, dtype in features.items()}     ) if features is not None else None
						)
						SCREAMING_SNAKE_CASE__     :				int   =					JsonDatasetReader({"train": jsonl_path}			,   features=_lowerCamelCase			,   cache_dir=_lowerCamelCase     ).read()
						_check_json_datasetdict(_lowerCamelCase			,   _lowerCamelCase     )







@pytest.mark.parametrize("split"			,   [None, NamedSplit("train"     ), "train", "test"]     )
def       UpperCAmelCase							(     _lowerCamelCase      :  str			,   _lowerCamelCase      :  Union[str, Any]			,   _lowerCamelCase      :  List[Any]     ):





						'''simple docstring'''


						if split:
												SCREAMING_SNAKE_CASE__     :				int   =					{split: jsonl_path}
						else:
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					"train"
												SCREAMING_SNAKE_CASE__     :				Dict   =					{"train": jsonl_path, "test": jsonl_path}
						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					tmp_path / "cache"
						SCREAMING_SNAKE_CASE__     :				Optional[int]   =					{"col_1": "string", "col_2": "int64", "col_3": "float64"}
						SCREAMING_SNAKE_CASE__     :				int   =					JsonDatasetReader(_lowerCamelCase			,   cache_dir=_lowerCamelCase     ).read()
						_check_json_datasetdict(_lowerCamelCase			,   _lowerCamelCase			,   splits=list(path.keys()     )     )
						assert all(dataset[split].split == split for split in path.keys()     )







def       UpperCAmelCase							(     _lowerCamelCase      :  Any     ):





						'''simple docstring'''


						return json.load(_lowerCamelCase     )







def       UpperCAmelCase							(     _lowerCamelCase      :  Any     ):





						'''simple docstring'''


						return [json.loads(_lowerCamelCase     ) for line in buffer]



class    _a     :






							"""simple docstring"""

							@pytest.mark.parametrize("lines, load_json_function"							,	[(True, load_json_lines), (False, load_json)]    )
							def      A_  (      self				:     List[Any]							,	a				:     Tuple							,	a				:     List[str]							,	a				:     Optional[int]    )	->Optional[Any]:
													with io.BytesIO() as buffer:
																			JsonDatasetWriter(a							,	a							,	lines=a    ).write()
																			buffer.seek(0    )
																			SCREAMING_SNAKE_CASE__     :				str   =					load_json_function(a    )
													assert isinstance(a							,	a    )
													assert isinstance(exported_content[0]							,	a    )
													assert len(a    ) == 10

							@pytest.mark.parametrize(
							    "orient, container, keys, len_at"							,	[
							        ("records", list, {"tokens", "labels", "answers", "id"}, None),
							        ("split", dict, {"columns", "data"}, "data"),
							        ("index", dict, set("0123456789"    ), None),
							        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
							        ("values", list, None, None),
							        ("table", dict, {"schema", "data"}, "data"),
							    ]							,	)
							def      A_  (      self				:     Optional[int]							,	a				:     List[Any]							,	a				:     Tuple							,	a				:     Optional[Any]							,	a				:     Optional[int]							,	a				:     List[str]    )	->str:
													with io.BytesIO() as buffer:
																			JsonDatasetWriter(a							,	a							,	lines=a							,	orient=a    ).write()
																			buffer.seek(0    )
																			SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					load_json(a    )
													assert isinstance(a							,	a    )
													if keys:
																			if container is dict:
																									assert exported_content.keys() == keys
																			else:
																									assert exported_content[0].keys() == keys
													else:
																			assert not hasattr(a							,	"keys"    ) and not hasattr(exported_content[0]							,	"keys"    )
													if len_at:
																			assert len(exported_content[len_at]    ) == 10
													else:
																			assert len(a    ) == 10

							@pytest.mark.parametrize("lines, load_json_function"							,	[(True, load_json_lines), (False, load_json)]    )
							def      A_  (      self				:     List[Any]							,	a				:     Optional[Any]							,	a				:     str							,	a				:     str    )	->Optional[Any]:
													with io.BytesIO() as buffer:
																			JsonDatasetWriter(a							,	a							,	lines=a							,	num_proc=2    ).write()
																			buffer.seek(0    )
																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					load_json_function(a    )
													assert isinstance(a							,	a    )
													assert isinstance(exported_content[0]							,	a    )
													assert len(a    ) == 10

							@pytest.mark.parametrize(
							    "orient, container, keys, len_at"							,	[
							        ("records", list, {"tokens", "labels", "answers", "id"}, None),
							        ("split", dict, {"columns", "data"}, "data"),
							        ("index", dict, set("0123456789"    ), None),
							        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
							        ("values", list, None, None),
							        ("table", dict, {"schema", "data"}, "data"),
							    ]							,	)
							def      A_  (      self				:     Optional[Any]							,	a				:     Optional[Any]							,	a				:     List[str]							,	a				:     Dict							,	a				:     Optional[Any]							,	a				:     Optional[Any]    )	->Any:
													with io.BytesIO() as buffer:
																			JsonDatasetWriter(a							,	a							,	lines=a							,	orient=a							,	num_proc=2    ).write()
																			buffer.seek(0    )
																			SCREAMING_SNAKE_CASE__     :				Dict   =					load_json(a    )
													assert isinstance(a							,	a    )
													if keys:
																			if container is dict:
																									assert exported_content.keys() == keys
																			else:
																									assert exported_content[0].keys() == keys
													else:
																			assert not hasattr(a							,	"keys"    ) and not hasattr(exported_content[0]							,	"keys"    )
													if len_at:
																			assert len(exported_content[len_at]    ) == 10
													else:
																			assert len(a    ) == 10

							def      A_  (      self				:     str							,	a				:     Dict    )	->Optional[Any]:
													with pytest.raises(a    ):
																			with io.BytesIO() as buffer:
																									JsonDatasetWriter(a							,	a							,	num_proc=0    )






							@pytest.mark.parametrize("compression, extension"							,	[("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]    )
							def      A_  (      self				:     Union[str, Any]							,	a				:     Dict							,	a				:     Union[str, Any]							,	a				:     List[Any]							,	a				:     List[Any]							,	a				:     Optional[Any]    )	->Tuple:
													SCREAMING_SNAKE_CASE__     :				Dict   =					tmp_path_factory.mktemp("data"    ) / f"""test.json.{extension}"""
													SCREAMING_SNAKE_CASE__     :				List[str]   =					str(shared_datadir / f"""test_file.json.{extension}"""    )
													JsonDatasetWriter(a							,	a							,	compression=a    ).write()

													with fsspec.open(a							,	"rb"							,	compression="infer"    ) as f:
																			SCREAMING_SNAKE_CASE__     :				Dict   =					f.read()
													with fsspec.open(a							,	"rb"							,	compression="infer"    ) as f:
																			SCREAMING_SNAKE_CASE__     :				Any   =					f.read()
													assert exported_content == original_content






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
				from ...onnx.config import PatchingSpec
				from ...tokenization_utils_base import PreTrainedTokenizerBase


__lowercase       :List[Any]       =  logging.get_logger(__name__)

__lowercase       :Optional[int]       =  {
    "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json",
    "allenai/longformer-large-4096": "https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json",
    "allenai/longformer-large-4096-finetuned-triviaqa": (
        "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json"
    ),
    "allenai/longformer-base-4096-extra.pos.embd.only": (
        "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json"
    ),
    "allenai/longformer-large-4096-extra.pos.embd.only": (
        "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json"
    ),
}



class    _a     (				lowercase__							):






							"""simple docstring"""

							snake_case_							 =						"longformer"

							def __init__(      self				:     List[str]							,	a				:     Union[List[int], int] = 5_12							,	a				:     int = 2							,	a				:     int = 1							,	a				:     int = 0							,	a				:     int = 2							,	a				:     int = 3_05_22							,	a				:     int = 7_68							,	a				:     int = 12							,	a				:     int = 12							,	a				:     int = 30_72							,	a				:     str = "gelu"							,	a				:     float = 0.1							,	a				:     float = 0.1							,	a				:     int = 5_12							,	a				:     int = 2							,	a				:     float = 0.02							,	a				:     float = 1E-12							,	a				:     bool = False							,	**a				:     Dict							,	)	->Tuple:
													super().__init__(pad_token_id=a							,	**a    )

													SCREAMING_SNAKE_CASE__     :				int   =					attention_window
													SCREAMING_SNAKE_CASE__     :				Any   =					sep_token_id
													SCREAMING_SNAKE_CASE__     :				str   =					bos_token_id
													SCREAMING_SNAKE_CASE__     :				List[str]   =					eos_token_id
													SCREAMING_SNAKE_CASE__     :				List[str]   =					vocab_size
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					hidden_size
													SCREAMING_SNAKE_CASE__     :				List[str]   =					num_hidden_layers
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					num_attention_heads
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					hidden_act
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					intermediate_size
													SCREAMING_SNAKE_CASE__     :				List[str]   =					hidden_dropout_prob
													SCREAMING_SNAKE_CASE__     :				Dict   =					attention_probs_dropout_prob
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					max_position_embeddings
													SCREAMING_SNAKE_CASE__     :				str   =					type_vocab_size
													SCREAMING_SNAKE_CASE__     :				Any   =					initializer_range
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					layer_norm_eps
													SCREAMING_SNAKE_CASE__     :				Any   =					onnx_export







class    _a     (				lowercase__							):






							"""simple docstring"""

							def __init__(      self				:     int							,	a				:     "PretrainedConfig"							,	a				:     str = "default"							,	a				:     "List[PatchingSpec]" = None    )	->str:
													super().__init__(a							,	a							,	a    )
													SCREAMING_SNAKE_CASE__     :				Any   =					True

							@property
							def      A_  (      self				:     int    )	->Mapping[str, Mapping[int, str]]:
													if self.task == "multiple-choice":
																			SCREAMING_SNAKE_CASE__     :				int   =					{0: "batch", 1: "choice", 2: "sequence"}
													else:
																			SCREAMING_SNAKE_CASE__     :				str   =					{0: "batch", 1: "sequence"}
													return OrderedDict(
													    [
													        ("input_ids", dynamic_axis),
													        ("attention_mask", dynamic_axis),
													        ("global_attention_mask", dynamic_axis),
													    ]    )

							@property
							def      A_  (      self				:     Optional[Any]    )	->Mapping[str, Mapping[int, str]]:
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					super().outputs
													if self.task == "default":
																			SCREAMING_SNAKE_CASE__     :				List[str]   =					{0: "batch"}
													return outputs

							@property
							def      A_  (      self				:     str    )	->float:
													return 1E-4

							@property
							def      A_  (      self				:     Any    )	->int:
													# needs to be >= 14 to support tril operator
													return max(super().default_onnx_opset							,	14    )






							def      A_  (      self				:     str							,	a				:     "PreTrainedTokenizerBase"							,	a				:     int = -1							,	a				:     int = -1							,	a				:     bool = False							,	a				:     Optional[TensorType] = None							,	)	->Mapping[str, Any]:
													SCREAMING_SNAKE_CASE__     :				Tuple   =					super().generate_dummy_inputs(
													    preprocessor=a							,	batch_size=a							,	seq_length=a							,	is_pair=a							,	framework=a    )
													import torch

													# for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
													# makes the export fail randomly
													SCREAMING_SNAKE_CASE__     :				Any   =					torch.zeros_like(inputs["input_ids"]    )
													# make every second token global
													SCREAMING_SNAKE_CASE__     :				str   =					1

													return inputs