






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def   lowerCamelCase__			(     lowercase						,     lowercase				):

  """simple docstring"""






  assert isinstance(lowercase						,     lowercase				)
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"						,     [False, True]				)
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      Dict							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    SCREAMING_SNAKE_CASE       :      int							=    JsonDatasetReader(lowercase						,     cache_dir=lowercase						,     keep_in_memory=lowercase				).read()
  _check_json_dataset(lowercase						,     lowercase				)


@pytest.mark.parametrize(
    "features"						,     [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]						,     )
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Dict							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      List[Any]							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE       :      List[Any]							=    features.copy() if features else default_expected_features
  SCREAMING_SNAKE_CASE       :      List[Any]							=    (
      Features({feature: Value(lowercase				) for feature, dtype in features.items()}				) if features is not None else None
  )
  SCREAMING_SNAKE_CASE       :      Tuple							=    JsonDatasetReader(lowercase						,     features=lowercase						,     cache_dir=lowercase				).read()
  _check_json_dataset(lowercase						,     lowercase				)


@pytest.mark.parametrize(
    "features"						,     [
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]						,     )
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      List[str]							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      Tuple							=    {"col_3": "float64", "col_1": "string", "col_2": "int64"}
  SCREAMING_SNAKE_CASE       :      List[str]							=    features.copy() if features else default_expected_features
  SCREAMING_SNAKE_CASE       :      Dict							=    (
      Features({feature: Value(lowercase				) for feature, dtype in features.items()}				) if features is not None else None
  )
  SCREAMING_SNAKE_CASE       :      List[Any]							=    JsonDatasetReader(lowercase						,     features=lowercase						,     cache_dir=lowercase				).read()
  assert isinstance(lowercase						,     lowercase				)
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_3", "col_1", "col_2"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype


def   lowerCamelCase__			(     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    {"col_2": "int64", "col_3": "float64", "col_1": "string"}
  SCREAMING_SNAKE_CASE       :      Any							=    features.copy()
  SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    (
      Features({feature: Value(lowercase				) for feature, dtype in features.items()}				) if features is not None else None
  )
  SCREAMING_SNAKE_CASE       :      str							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      Any							=    JsonDatasetReader(lowercase						,     features=lowercase						,     cache_dir=lowercase				).read()
  assert isinstance(lowercase						,     lowercase				)
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_2", "col_3", "col_1"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("split"						,     [None, NamedSplit("train"				), "train", "test"]				)
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      Dict							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    JsonDatasetReader(lowercase						,     cache_dir=lowercase						,     split=lowercase				).read()
  _check_json_dataset(lowercase						,     lowercase				)
  assert dataset.split == split if split else "train"


@pytest.mark.parametrize("path_type"						,     [str, list]				)
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  if issubclass(lowercase						,     lowercase				):
    SCREAMING_SNAKE_CASE       :      Any							=    jsonl_path
  elif issubclass(lowercase						,     lowercase				):
    SCREAMING_SNAKE_CASE       :      Optional[int]							=    [jsonl_path]
  SCREAMING_SNAKE_CASE       :      Tuple							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      Any							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE       :      List[str]							=    JsonDatasetReader(lowercase						,     cache_dir=lowercase				).read()
  _check_json_dataset(lowercase						,     lowercase				)


def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase=("train",)				):

  """simple docstring"""






  assert isinstance(lowercase						,     lowercase				)
  for split in splits:
    SCREAMING_SNAKE_CASE       :      Dict							=    dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"						,     [False, True]				)
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      List[str]							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    SCREAMING_SNAKE_CASE       :      Tuple							=    JsonDatasetReader({"train": jsonl_path}						,     cache_dir=lowercase						,     keep_in_memory=lowercase				).read()
  _check_json_datasetdict(lowercase						,     lowercase				)


@pytest.mark.parametrize(
    "features"						,     [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]						,     )
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      str							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      str							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE       :      Dict							=    features.copy() if features else default_expected_features
  SCREAMING_SNAKE_CASE       :      Tuple							=    (
      Features({feature: Value(lowercase				) for feature, dtype in features.items()}				) if features is not None else None
  )
  SCREAMING_SNAKE_CASE       :      Tuple							=    JsonDatasetReader({"train": jsonl_path}						,     features=lowercase						,     cache_dir=lowercase				).read()
  _check_json_datasetdict(lowercase						,     lowercase				)


@pytest.mark.parametrize("split"						,     [None, NamedSplit("train"				), "train", "test"]				)
def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase				):

  """simple docstring"""






  if split:
    SCREAMING_SNAKE_CASE       :      int							=    {split: jsonl_path}
  else:
    SCREAMING_SNAKE_CASE       :      str							=    "train"
    SCREAMING_SNAKE_CASE       :      List[str]							=    {"train": jsonl_path, "test": jsonl_path}
  SCREAMING_SNAKE_CASE       :      int							=    tmp_path / "cache"
  SCREAMING_SNAKE_CASE       :      Any							=    {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE       :      Optional[int]							=    JsonDatasetReader(lowercase						,     cache_dir=lowercase				).read()
  _check_json_datasetdict(lowercase						,     lowercase						,     splits=list(path.keys()				)				)
  assert all(dataset[split].split == split for split in path.keys()				)


def   lowerCamelCase__			(     lowercase				):

  """simple docstring"""






  return json.load(lowercase				)


def   lowerCamelCase__			(     lowercase				):

  """simple docstring"""






  return [json.loads(lowercase				) for line in buffer]


class  SCREAMING_SNAKE_CASE			:
     '''simple docstring'''


     @pytest.mark.parametrize("lines, load_json_function"					, [(True, load_json_lines), (False, load_json)]      )
     def 				_A       (				self       :	Union[str, Any]					, UpperCAmelCase_       :	Tuple					, UpperCAmelCase_       :	Optional[int]					, UpperCAmelCase_       :	Optional[Any]      ):
       with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCAmelCase_					, UpperCAmelCase_					, lines=UpperCAmelCase_      ).write()
         buffer.seek(0      )
         SCREAMING_SNAKE_CASE       :      Dict							=    load_json_function(UpperCAmelCase_      )
       assert isinstance(UpperCAmelCase_					, UpperCAmelCase_      )
       assert isinstance(exported_content[0]					, UpperCAmelCase_      )
       assert len(UpperCAmelCase_      ) == 10


     @pytest.mark.parametrize(
         "orient, container, keys, len_at"					, [
             ("records", list, {"tokens", "labels", "answers", "id"}, None),
             ("split", dict, {"columns", "data"}, "data"),
             ("index", dict, set("0123456789"      ), None),
             ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
             ("values", list, None, None),
             ("table", dict, {"schema", "data"}, "data"),
         ]					, )
     def 				_A       (				self       :	Optional[Any]					, UpperCAmelCase_       :	Union[str, Any]					, UpperCAmelCase_       :	Any					, UpperCAmelCase_       :	int					, UpperCAmelCase_       :	Any					, UpperCAmelCase_       :	Optional[Any]      ):
       with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCAmelCase_					, UpperCAmelCase_					, lines=UpperCAmelCase_					, orient=UpperCAmelCase_      ).write()
         buffer.seek(0      )
         SCREAMING_SNAKE_CASE       :      str							=    load_json(UpperCAmelCase_      )
       assert isinstance(UpperCAmelCase_					, UpperCAmelCase_      )
       if keys:
         if container is dict:
           assert exported_content.keys() == keys
         else:
           assert exported_content[0].keys() == keys
       else:
         assert not hasattr(UpperCAmelCase_					, "keys"      ) and not hasattr(exported_content[0]					, "keys"      )
       if len_at:
         assert len(exported_content[len_at]      ) == 10
       else:
         assert len(UpperCAmelCase_      ) == 10


     @pytest.mark.parametrize("lines, load_json_function"					, [(True, load_json_lines), (False, load_json)]      )
     def 				_A       (				self       :	List[str]					, UpperCAmelCase_       :	str					, UpperCAmelCase_       :	List[Any]					, UpperCAmelCase_       :	List[Any]      ):
       with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCAmelCase_					, UpperCAmelCase_					, lines=UpperCAmelCase_					, num_proc=2      ).write()
         buffer.seek(0      )
         SCREAMING_SNAKE_CASE       :      List[str]							=    load_json_function(UpperCAmelCase_      )
       assert isinstance(UpperCAmelCase_					, UpperCAmelCase_      )
       assert isinstance(exported_content[0]					, UpperCAmelCase_      )
       assert len(UpperCAmelCase_      ) == 10


     @pytest.mark.parametrize(
         "orient, container, keys, len_at"					, [
             ("records", list, {"tokens", "labels", "answers", "id"}, None),
             ("split", dict, {"columns", "data"}, "data"),
             ("index", dict, set("0123456789"      ), None),
             ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
             ("values", list, None, None),
             ("table", dict, {"schema", "data"}, "data"),
         ]					, )
     def 				_A       (				self       :	str					, UpperCAmelCase_       :	int					, UpperCAmelCase_       :	str					, UpperCAmelCase_       :	Any					, UpperCAmelCase_       :	str					, UpperCAmelCase_       :	str      ):
       with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCAmelCase_					, UpperCAmelCase_					, lines=UpperCAmelCase_					, orient=UpperCAmelCase_					, num_proc=2      ).write()
         buffer.seek(0      )
         SCREAMING_SNAKE_CASE       :      Any							=    load_json(UpperCAmelCase_      )
       assert isinstance(UpperCAmelCase_					, UpperCAmelCase_      )
       if keys:
         if container is dict:
           assert exported_content.keys() == keys
         else:
           assert exported_content[0].keys() == keys
       else:
         assert not hasattr(UpperCAmelCase_					, "keys"      ) and not hasattr(exported_content[0]					, "keys"      )
       if len_at:
         assert len(exported_content[len_at]      ) == 10
       else:
         assert len(UpperCAmelCase_      ) == 10


     def 				_A       (				self       :	Any					, UpperCAmelCase_       :	Any      ):
       with pytest.raises(UpperCAmelCase_      ):
         with io.BytesIO() as buffer:
           JsonDatasetWriter(UpperCAmelCase_					, UpperCAmelCase_					, num_proc=0      )






     @pytest.mark.parametrize("compression, extension"					, [("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]      )
     def 				_A       (				self       :	Tuple					, UpperCAmelCase_       :	Any					, UpperCAmelCase_       :	Union[str, Any]					, UpperCAmelCase_       :	Union[str, Any]					, UpperCAmelCase_       :	str					, UpperCAmelCase_       :	Optional[int]      ):
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tmp_path_factory.mktemp("data"      ) / f'''test.json.{extension}'''
       SCREAMING_SNAKE_CASE       :      Any							=    str(shared_datadir / f'''test_file.json.{extension}'''      )
       JsonDatasetWriter(UpperCAmelCase_					, UpperCAmelCase_					, compression=UpperCAmelCase_      ).write()

       with fsspec.open(UpperCAmelCase_					, "rb"					, compression="infer"      ) as f:
         SCREAMING_SNAKE_CASE       :      Any							=    f.read()
       with fsspec.open(UpperCAmelCase_					, "rb"					, compression="infer"      ) as f:
         SCREAMING_SNAKE_CASE       :      Optional[int]							=    f.read()
       assert exported_content == original_content














from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case    				=		logging.get_logger(__name__)

snake_case    				=		{
    """microsoft/markuplm-base""": """https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json""",
    """microsoft/markuplm-large""": """https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json""",
}


class  SCREAMING_SNAKE_CASE			(			lowerCAmelCase					):
     '''simple docstring'''


     UpperCamelCase_							:		Optional[int]			=	'''markuplm'''


     def __init__(				self       :	int					, UpperCAmelCase_       :	List[str]=3_0522					, UpperCAmelCase_       :	Optional[Any]=768					, UpperCAmelCase_       :	Any=12					, UpperCAmelCase_       :	Tuple=12					, UpperCAmelCase_       :	Union[str, Any]=3072					, UpperCAmelCase_       :	Dict="gelu"					, UpperCAmelCase_       :	Optional[int]=0.1					, UpperCAmelCase_       :	Tuple=0.1					, UpperCAmelCase_       :	Dict=512					, UpperCAmelCase_       :	Optional[int]=2					, UpperCAmelCase_       :	Any=0.02					, UpperCAmelCase_       :	Dict=1E-12					, UpperCAmelCase_       :	int=0					, UpperCAmelCase_       :	Optional[Any]=0					, UpperCAmelCase_       :	Optional[int]=2					, UpperCAmelCase_       :	str=256					, UpperCAmelCase_       :	str=1024					, UpperCAmelCase_       :	List[str]=216					, UpperCAmelCase_       :	List[Any]=1001					, UpperCAmelCase_       :	Union[str, Any]=32					, UpperCAmelCase_       :	Dict=50					, UpperCAmelCase_       :	Optional[int]="absolute"					, UpperCAmelCase_       :	Optional[int]=True					, UpperCAmelCase_       :	Any=None					, **UpperCAmelCase_       :	Any					, ):
       super().__init__(
           pad_token_id=UpperCAmelCase_					, bos_token_id=UpperCAmelCase_					, eos_token_id=UpperCAmelCase_					, **UpperCAmelCase_					, )
       SCREAMING_SNAKE_CASE       :      Any							=    vocab_size
       SCREAMING_SNAKE_CASE       :      List[Any]							=    hidden_size
       SCREAMING_SNAKE_CASE       :      List[Any]							=    num_hidden_layers
       SCREAMING_SNAKE_CASE       :      Tuple							=    num_attention_heads
       SCREAMING_SNAKE_CASE       :      List[str]							=    hidden_act
       SCREAMING_SNAKE_CASE       :      Tuple							=    intermediate_size
       SCREAMING_SNAKE_CASE       :      Any							=    hidden_dropout_prob
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE       :      int							=    max_position_embeddings
       SCREAMING_SNAKE_CASE       :      Tuple							=    type_vocab_size
       SCREAMING_SNAKE_CASE       :      Any							=    initializer_range
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    layer_norm_eps
       SCREAMING_SNAKE_CASE       :      int							=    position_embedding_type
       SCREAMING_SNAKE_CASE       :      List[str]							=    use_cache
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    classifier_dropout
       # additional properties
       SCREAMING_SNAKE_CASE       :      Any							=    max_depth
       SCREAMING_SNAKE_CASE       :      int							=    max_xpath_tag_unit_embeddings
       SCREAMING_SNAKE_CASE       :      List[str]							=    max_xpath_subs_unit_embeddings
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    tag_pad_id
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    subs_pad_id
       SCREAMING_SNAKE_CASE       :      List[Any]							=    xpath_unit_hidden_size






