






"""simple docstring"""

import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def lowerCAmelCase_      (				UpperCamelCase__	: Any				,   UpperCamelCase__	: Any ):






 """simple docstring"""







 assert isinstance(a_				,   a_ )
 assert dataset.num_rows == 4
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_1", "col_2", "col_3"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""				,   [False, True] )
def lowerCAmelCase_      (				UpperCamelCase__	: Optional[Any]				,   UpperCamelCase__	: Tuple				,   UpperCamelCase__	: List[Any] ):






 """simple docstring"""







 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  __lowercase									=	JsonDatasetReader(a_				,   cache_dir=a_				,   keep_in_memory=a_ ).read()
 _check_json_dataset(a_				,   a_ )


@pytest.mark.parametrize(
    """features"""				,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]				,   )
def lowerCAmelCase_      (				UpperCamelCase__	: Dict				,   UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: int ):






 """simple docstring"""







 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 __lowercase									=	features.copy() if features else default_expected_features
 __lowercase									=	(
     Features({feature: Value(a_ ) for feature, dtype in features.items()} ) if features is not None else None
 )
 __lowercase									=	JsonDatasetReader(a_				,   features=a_				,   cache_dir=a_ ).read()
 _check_json_dataset(a_				,   a_ )


@pytest.mark.parametrize(
    """features"""				,   [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]				,   )
def lowerCAmelCase_      (				UpperCamelCase__	: Optional[int]				,   UpperCamelCase__	: int				,   UpperCamelCase__	: str ):






 """simple docstring"""







 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
 __lowercase									=	features.copy() if features else default_expected_features
 __lowercase									=	(
     Features({feature: Value(a_ ) for feature, dtype in features.items()} ) if features is not None else None
 )
 __lowercase									=	JsonDatasetReader(a_				,   features=a_				,   cache_dir=a_ ).read()
 assert isinstance(a_				,   a_ )
 assert dataset.num_rows == 2
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_3", "col_1", "col_2"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype


def lowerCAmelCase_      (				UpperCamelCase__	: Optional[int]				,   UpperCamelCase__	: int ):






 """simple docstring"""







 __lowercase									=	{'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
 __lowercase									=	features.copy()
 __lowercase									=	(
     Features({feature: Value(a_ ) for feature, dtype in features.items()} ) if features is not None else None
 )
 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	JsonDatasetReader(a_				,   features=a_				,   cache_dir=a_ ).read()
 assert isinstance(a_				,   a_ )
 assert dataset.num_rows == 2
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_2", "col_3", "col_1"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""split"""				,   [None, NamedSplit("""train""" ), """train""", """test"""] )
def lowerCAmelCase_      (				UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: List[Any]				,   UpperCamelCase__	: List[str] ):






 """simple docstring"""







 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 __lowercase									=	JsonDatasetReader(a_				,   cache_dir=a_				,   split=a_ ).read()
 _check_json_dataset(a_				,   a_ )
 assert dataset.split == split if split else "train"


@pytest.mark.parametrize("""path_type"""				,   [str, list] )
def lowerCAmelCase_      (				UpperCamelCase__	: Optional[Any]				,   UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: Tuple ):






 """simple docstring"""







 if issubclass(a_				,   a_ ):
  __lowercase									=	jsonl_path
 elif issubclass(a_				,   a_ ):
  __lowercase									=	[jsonl_path]
 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 __lowercase									=	JsonDatasetReader(a_				,   cache_dir=a_ ).read()
 _check_json_dataset(a_				,   a_ )


def lowerCAmelCase_      (				UpperCamelCase__	: str				,   UpperCamelCase__	: Dict				,   UpperCamelCase__	: Any=("train",) ):






 """simple docstring"""







 assert isinstance(a_				,   a_ )
 for split in splits:
  __lowercase									=	dataset_dict[split]
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
   assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""				,   [False, True] )
def lowerCAmelCase_      (				UpperCamelCase__	: int				,   UpperCamelCase__	: List[Any]				,   UpperCamelCase__	: Dict ):






 """simple docstring"""







 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  __lowercase									=	JsonDatasetReader({"""train""": jsonl_path}				,   cache_dir=a_				,   keep_in_memory=a_ ).read()
 _check_json_datasetdict(a_				,   a_ )


@pytest.mark.parametrize(
    """features"""				,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]				,   )
def lowerCAmelCase_      (				UpperCamelCase__	: Any				,   UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: int ):






 """simple docstring"""







 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 __lowercase									=	features.copy() if features else default_expected_features
 __lowercase									=	(
     Features({feature: Value(a_ ) for feature, dtype in features.items()} ) if features is not None else None
 )
 __lowercase									=	JsonDatasetReader({"""train""": jsonl_path}				,   features=a_				,   cache_dir=a_ ).read()
 _check_json_datasetdict(a_				,   a_ )


@pytest.mark.parametrize("""split"""				,   [None, NamedSplit("""train""" ), """train""", """test"""] )
def lowerCAmelCase_      (				UpperCamelCase__	: Optional[Any]				,   UpperCamelCase__	: Any				,   UpperCamelCase__	: str ):






 """simple docstring"""







 if split:
  __lowercase									=	{split: jsonl_path}
 else:
  __lowercase									=	'''train'''
  __lowercase									=	{'''train''': jsonl_path, '''test''': jsonl_path}
 __lowercase									=	tmp_path / '''cache'''
 __lowercase									=	{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
 __lowercase									=	JsonDatasetReader(a_				,   cache_dir=a_ ).read()
 _check_json_datasetdict(a_				,   a_				,   splits=list(path.keys() ) )
 assert all(dataset[split].split == split for split in path.keys() )


def lowerCAmelCase_      (				UpperCamelCase__	: int ):






 """simple docstring"""







 return json.load(a_ )


def lowerCAmelCase_      (				UpperCamelCase__	: Union[str, Any] ):






 """simple docstring"""







 return [json.loads(a_ ) for line in buffer]





class    lowerCamelCase__						:
      @pytest.mark.parametrize("""lines, load_json_function"""      ,       [(True, load_json_lines), (False, load_json)]							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[str]      ,       A_	:	Tuple      ,       A_	:	Union[str, Any]      ,       A_	:	str							):






       '''simple docstring'''


       with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case      ,       __snake_case      ,       lines=__snake_case							).write()
        buffer.seek(0							)
        __lowercase									=	load_json_function(__snake_case							)
       assert isinstance(__snake_case      ,       __snake_case							)
       assert isinstance(exported_content[0]      ,       __snake_case							)
       assert len(__snake_case							) == 1_0
      @pytest.mark.parametrize(
          """orient, container, keys, len_at"""      ,       [
              ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
              ("""split""", dict, {"""columns""", """data"""}, """data"""),
              ("""index""", dict, set("""0123456789"""							), None),
              ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
              ("""values""", list, None, None),
              ("""table""", dict, {"""schema""", """data"""}, """data"""),
          ]      ,       )
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]      ,       A_	:	int      ,       A_	:	Optional[int]      ,       A_	:	Any      ,       A_	:	Union[str, Any]      ,       A_	:	Any							):






       '''simple docstring'''


       with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case      ,       __snake_case      ,       lines=__snake_case      ,       orient=__snake_case							).write()
        buffer.seek(0							)
        __lowercase									=	load_json(__snake_case							)
       assert isinstance(__snake_case      ,       __snake_case							)
       if keys:
        if container is dict:
         assert exported_content.keys() == keys
        else:
         assert exported_content[0].keys() == keys
       else:
        assert not hasattr(__snake_case      ,       """keys"""							) and not hasattr(exported_content[0]      ,       """keys"""							)
       if len_at:
        assert len(exported_content[len_at]							) == 1_0
       else:
        assert len(__snake_case							) == 1_0
      @pytest.mark.parametrize("""lines, load_json_function"""      ,       [(True, load_json_lines), (False, load_json)]							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple      ,       A_	:	Optional[Any]      ,       A_	:	int      ,       A_	:	List[str]							):






       '''simple docstring'''


       with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case      ,       __snake_case      ,       lines=__snake_case      ,       num_proc=2							).write()
        buffer.seek(0							)
        __lowercase									=	load_json_function(__snake_case							)
       assert isinstance(__snake_case      ,       __snake_case							)
       assert isinstance(exported_content[0]      ,       __snake_case							)
       assert len(__snake_case							) == 1_0
      @pytest.mark.parametrize(
          """orient, container, keys, len_at"""      ,       [
              ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
              ("""split""", dict, {"""columns""", """data"""}, """data"""),
              ("""index""", dict, set("""0123456789"""							), None),
              ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
              ("""values""", list, None, None),
              ("""table""", dict, {"""schema""", """data"""}, """data"""),
          ]      ,       )
      def SCREAMING_SNAKE_CASE_	(		self	:	Any      ,       A_	:	int      ,       A_	:	List[str]      ,       A_	:	str      ,       A_	:	Optional[int]      ,       A_	:	Optional[Any]							):






       '''simple docstring'''


       with io.BytesIO() as buffer:
        JsonDatasetWriter(__snake_case      ,       __snake_case      ,       lines=__snake_case      ,       orient=__snake_case      ,       num_proc=2							).write()
        buffer.seek(0							)
        __lowercase									=	load_json(__snake_case							)
       assert isinstance(__snake_case      ,       __snake_case							)
       if keys:
        if container is dict:
         assert exported_content.keys() == keys
        else:
         assert exported_content[0].keys() == keys
       else:
        assert not hasattr(__snake_case      ,       """keys"""							) and not hasattr(exported_content[0]      ,       """keys"""							)
       if len_at:
        assert len(exported_content[len_at]							) == 1_0
       else:
        assert len(__snake_case							) == 1_0
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]      ,       A_	:	Dict							):






       '''simple docstring'''


       with pytest.raises(__snake_case							):
        with io.BytesIO() as buffer:
         JsonDatasetWriter(__snake_case      ,       __snake_case      ,       num_proc=0							)






      @pytest.mark.parametrize("""compression, extension"""      ,       [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	Dict      ,       A_	:	List[Any]      ,       A_	:	Dict      ,       A_	:	List[str]      ,       A_	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	tmp_path_factory.mktemp("""data"""							) / F'''test.json.{extension}'''
       __lowercase									=	str(shared_datadir / F'''test_file.json.{extension}'''							)
       JsonDatasetWriter(__snake_case      ,       __snake_case      ,       compression=__snake_case							).write()

       with fsspec.open(__snake_case      ,       """rb"""      ,       compression="""infer"""							) as f:
        __lowercase									=	f.read()
       with fsspec.open(__snake_case      ,       """rb"""      ,       compression="""infer"""							) as f:
        __lowercase									=	f.read()
       assert exported_content == original_content




"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase__  							={
    "configuration_data2vec_audio": ["DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP", "Data2VecAudioConfig"],
    "configuration_data2vec_text": [
        "DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "Data2VecTextConfig",
        "Data2VecTextOnnxConfig",
    ],
    "configuration_data2vec_vision": [
        "DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "Data2VecVisionConfig",
        "Data2VecVisionOnnxConfig",
    ],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase__  							=[
           "DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST",
           "Data2VecAudioForAudioFrameClassification",
           "Data2VecAudioForCTC",
           "Data2VecAudioForSequenceClassification",
           "Data2VecAudioForXVector",
           "Data2VecAudioModel",
           "Data2VecAudioPreTrainedModel",
       ]
       UpperCAmelCase__  							=[
           "DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
           "Data2VecTextForCausalLM",
           "Data2VecTextForMaskedLM",
           "Data2VecTextForMultipleChoice",
           "Data2VecTextForQuestionAnswering",
           "Data2VecTextForSequenceClassification",
           "Data2VecTextForTokenClassification",
           "Data2VecTextModel",
           "Data2VecTextPreTrainedModel",
       ]
       UpperCAmelCase__  							=[
           "DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST",
           "Data2VecVisionForImageClassification",
           "Data2VecVisionForMaskedImageModeling",
           "Data2VecVisionForSemanticSegmentation",
           "Data2VecVisionModel",
           "Data2VecVisionPreTrainedModel",
       ]

if is_tf_available():
       UpperCAmelCase__  							=[
           "TFData2VecVisionForImageClassification",
           "TFData2VecVisionForSemanticSegmentation",
           "TFData2VecVisionModel",
           "TFData2VecVisionPreTrainedModel",
       ]

if TYPE_CHECKING:
       from .configuration_dataavec_audio import DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP, DataaVecAudioConfig
       from .configuration_dataavec_text import (
           DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP,
           DataaVecTextConfig,
           DataaVecTextOnnxConfig,
       )
       from .configuration_dataavec_vision import (
           DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP,
           DataaVecVisionConfig,
           DataaVecVisionOnnxConfig,
       )

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_dataavec_audio import (
                  DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST,
                  DataaVecAudioForAudioFrameClassification,
                  DataaVecAudioForCTC,
                  DataaVecAudioForSequenceClassification,
                  DataaVecAudioForXVector,
                  DataaVecAudioModel,
                  DataaVecAudioPreTrainedModel,
              )
              from .modeling_dataavec_text import (
                  DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  DataaVecTextForCausalLM,
                  DataaVecTextForMaskedLM,
                  DataaVecTextForMultipleChoice,
                  DataaVecTextForQuestionAnswering,
                  DataaVecTextForSequenceClassification,
                  DataaVecTextForTokenClassification,
                  DataaVecTextModel,
                  DataaVecTextPreTrainedModel,
              )
              from .modeling_dataavec_vision import (
                  DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST,
                  DataaVecVisionForImageClassification,
                  DataaVecVisionForMaskedImageModeling,
                  DataaVecVisionForSemanticSegmentation,
                  DataaVecVisionModel,
                  DataaVecVisionPreTrainedModel,
              )
       if is_tf_available():
              from .modeling_tf_dataavec_vision import (
                  TFDataaVecVisionForImageClassification,
                  TFDataaVecVisionForSemanticSegmentation,
                  TFDataaVecVisionModel,
                  TFDataaVecVisionPreTrainedModel,
              )

else:
       import sys

       UpperCAmelCase__  							=_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
