
"""simple docstring"""



import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def  _UpperCamelCase	(						_A					,      _A      )    ->  Union[str, Any]:





    """simple docstring"""

    assert isinstance(lowercase_					,      lowercase_      )
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""					,      [False, True]      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  Optional[int]:





    """simple docstring"""

    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      cache_dir=lowercase_					,      keep_in_memory=lowercase_      ).read()
    _check_json_dataset(lowercase_					,      lowercase_      )


@pytest.mark.parametrize(
    """features"""					,      [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  int:





    """simple docstring"""

    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    _UpperCAmelCase          =   features.copy() if features else default_expected_features
    _UpperCAmelCase          =   (
        Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      features=lowercase_					,      cache_dir=lowercase_      ).read()
    _check_json_dataset(lowercase_					,      lowercase_      )


@pytest.mark.parametrize(
    """features"""					,      [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]					,      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  Dict:





    """simple docstring"""

    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
    _UpperCAmelCase          =   features.copy() if features else default_expected_features
    _UpperCAmelCase          =   (
        Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      features=lowercase_					,      cache_dir=lowercase_      ).read()
    assert isinstance(lowercase_					,      lowercase_      )
    assert dataset.num_rows == 2
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_3", "col_1", "col_2"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype


def  _UpperCamelCase	(						_A					,      _A      )    ->  int:





    """simple docstring"""

    _UpperCAmelCase          =   {"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
    _UpperCAmelCase          =   features.copy()
    _UpperCAmelCase          =   (
        Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      features=lowercase_					,      cache_dir=lowercase_      ).read()
    assert isinstance(lowercase_					,      lowercase_      )
    assert dataset.num_rows == 2
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_2", "col_3", "col_1"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""split"""					,      [None, NamedSplit("""train"""      ), """train""", """test"""]      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  Optional[Any]:





    """simple docstring"""

    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      cache_dir=lowercase_					,      split=lowercase_      ).read()
    _check_json_dataset(lowercase_					,      lowercase_      )
    assert dataset.split == split if split else "train"


@pytest.mark.parametrize("""path_type"""					,      [str, list]      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  List[Any]:





    """simple docstring"""

    if issubclass(lowercase_					,      lowercase_      ):
        _UpperCAmelCase          =   jsonl_path
    elif issubclass(lowercase_					,      lowercase_      ):
        _UpperCAmelCase          =   [jsonl_path]
    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      cache_dir=lowercase_      ).read()
    _check_json_dataset(lowercase_					,      lowercase_      )


def  _UpperCamelCase	(						_A					,      _A					,      _A=("train",)      )    ->  Optional[int]:





    """simple docstring"""

    assert isinstance(lowercase_					,      lowercase_      )
    for split in splits:
        _UpperCAmelCase          =   dataset_dict[split]
        assert dataset.num_rows == 4
        assert dataset.num_columns == 3
        assert dataset.column_names == ["col_1", "col_2", "col_3"]
        for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""					,      [False, True]      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  Dict:





    """simple docstring"""

    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        _UpperCAmelCase          =   JsonDatasetReader({"""train""": jsonl_path}					,      cache_dir=lowercase_					,      keep_in_memory=lowercase_      ).read()
    _check_json_datasetdict(lowercase_					,      lowercase_      )


@pytest.mark.parametrize(
    """features"""					,      [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  int:





    """simple docstring"""

    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    _UpperCAmelCase          =   features.copy() if features else default_expected_features
    _UpperCAmelCase          =   (
        Features({feature: Value(lowercase_      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    _UpperCAmelCase          =   JsonDatasetReader({"""train""": jsonl_path}					,      features=lowercase_					,      cache_dir=lowercase_      ).read()
    _check_json_datasetdict(lowercase_					,      lowercase_      )


@pytest.mark.parametrize("""split"""					,      [None, NamedSplit("""train"""      ), """train""", """test"""]      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  List[str]:





    """simple docstring"""

    if split:
        _UpperCAmelCase          =   {split: jsonl_path}
    else:
        _UpperCAmelCase          =   """train"""
        _UpperCAmelCase          =   {"""train""": jsonl_path, """test""": jsonl_path}
    _UpperCAmelCase          =   tmp_path / """cache"""
    _UpperCAmelCase          =   {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
    _UpperCAmelCase          =   JsonDatasetReader(lowercase_					,      cache_dir=lowercase_      ).read()
    _check_json_datasetdict(lowercase_					,      lowercase_					,      splits=list(path.keys()      )      )
    assert all(dataset[split].split == split for split in path.keys()      )


def  _UpperCamelCase	(						_A      )    ->  Optional[Any]:





    """simple docstring"""

    return json.load(lowercase_      )


def  _UpperCamelCase	(						_A      )    ->  Optional[int]:





    """simple docstring"""

    return [json.loads(lowercase_      ) for line in buffer]







class 		a_						:







    @pytest.mark.parametrize("""lines, load_json_function"""     ,					[(True, load_json_lines), (False, load_json)]     )
    def      _snake_case					(			self			:       Dict     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       List[str]     ,					__UpperCamelCase			:       Union[str, Any]     )		->List[Any]:






        '''simple docstring'''






        with io.BytesIO() as buffer:
            JsonDatasetWriter(A__     ,					A__     ,					lines=A__     ).write()
            buffer.seek(0     )
            _UpperCAmelCase          =   load_json_function(A__     )
        assert isinstance(A__     ,					A__     )
        assert isinstance(exported_content[0]     ,					A__     )
        assert len(A__     ) == 10







    @pytest.mark.parametrize(
        """orient, container, keys, len_at"""     ,					[
            ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
            ("""split""", dict, {"""columns""", """data"""}, """data"""),
            ("""index""", dict, set("""0123456789"""     ), None),
            ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
            ("""values""", list, None, None),
            ("""table""", dict, {"""schema""", """data"""}, """data"""),
        ]     ,					)
    def      _snake_case					(			self			:       Any     ,					__UpperCamelCase			:       Union[str, Any]     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       Optional[Any]     ,					__UpperCamelCase			:       Tuple     ,					__UpperCamelCase			:       Any     )		->Optional[int]:






        '''simple docstring'''






        with io.BytesIO() as buffer:
            JsonDatasetWriter(A__     ,					A__     ,					lines=A__     ,					orient=A__     ).write()
            buffer.seek(0     )
            _UpperCAmelCase          =   load_json(A__     )
        assert isinstance(A__     ,					A__     )
        if keys:
            if container is dict:
                assert exported_content.keys() == keys
            else:
                assert exported_content[0].keys() == keys
        else:
            assert not hasattr(A__     ,					"""keys"""     ) and not hasattr(exported_content[0]     ,					"""keys"""     )
        if len_at:
            assert len(exported_content[len_at]     ) == 10
        else:
            assert len(A__     ) == 10







    @pytest.mark.parametrize("""lines, load_json_function"""     ,					[(True, load_json_lines), (False, load_json)]     )
    def      _snake_case					(			self			:       Tuple     ,					__UpperCamelCase			:       int     ,					__UpperCamelCase			:       Dict     ,					__UpperCamelCase			:       str     )		->Any:






        '''simple docstring'''






        with io.BytesIO() as buffer:
            JsonDatasetWriter(A__     ,					A__     ,					lines=A__     ,					num_proc=2     ).write()
            buffer.seek(0     )
            _UpperCAmelCase          =   load_json_function(A__     )
        assert isinstance(A__     ,					A__     )
        assert isinstance(exported_content[0]     ,					A__     )
        assert len(A__     ) == 10







    @pytest.mark.parametrize(
        """orient, container, keys, len_at"""     ,					[
            ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
            ("""split""", dict, {"""columns""", """data"""}, """data"""),
            ("""index""", dict, set("""0123456789"""     ), None),
            ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
            ("""values""", list, None, None),
            ("""table""", dict, {"""schema""", """data"""}, """data"""),
        ]     ,					)
    def      _snake_case					(			self			:       List[Any]     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       Dict     ,					__UpperCamelCase			:       Tuple     ,					__UpperCamelCase			:       Optional[Any]     ,					__UpperCamelCase			:       Any     )		->Optional[int]:






        '''simple docstring'''






        with io.BytesIO() as buffer:
            JsonDatasetWriter(A__     ,					A__     ,					lines=A__     ,					orient=A__     ,					num_proc=2     ).write()
            buffer.seek(0     )
            _UpperCAmelCase          =   load_json(A__     )
        assert isinstance(A__     ,					A__     )
        if keys:
            if container is dict:
                assert exported_content.keys() == keys
            else:
                assert exported_content[0].keys() == keys
        else:
            assert not hasattr(A__     ,					"""keys"""     ) and not hasattr(exported_content[0]     ,					"""keys"""     )
        if len_at:
            assert len(exported_content[len_at]     ) == 10
        else:
            assert len(A__     ) == 10







    def      _snake_case					(			self			:       Tuple     ,					__UpperCamelCase			:       Tuple     )		->Dict:






        '''simple docstring'''






        with pytest.raises(A__     ):
            with io.BytesIO() as buffer:
                JsonDatasetWriter(A__     ,					A__     ,					num_proc=0     )







    @pytest.mark.parametrize("""compression, extension"""     ,					[("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]     )
    def      _snake_case					(			self			:       Any     ,					__UpperCamelCase			:       List[Any]     ,					__UpperCamelCase			:       Tuple     ,					__UpperCamelCase			:       Union[str, Any]     ,					__UpperCamelCase			:       List[Any]     ,					__UpperCamelCase			:       List[Any]     )		->Tuple:






        '''simple docstring'''






        _UpperCAmelCase          =   tmp_path_factory.mktemp("""data"""     ) / f"""test.json.{extension}"""
        _UpperCAmelCase          =   str(shared_datadir / f"""test_file.json.{extension}"""     )
        JsonDatasetWriter(A__     ,					A__     ,					compression=A__     ).write()

        with fsspec.open(A__     ,					"""rb"""     ,					compression="""infer"""     ) as f:
            _UpperCAmelCase          =   f.read()
        with fsspec.open(A__     ,					"""rb"""     ,					compression="""infer"""     ) as f:
            _UpperCAmelCase          =   f.read()
        assert exported_content == original_content

"""simple docstring"""



from pathlib import PurePosixPath
from typing import Optional

import fsspec
from fsspec import AbstractFileSystem
from huggingface_hub.hf_api import DatasetInfo

from ..utils.file_utils import get_authentication_headers_for_url
from ..utils.hub import hf_hub_url




class 		a_						( _UpperCAmelCase	):
    a      :				List[Any]     =				''
    a      :				Union[str, Any]     =				'hf-legacy'  # "hf://"" is reserved for hffs







    def __init__(			self			:       Tuple     ,					__UpperCamelCase			:       Optional[DatasetInfo] = None     ,					__UpperCamelCase			:       Optional[str] = None     ,					**__UpperCamelCase			:       Any     ,					)		->Any:






        '''simple docstring'''






        super().__init__(self     ,					**__UpperCamelCase     )
        _UpperCAmelCase          =   repo_info
        _UpperCAmelCase          =   token
        _UpperCAmelCase          =   None







    def      _snake_case					(			self			:       List[str]     )		->List[str]:






        '''simple docstring'''






        if self.dir_cache is None:
            _UpperCAmelCase          =   {}
            for hf_file in self.repo_info.siblings:
                # TODO(QL): add sizes
                _UpperCAmelCase          =   {
                    """name""": hf_file.rfilename,
                    """size""": None,
                    """type""": """file""",
                }
                self.dir_cache.update(
                    {
                        str(__UpperCamelCase     ): {"""name""": str(__UpperCamelCase     ), """size""": None, """type""": """directory"""}
                        for d in list(PurePosixPath(hf_file.rfilename     ).parents     )[:-1]
                    }     )







    def      _snake_case					(			self			:       Tuple     ,					__UpperCamelCase			:       str     ,					__UpperCamelCase			:       str = "rb"     ,					**__UpperCamelCase			:       Any     ,					)		->List[str]:






        '''simple docstring'''






        if not isinstance(self.repo_info     ,					__UpperCamelCase     ):
            raise NotImplementedError(f"""Open is only implemented for dataset repositories, but got {self.repo_info}"""     )
        _UpperCAmelCase          =   hf_hub_url(self.repo_info.id     ,					__UpperCamelCase     ,					revision=self.repo_info.sha     )
        return fsspec.open(
            __UpperCamelCase     ,					mode=__UpperCamelCase     ,					headers=get_authentication_headers_for_url(__UpperCamelCase     ,					use_auth_token=self.token     )     ,					client_kwargs={"""trust_env""": True}     ,					).open()







    def      _snake_case					(			self			:       int     ,					__UpperCamelCase			:       int     ,					**__UpperCamelCase			:       Dict     )		->Tuple:






        '''simple docstring'''






        self._get_dirs()
        _UpperCAmelCase          =   self._strip_protocol(__UpperCamelCase     )
        if path in self.dir_cache:
            return self.dir_cache[path]
        else:
            raise FileNotFoundError(__UpperCamelCase     )







    def      _snake_case					(			self			:       List[str]     ,					__UpperCamelCase			:       Union[str, Any]     ,					__UpperCamelCase			:       Tuple=False     ,					**__UpperCamelCase			:       List[str]     )		->Optional[Any]:






        '''simple docstring'''






        self._get_dirs()
        _UpperCAmelCase          =   PurePosixPath(path.strip("""/"""     )     )
        _UpperCAmelCase          =   {}
        for p, f in self.dir_cache.items():
            _UpperCAmelCase          =   PurePosixPath(p.strip("""/"""     )     )
            _UpperCAmelCase          =   p.parent
            if root == path:
                _UpperCAmelCase          =   f
        _UpperCAmelCase          =   list(paths.values()     )
        if detail:
            return out
        else:
            return sorted(f["""name"""] for f in out     )