
"""simple docstring"""



import importlib
import os

import fsspec
import pytest
from fsspec import register_implementation
from fsspec.registry import _registry as _fsspec_registry

from datasets.filesystems import COMPRESSION_FILESYSTEMS, HfFileSystem, extract_path_from_uri, is_remote_filesystem

from .utils import require_lza, require_zstandard


def  _UpperCamelCase	(						_A      )    ->  Dict:





    """simple docstring"""

    assert "mock" in _fsspec_registry
    assert "bz2" in _fsspec_registry


def  _UpperCamelCase	(						)    ->  Tuple:





    """simple docstring"""

    assert "mock" not in _fsspec_registry
    assert "bz2" in _fsspec_registry


def  _UpperCamelCase	(						)    ->  str:





    """simple docstring"""

    _UpperCAmelCase          =   """mock-s3-bucket"""
    _UpperCAmelCase          =   F"""s3://{mock_bucket}"""
    _UpperCAmelCase          =   extract_path_from_uri(_UpperCAmelCase      )
    assert dataset_path.startswith("""s3://"""      ) is False

    _UpperCAmelCase          =   """./local/path"""
    _UpperCAmelCase          =   extract_path_from_uri(_UpperCAmelCase      )
    assert dataset_path == new_dataset_path


def  _UpperCamelCase	(						_A      )    ->  str:





    """simple docstring"""

    _UpperCAmelCase          =   is_remote_filesystem(_UpperCAmelCase      )
    assert is_remote is True

    _UpperCAmelCase          =   fsspec.filesystem("""file"""      )

    _UpperCAmelCase          =   is_remote_filesystem(_UpperCAmelCase      )
    assert is_remote is False


@pytest.mark.parametrize("""compression_fs_class"""					,      _UpperCAmelCase      )
def  _UpperCamelCase	(						_A					,      _A					,      _A					,      _A					,      _A					,      _A					,      _A      )    ->  Any:





    """simple docstring"""

    _UpperCAmelCase          =   {"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_file, """bz2""": bza_file, """lz4""": lza_file}
    _UpperCAmelCase          =   input_paths[compression_fs_class.protocol]
    if input_path is None:
        _UpperCAmelCase          =   F"""for \'{compression_fs_class.protocol}\' compression protocol, """
        if compression_fs_class.protocol == "lz4":
            reason += require_lza.kwargs["reason"]
        elif compression_fs_class.protocol == "zstd":
            reason += require_zstandard.kwargs["reason"]
        pytest.skip(_UpperCAmelCase      )
    _UpperCAmelCase          =   fsspec.filesystem(compression_fs_class.protocol					,      fo=_UpperCAmelCase      )
    assert isinstance(_UpperCAmelCase					,      _UpperCAmelCase      )
    _UpperCAmelCase          =   os.path.basename(_UpperCAmelCase      )
    _UpperCAmelCase          =   expected_filename[: expected_filename.rindex("""."""      )]
    assert fs.glob("""*"""      ) == [expected_filename]
    with fs.open(_UpperCAmelCase					,      """r"""					,      encoding="""utf-8"""      ) as f, open(_UpperCAmelCase					,      encoding="""utf-8"""      ) as expected_file:
        assert f.read() == expected_file.read()


@pytest.mark.parametrize("""protocol"""					,      ["""zip""", """gzip"""]      )
def  _UpperCamelCase	(						_A					,      _A					,      _A      )    ->  Optional[int]:





    """simple docstring"""

    _UpperCAmelCase          =   {"""zip""": zip_jsonl_path, """gzip""": jsonl_gz_path}
    _UpperCAmelCase          =   compressed_file_paths[protocol]
    _UpperCAmelCase          =   """dataset.jsonl"""
    _UpperCAmelCase          =   F"""{protocol}://{member_file_path}::{compressed_file_path}"""
    _UpperCAmelCase    ,*_UpperCAmelCase          =   fsspec.get_fs_token_paths(_UpperCAmelCase      )
    assert fs.isfile(_UpperCAmelCase      )
    assert not fs.isfile("""non_existing_""" + member_file_path      )


@pytest.mark.integration
def  _UpperCamelCase	(						_A					,      _A					,      _A					,      _A      )    ->  Any:





    """simple docstring"""

    _UpperCAmelCase          =   hf_api.dataset_info(_UpperCAmelCase					,      token=_UpperCAmelCase      )
    _UpperCAmelCase          =   HfFileSystem(repo_info=_UpperCAmelCase					,      token=_UpperCAmelCase      )
    assert sorted(hffs.glob("""*"""      )      ) == [".gitattributes", "data"]
    assert hffs.isdir("""data"""      )
    assert hffs.isfile(""".gitattributes"""      ) and hffs.isfile("""data/text_data.txt"""      )
    with open(_UpperCAmelCase      ) as f:
        assert hffs.open("""data/text_data.txt"""					,      """r"""      ).read() == f.read()






def  _UpperCamelCase	(						)    ->  Dict:





    """simple docstring"""

    _UpperCAmelCase          =   """bz2"""

    # Import module
    import datasets.filesystems

    # Overwrite protocol and reload
    register_implementation(_UpperCAmelCase					,      _UpperCAmelCase					,      clobber=_UpperCAmelCase      )
    with pytest.warns(_UpperCAmelCase      ) as warning_info:
        importlib.reload(datasets.filesystems      )

    assert len(_UpperCAmelCase      ) == 1
    assert (
        str(warning_info[0].message      )
        == F"""A filesystem protocol was already set for {protocol} and will be overwritten."""
    )

"""simple docstring"""



# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
 import jax
 import jaxlib

a					:	List[Any]							=		get_logger()

a					:	Optional[dict]							=		None




class 		a_						( TensorFormatter[Mapping, 'jax.Array', Mapping]	):







    def __init__(			self			:       int     ,					__UpperCamelCase			:       List[str]=None     ,					__UpperCamelCase			:       Optional[int]=None     ,					**__UpperCamelCase			:       int     )		->Tuple:






        '''simple docstring'''






        super().__init__(features=__UpperCamelCase     )
        import jax
        from jaxlib.xla_client import Device

        if isinstance(__UpperCamelCase     ,					__UpperCamelCase     ):
            raise ValueError(
                f"""Expected {device} to be a `str` not {type(__UpperCamelCase     )}, as `jaxlib.xla_extension.Device` """
                """is not serializable neither with `pickle` nor with `dill`. Instead you can surround """
                """the device with `str()` to get its string identifier that will be internally mapped """
                """to the actual `jaxlib.xla_extension.Device`."""     )
        _UpperCAmelCase          =   device if isinstance(__UpperCamelCase     ,					__UpperCamelCase     ) else str(jax.devices()[0]     )
        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
        global DEVICE_MAPPING
        if DEVICE_MAPPING is None:
            _UpperCAmelCase          =   self._map_devices_to_str()
        if self.device not in list(DEVICE_MAPPING.keys()     ):
            logger.warning(
                f"""Device with string identifier {self.device} not listed among the available """
                f"""devices: {list(DEVICE_MAPPING.keys()     )}, so falling back to the default """
                f"""device: {str(jax.devices()[0]     )}."""     )
            _UpperCAmelCase          =   str(jax.devices()[0]     )
        _UpperCAmelCase          =   jnp_array_kwargs







    @staticmethod
    def      _snake_case					(			)		->Dict[str, "jaxlib.xla_extension.Device"]:






        '''simple docstring'''






        import jax

        return {str(__UpperCamelCase     ): device for device in jax.devices()}







    def      _snake_case					(			self			:       Dict     ,					__UpperCamelCase			:       Any     )		->Union[str, Any]:






        '''simple docstring'''






        import jax
        import jax.numpy as jnp

        if isinstance(__UpperCamelCase     ,					__UpperCamelCase     ) and column:
            if all(
                isinstance(__UpperCamelCase     ,					jax.Array     ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column     ):
                return jnp.stack(__UpperCamelCase     ,					axis=0     )
        return column







    def      _snake_case					(			self			:       List[str]     ,					__UpperCamelCase			:       Any     )		->Optional[int]:






        '''simple docstring'''






        import jax
        import jax.numpy as jnp

        if isinstance(__UpperCamelCase     ,					(str, bytes, type(__UpperCamelCase     ))     ):
            return value
        elif isinstance(__UpperCamelCase     ,					(np.character, np.ndarray)     ) and np.issubdtype(value.dtype     ,					np.character     ):
            return value.tolist()

        _UpperCAmelCase          =   {}

        if isinstance(__UpperCamelCase     ,					(np.number, np.ndarray)     ) and np.issubdtype(value.dtype     ,					np.integer     ):
            # the default int precision depends on the jax config
            # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
            if jax.config.jax_enable_xaa:
                _UpperCAmelCase          =   {"""dtype""": jnp.intaa}
            else:
                _UpperCAmelCase          =   {"""dtype""": jnp.intaa}
        elif isinstance(__UpperCamelCase     ,					(np.number, np.ndarray)     ) and np.issubdtype(value.dtype     ,					np.floating     ):
            _UpperCAmelCase          =   {"""dtype""": jnp.floataa}
        elif config.PIL_AVAILABLE and "PIL" in sys.modules:
            import PIL.Image

            if isinstance(__UpperCamelCase     ,					PIL.Image.Image     ):
                _UpperCAmelCase          =   np.asarray(__UpperCamelCase     )

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
        global DEVICE_MAPPING
        if DEVICE_MAPPING is None:
            _UpperCAmelCase          =   self._map_devices_to_str()

        with jax.default_device(DEVICE_MAPPING[self.device]     ):
            # calling jnp.array on a np.ndarray does copy the data
            # see https://github.com/google/jax/issues/4486
            return jnp.array(__UpperCamelCase     ,					**{**default_dtype, **self.jnp_array_kwargs}     )







    def      _snake_case					(			self			:       Union[str, Any]     ,					__UpperCamelCase			:       List[str]     )		->Any:






        '''simple docstring'''






        import jax

        # support for torch, tf, jax etc.
        if config.TORCH_AVAILABLE and "torch" in sys.modules:
            import torch

            if isinstance(__UpperCamelCase     ,					torch.Tensor     ):
                return self._tensorize(data_struct.detach().cpu().numpy()[()]     )
        if hasattr(__UpperCamelCase     ,					"""__array__"""     ) and not isinstance(__UpperCamelCase     ,					jax.Array     ):
            _UpperCAmelCase          =   data_struct.__array__()
        # support for nested types like struct of list of struct
        if isinstance(__UpperCamelCase     ,					np.ndarray     ):
            if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
                return self._consolidate([self.recursive_tensorize(__UpperCamelCase     ) for substruct in data_struct]     )
        elif isinstance(__UpperCamelCase     ,					(list, tuple)     ):
            return self._consolidate([self.recursive_tensorize(__UpperCamelCase     ) for substruct in data_struct]     )
        return self._tensorize(__UpperCamelCase     )







    def      _snake_case					(			self			:       List[str]     ,					__UpperCamelCase			:       dict     )		->int:






        '''simple docstring'''






        return map_nested(self._recursive_tensorize     ,					__UpperCamelCase     ,					map_list=__UpperCamelCase     )







    def      _snake_case					(			self			:       Dict     ,					__UpperCamelCase			:       pa.Table     )		->Mapping:






        '''simple docstring'''






        _UpperCAmelCase          =   self.numpy_arrow_extractor().extract_row(__UpperCamelCase     )
        _UpperCAmelCase          =   self.python_features_decoder.decode_row(__UpperCamelCase     )
        return self.recursive_tensorize(__UpperCamelCase     )







    def      _snake_case					(			self			:       Optional[int]     ,					__UpperCamelCase			:       pa.Table     )		->"jax.Array":






        '''simple docstring'''






        _UpperCAmelCase          =   self.numpy_arrow_extractor().extract_column(__UpperCamelCase     )
        _UpperCAmelCase          =   self.python_features_decoder.decode_column(__UpperCamelCase     ,					pa_table.column_names[0]     )
        _UpperCAmelCase          =   self.recursive_tensorize(__UpperCamelCase     )
        _UpperCAmelCase          =   self._consolidate(__UpperCamelCase     )
        return column







    def      _snake_case					(			self			:       Optional[Any]     ,					__UpperCamelCase			:       pa.Table     )		->Mapping:






        '''simple docstring'''






        _UpperCAmelCase          =   self.numpy_arrow_extractor().extract_batch(__UpperCamelCase     )
        _UpperCAmelCase          =   self.python_features_decoder.decode_batch(__UpperCamelCase     )
        _UpperCAmelCase          =   self.recursive_tensorize(__UpperCamelCase     )
        for column_name in batch:
            _UpperCAmelCase          =   self._consolidate(batch[column_name]     )
        return batch