






import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


__lowerCamelCase      :    Optional[int]   		=   logging.get_logger(__name__)



class 							SCREAMING_SNAKE_CASE__		:





					"""simple docstring"""






					a_			=	None







@experimental
def   SCREAMING_SNAKE_CASE       (				snake_case_      :							Tuple  ,						snake_case_      :							Dict  ,						snake_case_      :							Optional[int]  ,						snake_case_      :							Optional[Any]  ,						snake_case_      :							List[Any]  ,						snake_case_      :							List[str]  ,						snake_case_      :							Optional[int]     ):
	if ParallelBackendConfig.backend_name is None:
		return _map_with_multiprocessing_pool(
		    snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_     )

	return _map_with_joblib(snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_  ,						snake_case_     )







def   SCREAMING_SNAKE_CASE       (				snake_case_      :							Optional[int]  ,						snake_case_      :							Union[str, Any]  ,						snake_case_      :							Union[str, Any]  ,						snake_case_      :							Optional[int]  ,						snake_case_      :							List[str]  ,						snake_case_      :							int  ,						snake_case_      :							int     ):
	snake_case__				:   Union[str, Any]     =						num_proc if num_proc <= len(snake_case_     ) else len(snake_case_     )
	snake_case__				:   Dict     =						[]  # We organize the splits ourselve (contiguous splits)
	for index in range(snake_case_     ):
		snake_case__				:   Tuple     =						len(snake_case_     ) // num_proc
		snake_case__				:   Dict     =						len(snake_case_     ) % num_proc
		snake_case__				:   List[str]     =						div * index + min(snake_case_  ,						snake_case_     )
		snake_case__				:   List[Any]     =						start + div + (1 if index < mod else 0)
		split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)     )

	if len(snake_case_     ) != sum(len(i[1]     ) for i in split_kwds     ):
		raise ValueError(
		    F'''Error dividing inputs iterable among processes. '''
		    F'''Total number of objects {len(snake_case_     )}, '''
		    F'''length: {sum(len(i[1]     ) for i in split_kwds     )}'''     )

	logger.info(
	    F'''Spawning {num_proc} processes for {len(snake_case_     )} objects in slices of {[len(i[1]     ) for i in split_kwds]}'''     )
	snake_case__,		snake_case__				:   Any     =						None, None
	if not disable_tqdm:
		snake_case__,		snake_case__				:   Optional[Any]     =						(RLock(),), tqdm.set_lock
	with Pool(snake_case_  ,						initargs=snake_case_  ,						initializer=snake_case_     ) as pool:
		snake_case__				:   List[Any]     =						pool.map(snake_case_  ,						snake_case_     )
	logger.info(F'''Finished {num_proc} processes'''     )
	snake_case__				:   Union[str, Any]     =						[obj for proc_res in mapped for obj in proc_res]
	logger.info(F'''Unpacked {len(snake_case_     )} objects'''     )

	return mapped







def   SCREAMING_SNAKE_CASE       (				snake_case_      :							Optional[int]  ,						snake_case_      :							str  ,						snake_case_      :							Any  ,						snake_case_      :							Union[str, Any]  ,						snake_case_      :							str  ,						snake_case_      :							Optional[int]  ,						snake_case_      :							Union[str, Any]     ):
	# progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
	# and it requires monkey-patching joblib internal classes which is subject to change
	import joblib

	with joblib.parallel_backend(ParallelBackendConfig.backend_name  ,						n_jobs=snake_case_     ):
		return joblib.Parallel()(
		    joblib.delayed(snake_case_     )((function, obj, types, None, True, None)     ) for obj in iterable     )







@experimental
@contextlib.contextmanager
def   SCREAMING_SNAKE_CASE       (				snake_case_      :							str     ):
	snake_case__				:   Optional[Any]     =						backend_name

	if backend_name == "spark":
		from joblibspark import register_spark

		register_spark()

		# TODO: call create_cache_and_write_probe if "download" in steps
		# TODO: raise NotImplementedError when Dataset.map etc is called

	try:
		yield
	finally:
		snake_case__				:   Optional[Any]     =						None











# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

__lowerCamelCase      :    Union[str, Any]   		=   """2.13.1"""

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse("""3.7"""):
	raise ImportWarning(
	    """To use `datasets`, Python>=3.7 is required, and the current version of Python doesn't match this condition."""
	)

if version.parse(pyarrow.__version__).major < 8:
	raise ImportWarning(
	    """To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn't match this condition.\n"""
	    """If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`."""
	)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

__lowerCamelCase      :    List[Any]   		=   concatenate_datasets
__lowerCamelCase      :    List[str]   		=   DownloadConfig
__lowerCamelCase      :    Union[str, Any]   		=   DownloadManager
__lowerCamelCase      :    str   		=   DownloadMode
__lowerCamelCase      :    Union[str, Any]   		=   DownloadConfig
__lowerCamelCase      :    List[str]   		=   DownloadMode
__lowerCamelCase      :    Dict   		=   DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager



