



"""simple docstring"""

import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader







class    __lowercase     (					lowercase__):







						"""simple docstring"""







						def __init__(self   ,		lowercase__   ,		lowercase__ = None   ,		lowercase__ = None   ,		lowercase__ = None   ,		lowercase__ = False   ,		lowercase__ = False   ,		lowercase__ = None   ,		lowercase__ = None   ,		**lowercase__   ,		):
											super().__init__(
											    lowercase__   ,		split=lowercase__   ,		features=lowercase__   ,		cache_dir=lowercase__   ,		keep_in_memory=lowercase__   ,		streaming=lowercase__   ,		num_proc=lowercase__   ,		**lowercase__   ,		)
											snake_case_					:		int 					=					field
											snake_case_					:		int 					=					path_or_paths if isinstance(lowercase__   ,		lowercase__       ) else {self.split: path_or_paths}
											snake_case_					:		Tuple 					=					Json(
											    cache_dir=lowercase__   ,		data_files=lowercase__   ,		features=lowercase__   ,		field=lowercase__   ,		**lowercase__   ,		)







						def 						__UpperCamelCase							(self       ):
											# Build iterable dataset
											if self.streaming:
																snake_case_					:		Union[str, Any] 					=					self.builder.as_streaming_dataset(split=self.split       )
											# Build regular (map-style) dataset
											else:
																snake_case_					:		Union[str, Any] 					=					None
																snake_case_					:		Union[str, Any] 					=					None
																snake_case_					:		str 					=					None
																snake_case_					:		List[Any] 					=					None

																self.builder.download_and_prepare(
																    download_config=lowercase__   ,		download_mode=lowercase__   ,		verification_mode=lowercase__   ,		base_path=lowercase__   ,		num_proc=self.num_proc   ,		)
																snake_case_					:		List[Any] 					=					self.builder.as_dataset(
																    split=self.split   ,		verification_mode=lowercase__   ,		in_memory=self.keep_in_memory       )
											return dataset







class    __lowercase     :







						"""simple docstring"""







						def __init__(self   ,		lowercase__   ,		lowercase__   ,		lowercase__ = None   ,		lowercase__ = None   ,		**lowercase__   ,		):
											if num_proc is not None and num_proc <= 0:
																raise ValueError(f'num_proc {num_proc} must be an integer > 0.'       )

											snake_case_					:		Tuple 					=					dataset
											snake_case_					:		List[str] 					=					path_or_buf
											snake_case_					:		Optional[int] 					=					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											snake_case_					:		Any 					=					num_proc
											snake_case_					:		Optional[int] 					=					"utf-8"
											snake_case_					:		Any 					=					to_json_kwargs







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self.to_json_kwargs.pop("""path_or_buf"""   ,		lowercase__       )
											snake_case_					:		List[str] 					=					self.to_json_kwargs.pop("""orient"""   ,		"""records"""       )
											snake_case_					:		Any 					=					self.to_json_kwargs.pop("""lines"""   ,		True if orient == """records""" else False       )
											snake_case_					:		Union[str, Any] 					=					self.to_json_kwargs.pop("""index"""   ,		False if orient in ["""split""", """table"""] else True       )
											snake_case_					:		Any 					=					self.to_json_kwargs.pop("""compression"""   ,		lowercase__       )

											if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																raise NotImplementedError(f'`datasets` currently does not support {compression} compression'       )

											if isinstance(self.path_or_buf   ,		(str, bytes, os.PathLike)       ):
																with fsspec.open(self.path_or_buf   ,		"""wb"""   ,		compression=lowercase__       ) as buffer:
																					snake_case_					:		Optional[int] 					=					self._write(file_obj=lowercase__   ,		orient=lowercase__   ,		lines=lowercase__   ,		index=lowercase__   ,		**self.to_json_kwargs       )
											else:
																if compression:
																					raise NotImplementedError(
																					    f'The compression parameter is not supported when writing to a buffer, but compression={compression}'
																					    """ was passed. Please provide a local path instead."""       )
																snake_case_					:		Any 					=					self._write(
																    file_obj=self.path_or_buf   ,		orient=lowercase__   ,		lines=lowercase__   ,		index=lowercase__   ,		**self.to_json_kwargs       )
											return written







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					:		Dict 					=					args

											snake_case_					:		str 					=					query_table(
											    table=self.dataset.data   ,		key=slice(lowercase__   ,		offset + self.batch_size       )   ,		indices=self.dataset._indices   ,		)
											snake_case_					:		Dict 					=					batch.to_pandas().to_json(
											    path_or_buf=lowercase__   ,		orient=lowercase__   ,		lines=lowercase__   ,		index=lowercase__   ,		**lowercase__       )
											if not json_str.endswith("""\n"""       ):
																json_str += "\n"
											return json_str.encode(self.encoding       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		**lowercase__   ,		):
											snake_case_					:		List[str] 					=					0

											if self.num_proc is None or self.num_proc == 1:
																for offset in logging.tqdm(
																    range(0   ,		len(self.dataset       )   ,		self.batch_size       )   ,		unit="""ba"""   ,		disable=not logging.is_progress_bar_enabled()   ,		desc="""Creating json from Arrow format"""   ,		):
																					snake_case_					:		int 					=					self._batch_json((offset, orient, lines, index, to_json_kwargs)       )
																					written += file_obj.write(lowercase__       )
											else:
																snake_case_					:		Any 					=					len(self.dataset       ), self.batch_size
																with multiprocessing.Pool(self.num_proc       ) as pool:
																					for json_str in logging.tqdm(
																					    pool.imap(
																					        self._batch_json   ,		[(offset, orient, lines, index, to_json_kwargs) for offset in range(0   ,		lowercase__   ,		lowercase__       )]   ,		)   ,		total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   ,		unit="""ba"""   ,		disable=not logging.is_progress_bar_enabled()   ,		desc="""Creating json from Arrow format"""   ,		):
																										written += file_obj.write(lowercase__       )

											return written









"""simple docstring"""

from copy import deepcopy







class    __lowercase     :







						"""simple docstring"""







						def __init__(self   ,		lowercase__ = None   ,		lowercase__ = None       ):

											if arr is None and size is not None:
																snake_case_					:		str 					=					size
																snake_case_					:		Optional[Any] 					=					[0] * size
											elif arr is not None:
																self.init(lowercase__       )
											else:
																raise ValueError("""Either arr or size must be specified"""       )







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					:		Optional[Any] 					=					len(lowercase__       )
											snake_case_					:		int 					=					deepcopy(lowercase__       )
											for i in range(1   ,		self.size       ):
																snake_case_					:		Optional[Any] 					=					self.next_(lowercase__       )
																if j < self.size:
																					self.tree[j] += self.tree[i]







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Dict 					=					self.tree[:]
											for i in range(self.size - 1   ,		0   ,		-1       ):
																snake_case_					:		Optional[int] 					=					self.next_(lowercase__       )
																if j < self.size:
																					arr[j] -= arr[i]
											return arr







						@staticmethod
						def 						__UpperCamelCase							(lowercase__       ):
											return index + (index & (-index))







						@staticmethod
						def 						__UpperCamelCase							(lowercase__       ):
											return index - (index & (-index))







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__       ):
											if index == 0:
																self.tree[0] += value
																return
											while index < self.size:
																self.tree[index] += value
																snake_case_					:		Tuple 					=					self.next_(lowercase__       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__       ):
											self.add(lowercase__   ,		value - self.get(lowercase__       )       )







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											if right == 0:
																return 0
											snake_case_					:		List[str] 					=					self.tree[0]
											right -= 1  # make right inclusive
											while right > 0:
																result += self.tree[right]
																snake_case_					:		Optional[int] 					=					self.prev(lowercase__       )
											return result







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__       ):
											return self.prefix(lowercase__       ) - self.prefix(lowercase__       )







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											return self.query(lowercase__   ,		index + 1       )







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											value -= self.tree[0]
											if value < 0:
																return -1

											snake_case_					:		Tuple 					=					1  # Largest power of 2 <= size
											while j * 2 < self.size:
																j *= 2

											snake_case_					:		Tuple 					=					0

											while j > 0:
																if i + j < self.size and self.tree[i + j] <= value:
																					value -= self.tree[i + j]
																					i += j
																j //= 2
											return i


if __name__ == "__main__":
			import doctest

			doctest.testmod()




