






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class 		_snake_case  (  A__					):






			def __init__(       self		,					a		,					a = None		,					a = None		,					a = None		,					a = False		,					a = False		,					a = None		,					a = None		,					**a		,					)      ->       List[Any]:
						super().__init__(
						    a		,					split=a		,					features=a		,					cache_dir=a		,					keep_in_memory=a		,					streaming=a		,					num_proc=a		,					**a		,					)
						SCREAMING_SNAKE_CASE						= field
						SCREAMING_SNAKE_CASE						= path_or_paths if isinstance(a		,					a) else {self.split: path_or_paths}
						SCREAMING_SNAKE_CASE						= Json(
						    cache_dir=a		,					data_files=a		,					features=a		,					field=a		,					**a		,					)







			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Optional[int]:
						# Build iterable dataset
						if self.streaming:
									SCREAMING_SNAKE_CASE						= self.builder.as_streaming_dataset(split=self.split)
						# Build regular (map-style) dataset
						else:
									SCREAMING_SNAKE_CASE						= None
									SCREAMING_SNAKE_CASE						= None
									SCREAMING_SNAKE_CASE						= None
									SCREAMING_SNAKE_CASE						= None

									self.builder.download_and_prepare(
									    download_config=a		,					download_mode=a		,					verification_mode=a		,					base_path=a		,					num_proc=self.num_proc		,					)
									SCREAMING_SNAKE_CASE						= self.builder.as_dataset(
									    split=self.split		,					verification_mode=a		,					in_memory=self.keep_in_memory)
						return dataset



class 		_snake_case  :






			def __init__(       self		,					a		,					a		,					a = None		,					a = None		,					**a		,					)      ->       Optional[Any]:
						if num_proc is not None and num_proc <= 0:
									raise ValueError(f'''num_proc {num_proc} must be an integer > 0.''')

						SCREAMING_SNAKE_CASE						= dataset
						SCREAMING_SNAKE_CASE						= path_or_buf
						SCREAMING_SNAKE_CASE						= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
						SCREAMING_SNAKE_CASE						= num_proc
						SCREAMING_SNAKE_CASE						= 'utf-8'
						SCREAMING_SNAKE_CASE						= to_json_kwargs






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       int:
						SCREAMING_SNAKE_CASE						= self.to_json_kwargs.pop('path_or_buf'		,					a)
						SCREAMING_SNAKE_CASE						= self.to_json_kwargs.pop('orient'		,					'records')
						SCREAMING_SNAKE_CASE						= self.to_json_kwargs.pop('lines'		,					True if orient == 'records' else False)
						SCREAMING_SNAKE_CASE						= self.to_json_kwargs.pop('index'		,					False if orient in ['split', 'table'] else True)
						SCREAMING_SNAKE_CASE						= self.to_json_kwargs.pop('compression'		,					a)

						if compression not in [None, "infer", "gzip", "bz2", "xz"]:
									raise NotImplementedError(f'''`datasets` currently does not support {compression} compression''')

						if isinstance(self.path_or_buf		,					(str, bytes, os.PathLike)):
									with fsspec.open(self.path_or_buf		,					'wb'		,					compression=a) as buffer:
												SCREAMING_SNAKE_CASE						= self._write(file_obj=a		,					orient=a		,					lines=a		,					index=a		,					**self.to_json_kwargs)
						else:
									if compression:
												raise NotImplementedError(
												    f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
												    ' was passed. Please provide a local path instead.')
									SCREAMING_SNAKE_CASE						= self._write(
									    file_obj=self.path_or_buf		,					orient=a		,					lines=a		,					index=a		,					**self.to_json_kwargs)
						return written






			def    SCREAMING_SNAKE_CASE__    (       self		,					a)      ->       Tuple:
						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= args

						SCREAMING_SNAKE_CASE						= query_table(
						    table=self.dataset.data		,					key=slice(a		,					offset + self.batch_size)		,					indices=self.dataset._indices		,					)
						SCREAMING_SNAKE_CASE						= batch.to_pandas().to_json(
						    path_or_buf=a		,					orient=a		,					lines=a		,					index=a		,					**a)
						if not json_str.endswith('\n'):
									json_str += "\n"
						return json_str.encode(self.encoding)







			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a		,					a		,					a		,					**a		,					)      ->       int:
						SCREAMING_SNAKE_CASE						= 0

						if self.num_proc is None or self.num_proc == 1:
									for offset in logging.tqdm(
									    range(0		,					len(self.dataset)		,					self.batch_size)		,					unit='ba'		,					disable=not logging.is_progress_bar_enabled()		,					desc='Creating json from Arrow format'		,					):
												SCREAMING_SNAKE_CASE						= self._batch_json((offset, orient, lines, index, to_json_kwargs))
												written += file_obj.write(a)
						else:
									SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= len(self.dataset), self.batch_size
									with multiprocessing.Pool(self.num_proc) as pool:
												for json_str in logging.tqdm(
												    pool.imap(
												        self._batch_json		,					[(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,					a		,					a)]		,					)		,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,					unit='ba'		,					disable=not logging.is_progress_bar_enabled()		,					desc='Creating json from Arrow format'		,					):
															written += file_obj.write(a)

						return written












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a_		:						List[Any]             =    {
    'configuration_bigbird_pegasus': [
        'BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'BigBirdPegasusConfig',
        'BigBirdPegasusOnnxConfig',
    ],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				a_		:						Dict             =    [
				    'BIGBIRD_PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'BigBirdPegasusForCausalLM',
				    'BigBirdPegasusForConditionalGeneration',
				    'BigBirdPegasusForQuestionAnswering',
				    'BigBirdPegasusForSequenceClassification',
				    'BigBirdPegasusModel',
				    'BigBirdPegasusPreTrainedModel',
				]


if TYPE_CHECKING:
				from .configuration_bigbird_pegasus import (
				    BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP,
				    BigBirdPegasusConfig,
				    BigBirdPegasusOnnxConfig,
				)

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_bigbird_pegasus import (
								    BIGBIRD_PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST,
								    BigBirdPegasusForCausalLM,
								    BigBirdPegasusForConditionalGeneration,
								    BigBirdPegasusForQuestionAnswering,
								    BigBirdPegasusForSequenceClassification,
								    BigBirdPegasusModel,
								    BigBirdPegasusPreTrainedModel,
								)


else:
				import sys

				a_		:						Tuple             =    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




