


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__SCREAMING_SNAKE_CASE							:    Tuple		=    datasets.utils.logging.get_logger(__name__)





@dataclass
class      __A     (datasets.BuilderConfig):





							'''simple docstring'''

							__lowercase:		Optional[datasets.Features]							      =    None
							__lowercase:		str							      =    "utf-8"
							__lowercase:		Optional[str]							      =    None
							__lowercase:		Optional[str]							      =    None
							__lowercase:		bool							      =    True  # deprecated
							__lowercase:		Optional[int]							      =    None  # deprecated
							__lowercase:		int							      =    10 << 20  # 10MB
							__lowercase:		Optional[bool]							      =    None





class      __A     (datasets.ArrowBasedBuilder):





							'''simple docstring'''

							__lowercase:		List[str]							      =    JsonConfig



							def 		lowerCAmelCase       (				self							:      Optional[Any]       )					->Optional[Any]:



														"""simple docstring"""



														if self.config.block_size is not None:
																					logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""       )
																					snake_case_           =		self.config.block_size
														if self.config.use_threads is not True:
																					logger.warning(
																					    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""       )
														if self.config.newlines_in_values is not None:
																					raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""       )
														return datasets.DatasetInfo(features=self.config.features       )



							def 		lowerCAmelCase       (				self							:      Optional[Any]		,       UpperCAmelCase_							:      Optional[int]       )					->Optional[Any]:



														"""simple docstring"""



														if not self.config.data_files:
																					raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""       )
														snake_case_           =		dl_manager.download_and_extract(self.config.data_files       )
														if isinstance(A__		,       (str, list, tuple)       ):
																					snake_case_           =		data_files
																					if isinstance(A__		,       A__       ):
																												snake_case_           =		[files]
																					snake_case_           =		[dl_manager.iter_files(A__       ) for file in files]
																					return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,       gen_kwargs={"""files""": files}       )]
														snake_case_           =		[]
														for split_name, files in data_files.items():
																					if isinstance(A__		,       A__       ):
																												snake_case_           =		[files]
																					snake_case_           =		[dl_manager.iter_files(A__       ) for file in files]
																					splits.append(datasets.SplitGenerator(name=A__		,       gen_kwargs={"""files""": files}       )       )
														return splits



							def 		lowerCAmelCase       (				self							:      int		,       UpperCAmelCase_							:      Dict       )					->Optional[int]:



														"""simple docstring"""



														if self.config.features is not None:
																					# adding missing columns
																					for column_name in set(self.config.features       ) - set(pa_table.column_names       ):
																												snake_case_           =		self.config.features.arrow_schema.field(A__       ).type
																												snake_case_           =		pa_table.append_column(A__		,       pa.array([None] * len(A__       )		,       type=A__       )       )
																					# more expensive cast to support nested structures with keys in a different order
																					# allows str <-> int/float or str to Audio for example
																					snake_case_           =		table_cast(A__		,       self.config.features.arrow_schema       )
														return pa_table



							def 		lowerCAmelCase       (				self							:      Optional[int]		,       UpperCAmelCase_							:      Tuple       )					->Any:



														"""simple docstring"""



														for file_idx, file in enumerate(itertools.chain.from_iterable(A__       )       ):
																					# If the file is one json object and if we need to look at the list of items in one specific field
																					if self.config.field is not None:
																												with open(A__		,       encoding=self.config.encoding		,       errors=self.config.encoding_errors       ) as f:
																																			snake_case_           =		json.load(A__       )

																												# We keep only the field we are interested in
																												snake_case_           =		dataset[self.config.field]

																												# We accept two format: a list of dicts or a dict of lists
																												if isinstance(A__		,       (list, tuple)       ):
																																			snake_case_           =		set().union(*[row.keys() for row in dataset]       )
																																			snake_case_           =		{col: [row.get(A__       ) for row in dataset] for col in keys}
																												else:
																																			snake_case_           =		dataset
																												snake_case_           =		pa.Table.from_pydict(A__       )
																												yield file_idx, self._cast_table(A__       )

																					# If the file has one json object per line
																					else:
																												with open(A__		,       """rb"""       ) as f:
																																			snake_case_           =		0
																																			# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																			# Set a default minimum value of 16kB if the chunk size is really small
																																			snake_case_           =		max(self.config.chunksize // 32		,       16 << 10       )
																																			snake_case_           =		(
																																			    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																																			)
																																			while True:
																																										snake_case_           =		f.read(self.config.chunksize       )
																																										if not batch:
																																																	break
																																										# Finish current line
																																										try:
																																																	batch += f.readline()
																																										except (AttributeError, io.UnsupportedOperation):
																																																	batch += readline(A__       )
																																										# PyArrow only accepts utf-8 encoded bytes
																																										if self.config.encoding != "utf-8":
																																																	snake_case_           =		batch.decode(self.config.encoding		,       errors=A__       ).encode("""utf-8"""       )
																																										try:
																																																	while True:
																																																								try:
																																																															snake_case_           =		paj.read_json(
																																																															    io.BytesIO(A__       )		,       read_options=paj.ReadOptions(block_size=A__       )       )
																																																															break
																																																								except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																															if (
																																																															    isinstance(A__		,       pa.ArrowInvalid       )
																																																															    and "straddling" not in str(A__       )
																																																															    or block_size > len(A__       )
																																																															):
																																																																						raise
																																																															else:
																																																																						# Increase the block size in case it was too small.
																																																																						# The block size will be reset for the next file.
																																																																						logger.debug(
																																																																						    F"""Batch of {len(A__       )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""       )
																																																																						block_size *= 2
																																										except pa.ArrowInvalid as e:
																																																	try:
																																																								with open(
																																																								    A__		,       encoding=self.config.encoding		,       errors=self.config.encoding_errors       ) as f:
																																																															snake_case_           =		json.load(A__       )
																																																	except json.JSONDecodeError:
																																																								logger.error(F"""Failed to read file '{file}' with error {type(A__       )}: {e}"""       )
																																																								raise e
																																																	# If possible, parse the file as a list of json objects and exit the loop
																																																	if isinstance(A__		,       A__       ):  # list is the only sequence type supported in JSON
																																																								try:
																																																															snake_case_           =		set().union(*[row.keys() for row in dataset]       )
																																																															snake_case_           =		{col: [row.get(A__       ) for row in dataset] for col in keys}
																																																															snake_case_           =		pa.Table.from_pydict(A__       )
																																																								except (pa.ArrowInvalid, AttributeError) as e:
																																																															logger.error(F"""Failed to read file '{file}' with error {type(A__       )}: {e}"""       )
																																																															raise ValueError(F"""Not able to read records in the JSON file at {file}."""       ) from None
																																																								yield file_idx, self._cast_table(A__       )
																																																								break
																																																	else:
																																																								logger.error(F"""Failed to read file '{file}' with error {type(A__       )}: {e}"""       )
																																																								raise ValueError(
																																																								    F"""Not able to read records in the JSON file at {file}. """
																																																								    F"""You should probably indicate the field of the JSON file containing your records. """
																																																								    F"""This JSON file contain the following fields: {str(list(dataset.keys()       )       )}. """
																																																								    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """       ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield (file_idx, batch_idx), self._cast_table(A__       )
																																										batch_idx += 1







"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__SCREAMING_SNAKE_CASE							:    Optional[Any]		=    {
    'configuration_megatron_bert': ['MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegatronBertConfig'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__SCREAMING_SNAKE_CASE							:    Tuple		=    [
				    'MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'MegatronBertForCausalLM',
				    'MegatronBertForMaskedLM',
				    'MegatronBertForMultipleChoice',
				    'MegatronBertForNextSentencePrediction',
				    'MegatronBertForPreTraining',
				    'MegatronBertForQuestionAnswering',
				    'MegatronBertForSequenceClassification',
				    'MegatronBertForTokenClassification',
				    'MegatronBertModel',
				    'MegatronBertPreTrainedModel',
				]

if TYPE_CHECKING:
				from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_megatron_bert import (
								    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    MegatronBertForCausalLM,
								    MegatronBertForMaskedLM,
								    MegatronBertForMultipleChoice,
								    MegatronBertForNextSentencePrediction,
								    MegatronBertForPreTraining,
								    MegatronBertForQuestionAnswering,
								    MegatronBertForSequenceClassification,
								    MegatronBertForTokenClassification,
								    MegatronBertModel,
								    MegatronBertPreTrainedModel,
								)

else:
				import sys

				__SCREAMING_SNAKE_CASE							:    int		=    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



