

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A_ :				Union[str, Any]						 =    get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A_ :				List[str]						 =    25_6047
A_ :				Union[str, Any]						 =    25_6145



@require_sentencepiece
@require_tokenizers
class 					_a		(__magic_name__	,     unittest.TestCase       ):


					'''simple docstring'''






					UpperCAmelCase__:     Any													= NllbTokenizer
					UpperCAmelCase__:     int													= NllbTokenizerFast
					UpperCAmelCase__:     Union[str, Any]													= True
					UpperCAmelCase__:     Union[str, Any]													= True
					UpperCAmelCase__:     List[str]													= {}






					def 						__A    (				self    ):
						super().setUp()

						# We have a SentencePiece fixture for testing
						A__							:					Tuple       =	NllbTokenizer(A__					, keep_accents=A__    )
						tokenizer.save_pretrained(self.tmpdirname    )






					def 						__A    (				self    ):
						A__							:					Optional[int]       =	NllbTokenizer(A__					, keep_accents=A__    )

						A__							:					Dict       =	tokenizer.tokenize("""This is a test"""    )
						self.assertListEqual(A__					, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A__    )					, [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					, )

						A__							:					Dict       =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
						self.assertListEqual(
						    A__					, [
						        SPIECE_UNDERLINE + """I""",
						        SPIECE_UNDERLINE + """was""",
						        SPIECE_UNDERLINE + """b""",
						        """or""",
						        """n""",
						        SPIECE_UNDERLINE + """in""",
						        SPIECE_UNDERLINE + """""",
						        """9""",
						        """2""",
						        """0""",
						        """0""",
						        """0""",
						        """,""",
						        SPIECE_UNDERLINE + """and""",
						        SPIECE_UNDERLINE + """this""",
						        SPIECE_UNDERLINE + """is""",
						        SPIECE_UNDERLINE + """f""",
						        """al""",
						        """s""",
						        """é""",
						        """.""",
						    ]					, )
						A__							:					Dict       =	tokenizer.convert_tokens_to_ids(A__    )
						self.assertListEqual(
						    A__					, [
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]					, )

						A__							:					str       =	tokenizer.convert_ids_to_tokens(A__    )
						self.assertListEqual(
						    A__					, [
						        SPIECE_UNDERLINE + """I""",
						        SPIECE_UNDERLINE + """was""",
						        SPIECE_UNDERLINE + """b""",
						        """or""",
						        """n""",
						        SPIECE_UNDERLINE + """in""",
						        SPIECE_UNDERLINE + """""",
						        """<unk>""",
						        """2""",
						        """0""",
						        """0""",
						        """0""",
						        """,""",
						        SPIECE_UNDERLINE + """and""",
						        SPIECE_UNDERLINE + """this""",
						        SPIECE_UNDERLINE + """is""",
						        SPIECE_UNDERLINE + """f""",
						        """al""",
						        """s""",
						        """<unk>""",
						        """.""",
						    ]					, )






					def 						__A    (				self    ):
						A__							:					int       =	(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
								A__							:					Tuple       =	self.rust_tokenizer_class.from_pretrained(A__					, **A__    )
								A__							:					List[Any]       =	self.tokenizer_class.from_pretrained(A__					, **A__    )

								A__							:					Optional[int]       =	tempfile.mkdtemp()

								A__							:					str       =	tokenizer_r.save_pretrained(A__    )
								A__							:					str       =	tokenizer_p.save_pretrained(A__    )

								# Checks it save with the same files + the tokenizer.json file for the fast one
								self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )
								A__							:					List[str]       =	tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f    )
								self.assertSequenceEqual(A__					, A__    )

								# Checks everything loads correctly in the same way
								A__							:					str       =	tokenizer_r.from_pretrained(A__    )
								A__							:					Union[str, Any]       =	tokenizer_p.from_pretrained(A__    )

								# Check special tokens are set accordingly on Rust and Python
								for key in tokenizer_pp.special_tokens_map:
									self.assertTrue(hasattr(A__					, A__    )    )

								shutil.rmtree(A__    )

								# Save tokenizer rust, legacy_format=True
								A__							:					str       =	tempfile.mkdtemp()

								A__							:					Optional[Any]       =	tokenizer_r.save_pretrained(A__					, legacy_format=A__    )
								A__							:					Union[str, Any]       =	tokenizer_p.save_pretrained(A__    )

								# Checks it save with the same files
								self.assertSequenceEqual(A__					, A__    )

								# Checks everything loads correctly in the same way
								A__							:					Optional[int]       =	tokenizer_r.from_pretrained(A__    )
								A__							:					Union[str, Any]       =	tokenizer_p.from_pretrained(A__    )

								# Check special tokens are set accordingly on Rust and Python
								for key in tokenizer_pp.special_tokens_map:
									self.assertTrue(hasattr(A__					, A__    )    )

								shutil.rmtree(A__    )

								# Save tokenizer rust, legacy_format=False
								A__							:					Dict       =	tempfile.mkdtemp()

								A__							:					int       =	tokenizer_r.save_pretrained(A__					, legacy_format=A__    )
								A__							:					Optional[int]       =	tokenizer_p.save_pretrained(A__    )

								# Checks it saved the tokenizer.json file
								self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )

								# Checks everything loads correctly in the same way
								A__							:					int       =	tokenizer_r.from_pretrained(A__    )
								A__							:					str       =	tokenizer_p.from_pretrained(A__    )

								# Check special tokens are set accordingly on Rust and Python
								for key in tokenizer_pp.special_tokens_map:
									self.assertTrue(hasattr(A__					, A__    )    )

								shutil.rmtree(A__    )






					@require_torch
					def 						__A    (				self    ):
						if not self.test_seqaseq:
							return

						A__							:					List[Any]       =	self.get_tokenizers()
						for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
								# Longer text that will definitely require truncation.
								A__							:					Tuple       =	[
								    """ UN Chief Says There Is No Military Solution in Syria""",
								    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
								    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
								    """ will only worsen the violence and misery for millions of people.""",
								]
								A__							:					Tuple       =	[
								    """Şeful ONU declară că nu există o soluţie militară în Siria""",
								    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
								    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
								    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
								]
								try:
									A__							:					Dict       =	tokenizer.prepare_seqaseq_batch(
									    src_texts=A__					, tgt_texts=A__					, max_length=3					, max_target_length=10					, return_tensors="""pt"""					, src_lang="""eng_Latn"""					, tgt_lang="""ron_Latn"""					, )
								except NotImplementedError:
									return
								self.assertEqual(batch.input_ids.shape[1]					, 3    )
								self.assertEqual(batch.labels.shape[1]					, 10    )
								# max_target_length will default to max_length if not specified
								A__							:					Optional[int]       =	tokenizer.prepare_seqaseq_batch(
								    A__					, tgt_texts=A__					, max_length=3					, return_tensors="""pt"""    )
								self.assertEqual(batch.input_ids.shape[1]					, 3    )
								self.assertEqual(batch.labels.shape[1]					, 3    )

								A__							:					List[Any]       =	tokenizer.prepare_seqaseq_batch(
								    src_texts=A__					, max_length=3					, max_target_length=10					, return_tensors="""pt"""    )
								self.assertEqual(batch_encoder_only.input_ids.shape[1]					, 3    )
								self.assertEqual(batch_encoder_only.attention_mask.shape[1]					, 3    )
								self.assertNotIn("""decoder_input_ids"""					, A__    )






					@unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""    )
					def 						__A    (				self    ):
						pass






					def 						__A    (				self    ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
								A__							:					int       =	[AddedToken("""<special>"""					, lstrip=A__    )]

								A__							:					List[str]       =	self.rust_tokenizer_class.from_pretrained(
								    A__					, additional_special_tokens=A__					, **A__    )
								A__							:					Tuple       =	tokenizer_r.encode("""Hey this is a <special> token"""    )

								A__							:					int       =	tokenizer_r.encode("""<special>"""					, add_special_tokens=A__    )[0]

								self.assertTrue(special_token_id in r_output    )

								if self.test_slow_tokenizer:
									A__							:					Dict       =	self.rust_tokenizer_class.from_pretrained(
									    A__					, additional_special_tokens=A__					, **A__					, )
									A__							:					Union[str, Any]       =	self.tokenizer_class.from_pretrained(
									    A__					, additional_special_tokens=A__					, **A__    )

									A__							:					Tuple       =	tokenizer_p.encode("""Hey this is a <special> token"""    )

									A__							:					Any       =	tokenizer_cr.encode("""Hey this is a <special> token"""    )

									self.assertEqual(A__					, A__    )
									self.assertEqual(A__					, A__    )
									self.assertTrue(special_token_id in p_output    )
									self.assertTrue(special_token_id in cr_output    )



@require_torch
@require_sentencepiece
@require_tokenizers
class 					_a		(unittest.TestCase       ):


					'''simple docstring'''






					UpperCAmelCase__:     Optional[Any]													= '''facebook/nllb-200-distilled-600M'''
					UpperCAmelCase__:     List[Any]													= [
					    ''' UN Chief Says There Is No Military Solution in Syria''',
					    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
					]
					UpperCAmelCase__:     Optional[Any]													= [
					    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
					    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
					    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
					    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
					]
					UpperCAmelCase__:     Optional[int]													= [
					    25_60_47,
					    1_62_97,
					    13_44_08,
					    81_65,
					    24_80_66,
					    1_47_34,
					    9_50,
					    11_35,
					    10_57_21,
					    35_73,
					    83,
					    2_73_52,
					    1_08,
					    4_94_86,
					    2,
					]






					@classmethod
					def 						__A    (				cls    ):
						A__							:					NllbTokenizer       =	NllbTokenizer.from_pretrained(
						    cls.checkpoint_name					, src_lang="""eng_Latn"""					, tgt_lang="""ron_Latn"""    )
						A__							:					Any       =	1
						return cls






					def 						__A    (				self    ):
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]					, 25_6001    )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]					, 25_6002    )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]					, 25_6057    )






					def 						__A    (				self    ):
						A__							:					List[Any]       =	self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
						self.assertListEqual(self.expected_src_tokens					, A__    )






					def 						__A    (				self    ):
						self.assertIn(A__					, self.tokenizer.all_special_ids    )
						# fmt: off
						A__							:					Union[str, Any]       =	[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
						# fmt: on

						A__							:					List[Any]       =	self.tokenizer.decode(A__					, skip_special_tokens=A__    )
						A__							:					Optional[int]       =	self.tokenizer.decode(generated_ids[1:]					, skip_special_tokens=A__    )
						self.assertEqual(A__					, A__    )
						self.assertNotIn(self.tokenizer.eos_token					, A__    )






					def 						__A    (				self    ):
						A__							:					Optional[int]       =	["""this is gunna be a long sentence """ * 20]
						assert isinstance(src_text[0]					, A__    )
						A__							:					int       =	10
						A__							:					List[Any]       =	self.tokenizer(A__					, max_length=A__					, truncation=A__    ).input_ids[0]
						self.assertEqual(ids[-1]					, 2    )
						self.assertEqual(ids[0]					, A__    )
						self.assertEqual(len(A__    )					, A__    )






					def 						__A    (				self    ):
						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]    )					, [25_6203, 3]    )






					def 						__A    (				self    ):
						A__							:					int       =	tempfile.mkdtemp()
						A__							:					str       =	self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(A__    )
						A__							:					List[str]       =	NllbTokenizer.from_pretrained(A__    )
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids					, A__    )






					@require_torch
					def 						__A    (				self    ):
						A__							:					List[str]       =	self.tokenizer(
						    self.src_text					, text_target=self.tgt_text					, padding=A__					, truncation=A__					, max_length=len(self.expected_src_tokens    )					, return_tensors="""pt"""					, )
						A__							:					Optional[Any]       =	shift_tokens_right(
						    batch["""labels"""]					, self.tokenizer.pad_token_id					, self.tokenizer.lang_code_to_id["""ron_Latn"""]    )

						self.assertIsInstance(A__					, A__    )

						self.assertEqual((2, 15)					, batch.input_ids.shape    )
						self.assertEqual((2, 15)					, batch.attention_mask.shape    )
						A__							:					int       =	batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens					, A__    )
						self.assertEqual(A__					, batch.decoder_input_ids[0, 0]    )  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens					, [EN_CODE]    )
						self.assertEqual(self.tokenizer.suffix_tokens					, [self.tokenizer.eos_token_id]    )






					def 						__A    (				self    ):
						A__							:					List[str]       =	self.tokenizer(self.src_text					, padding=A__					, truncation=A__					, max_length=3					, return_tensors="""pt"""    )
						A__							:					str       =	self.tokenizer(
						    text_target=self.tgt_text					, padding=A__					, truncation=A__					, max_length=10					, return_tensors="""pt"""    )
						A__							:					Optional[int]       =	targets["""input_ids"""]
						A__							:					Dict       =	shift_tokens_right(
						    A__					, self.tokenizer.pad_token_id					, decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					, )

						self.assertEqual(batch.input_ids.shape[1]					, 3    )
						self.assertEqual(batch.decoder_input_ids.shape[1]					, 10    )






					@require_torch
					def 						__A    (				self    ):
						A__							:					List[Any]       =	self.tokenizer._build_translation_inputs(
						    """A test"""					, return_tensors="""pt"""					, src_lang="""eng_Latn"""					, tgt_lang="""fra_Latn"""    )

						self.assertEqual(
						    nested_simplify(A__    )					, {
						        # A, test, EOS, en_XX
						        """input_ids""": [[25_6047, 70, 7356, 2]],
						        """attention_mask""": [[1, 1, 1, 1]],
						        # ar_AR
						        """forced_bos_token_id""": 25_6057,
						    }					, )






					@require_torch
					def 						__A    (				self    ):
						A__							:					str       =	True
						A__							:					int       =	self.tokenizer(
						    """UN Chief says there is no military solution in Syria"""					, src_lang="""eng_Latn"""					, tgt_lang="""fra_Latn"""    )
						self.assertEqual(
						    inputs.input_ids					, [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]    )

						A__							:					Any       =	False
						A__							:					Any       =	self.tokenizer(
						    """UN Chief says there is no military solution in Syria"""					, src_lang="""eng_Latn"""					, tgt_lang="""fra_Latn"""    )
						self.assertEqual(
						    inputs.input_ids					, [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]    )






import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


A_ :				Tuple						 =    datasets.utils.logging.get_logger(__name__)



@dataclass
class 					_a		(datasets.BuilderConfig       ):


					'''simple docstring'''






					UpperCAmelCase__:     Optional[datasets.Features]													= None
					UpperCAmelCase__:     str													= "utf-8"
					UpperCAmelCase__:     Optional[str]													= None
					UpperCAmelCase__:     Optional[str]													= None
					UpperCAmelCase__:     bool													= True  # deprecated
					UpperCAmelCase__:     Optional[int]													= None  # deprecated
					UpperCAmelCase__:     int													= 10 << 20  # 10MB
					UpperCAmelCase__:     Optional[bool]													= None



class 					_a		(datasets.ArrowBasedBuilder       ):


					'''simple docstring'''






					UpperCAmelCase__:     List[str]													= JsonConfig






					def 						__A    (				self    ):
						if self.config.block_size is not None:
							logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""    )
							A__							:					Union[str, Any]       =	self.config.block_size
						if self.config.use_threads is not True:
							logger.warning(
							    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""    )
						if self.config.newlines_in_values is not None:
							raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""    )
						return datasets.DatasetInfo(features=self.config.features    )






					def 						__A    (				self					, A__    ):
						if not self.config.data_files:
							raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
						A__							:					int       =	dl_manager.download_and_extract(self.config.data_files    )
						if isinstance(A__					, (str, list, tuple)    ):
							A__							:					Optional[Any]       =	data_files
							if isinstance(A__					, A__    ):
								A__							:					List[str]       =	[files]
							A__							:					int       =	[dl_manager.iter_files(A__    ) for file in files]
							return [datasets.SplitGenerator(name=datasets.Split.TRAIN					, gen_kwargs={"""files""": files}    )]
						A__							:					List[str]       =	[]
						for split_name, files in data_files.items():
							if isinstance(A__					, A__    ):
								A__							:					Optional[int]       =	[files]
							A__							:					Optional[int]       =	[dl_manager.iter_files(A__    ) for file in files]
							splits.append(datasets.SplitGenerator(name=A__					, gen_kwargs={"""files""": files}    )    )
						return splits






					def 						__A    (				self					, A__    ):
						if self.config.features is not None:
							# adding missing columns
							for column_name in set(self.config.features    ) - set(pa_table.column_names    ):
								A__							:					Optional[Any]       =	self.config.features.arrow_schema.field(A__    ).type
								A__							:					str       =	pa_table.append_column(A__					, pa.array([None] * len(A__    )					, type=A__    )    )
							# more expensive cast to support nested structures with keys in a different order
							# allows str <-> int/float or str to Audio for example
							A__							:					Optional[int]       =	table_cast(A__					, self.config.features.arrow_schema    )
						return pa_table






					def 						__A    (				self					, A__    ):
						for file_idx, file in enumerate(itertools.chain.from_iterable(A__    )    ):
							# If the file is one json object and if we need to look at the list of items in one specific field
							if self.config.field is not None:
								with open(A__					, encoding=self.config.encoding					, errors=self.config.encoding_errors    ) as f:
									A__							:					Optional[Any]       =	json.load(A__    )

								# We keep only the field we are interested in
								A__							:					Optional[int]       =	dataset[self.config.field]

								# We accept two format: a list of dicts or a dict of lists
								if isinstance(A__					, (list, tuple)    ):
									A__							:					Union[str, Any]       =	set().union(*[row.keys() for row in dataset]    )
									A__							:					Any       =	{col: [row.get(A__    ) for row in dataset] for col in keys}
								else:
									A__							:					Any       =	dataset
								A__							:					Any       =	pa.Table.from_pydict(A__    )
								yield file_idx, self._cast_table(A__    )

							# If the file has one json object per line
							else:
								with open(A__					, """rb"""    ) as f:
									A__							:					List[str]       =	0
									# Use block_size equal to the chunk size divided by 32 to leverage multithreading
									# Set a default minimum value of 16kB if the chunk size is really small
									A__							:					List[str]       =	max(self.config.chunksize // 32					, 16 << 10    )
									A__							:					Any       =	(
									    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
									)
									while True:
										A__							:					Dict       =	f.read(self.config.chunksize    )
										if not batch:
											break
										# Finish current line
										try:
											batch += f.readline()
										except (AttributeError, io.UnsupportedOperation):
											batch += readline(A__    )
										# PyArrow only accepts utf-8 encoded bytes
										if self.config.encoding != "utf-8":
											A__							:					List[Any]       =	batch.decode(self.config.encoding					, errors=A__    ).encode("""utf-8"""    )
										try:
											while True:
												try:
													A__							:					str       =	paj.read_json(
													    io.BytesIO(A__    )					, read_options=paj.ReadOptions(block_size=A__    )    )
													break
												except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
													if (
													    isinstance(A__					, pa.ArrowInvalid    )
													    and "straddling" not in str(A__    )
													    or block_size > len(A__    )
													):
														raise
													else:
														# Increase the block size in case it was too small.
														# The block size will be reset for the next file.
														logger.debug(
														    F"""Batch of {len(A__    )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""    )
														block_size *= 2
										except pa.ArrowInvalid as e:
											try:
												with open(
												    A__					, encoding=self.config.encoding					, errors=self.config.encoding_errors    ) as f:
													A__							:					Optional[Any]       =	json.load(A__    )
											except json.JSONDecodeError:
												logger.error(F"""Failed to read file '{file}' with error {type(A__    )}: {e}"""    )
												raise e
											# If possible, parse the file as a list of json objects and exit the loop
											if isinstance(A__					, A__    ):  # list is the only sequence type supported in JSON
												try:
													A__							:					str       =	set().union(*[row.keys() for row in dataset]    )
													A__							:					List[str]       =	{col: [row.get(A__    ) for row in dataset] for col in keys}
													A__							:					int       =	pa.Table.from_pydict(A__    )
												except (pa.ArrowInvalid, AttributeError) as e:
													logger.error(F"""Failed to read file '{file}' with error {type(A__    )}: {e}"""    )
													raise ValueError(F"""Not able to read records in the JSON file at {file}."""    ) from None
												yield file_idx, self._cast_table(A__    )
												break
											else:
												logger.error(F"""Failed to read file '{file}' with error {type(A__    )}: {e}"""    )
												raise ValueError(
												    F"""Not able to read records in the JSON file at {file}. """
												    F"""You should probably indicate the field of the JSON file containing your records. """
												    F"""This JSON file contain the following fields: {str(list(dataset.keys()    )    )}. """
												    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """    ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
										yield (file_idx, batch_idx), self._cast_table(A__    )
										batch_idx += 1



