




'''simple docstring'''





import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMInverseScheduler,
    DDIMScheduler,
    DPMSolverMultistepInverseScheduler,
    DPMSolverMultistepScheduler,
    StableDiffusionDiffEditPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_image, slow
from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, require_torch_gpu, torch_device

from ..pipeline_params import TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, TEXT_GUIDED_IMAGE_INPAINTING_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


class 							__UpperCamelCase			( __UpperCAmelCase     ,					__UpperCAmelCase     ,					unittest.TestCase  ):





				'''simple docstring'''



				__a							: Any										=StableDiffusionDiffEditPipeline
				__a							: List[str]										=TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"""height""", """width""", """image"""} | {"""image_latents"""}
				__a							: Union[str, Any]										=TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS - {"""image"""} | {"""image_latents"""}
				__a							: int										=frozenset(
				    []  )  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
				__a							: str										=frozenset([]  )



				def       __snake_case		( self						):
											torch.manual_seed(0						)
											lowerCAmelCase   =		UNetaDConditionModel(
											    block_out_channels=(32, 64)       , layers_per_block=2       , sample_size=32       , in_channels=4       , out_channels=4       , down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')       , up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')       , cross_attention_dim=32       , attention_head_dim=(2, 4)       , use_linear_projection=lowerCAmelCase_       , )
											lowerCAmelCase   =		DDIMScheduler(
											    beta_start=0.00085       , beta_end=0.012       , beta_schedule='''scaled_linear'''       , clip_sample=lowerCAmelCase_       , set_alpha_to_one=lowerCAmelCase_       , )
											lowerCAmelCase   =		DDIMInverseScheduler(
											    beta_start=0.00085       , beta_end=0.012       , beta_schedule='''scaled_linear'''       , clip_sample=lowerCAmelCase_       , set_alpha_to_zero=lowerCAmelCase_       , )
											torch.manual_seed(0						)
											lowerCAmelCase   =		AutoencoderKL(
											    block_out_channels=[32, 64]       , in_channels=3       , out_channels=3       , down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']       , up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']       , latent_channels=4       , sample_size=1_28       , )
											torch.manual_seed(0						)
											lowerCAmelCase   =		CLIPTextConfig(
											    bos_token_id=0       , eos_token_id=2       , hidden_size=32       , intermediate_size=37       , layer_norm_eps=1E-0_5       , num_attention_heads=4       , num_hidden_layers=5       , pad_token_id=1       , vocab_size=10_00       , hidden_act='''gelu'''       , projection_dim=5_12       , )
											lowerCAmelCase   =		CLIPTextModel(lowerCAmelCase_						)
											lowerCAmelCase   =		CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''						)

											lowerCAmelCase   =		{
											    '''unet''': unet,
											    '''scheduler''': scheduler,
											    '''inverse_scheduler''': inverse_scheduler,
											    '''vae''': vae,
											    '''text_encoder''': text_encoder,
											    '''tokenizer''': tokenizer,
											    '''safety_checker''': None,
											    '''feature_extractor''': None,
											}

											return components



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_=0						):
											lowerCAmelCase   =		floats_tensor((1, 16, 16)       , rng=random.Random(lowerCAmelCase_						)						).to(lowerCAmelCase_						)
											lowerCAmelCase   =		floats_tensor((1, 2, 4, 16, 16)       , rng=random.Random(lowerCAmelCase_						)						).to(lowerCAmelCase_						)
											if str(lowerCAmelCase_						).startswith('''mps'''						):
																		lowerCAmelCase   =		torch.manual_seed(lowerCAmelCase_						)
											else:
																		lowerCAmelCase   =		torch.Generator(device=lowerCAmelCase_						).manual_seed(lowerCAmelCase_						)
											lowerCAmelCase   =		{
											    '''prompt''': '''a dog and a newt''',
											    '''mask_image''': mask,
											    '''image_latents''': latents,
											    '''generator''': generator,
											    '''num_inference_steps''': 2,
											    '''inpaint_strength''': 1.0,
											    '''guidance_scale''': 6.0,
											    '''output_type''': '''numpy''',
											}

											return inputs



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_=0						):
											lowerCAmelCase   =		floats_tensor((1, 3, 32, 32)       , rng=random.Random(lowerCAmelCase_						)						).to(lowerCAmelCase_						)
											lowerCAmelCase   =		image.cpu().permute(0       , 2       , 3       , 1						)[0]
											lowerCAmelCase   =		Image.fromarray(np.uinta(lowerCAmelCase_						)						).convert('''RGB'''						)
											if str(lowerCAmelCase_						).startswith('''mps'''						):
																		lowerCAmelCase   =		torch.manual_seed(lowerCAmelCase_						)
											else:
																		lowerCAmelCase   =		torch.Generator(device=lowerCAmelCase_						).manual_seed(lowerCAmelCase_						)
											lowerCAmelCase   =		{
											    '''image''': image,
											    '''source_prompt''': '''a cat and a frog''',
											    '''target_prompt''': '''a dog and a newt''',
											    '''generator''': generator,
											    '''num_inference_steps''': 2,
											    '''num_maps_per_mask''': 2,
											    '''mask_encode_strength''': 1.0,
											    '''guidance_scale''': 6.0,
											    '''output_type''': '''numpy''',
											}

											return inputs



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_=0						):
											lowerCAmelCase   =		floats_tensor((1, 3, 32, 32)       , rng=random.Random(lowerCAmelCase_						)						).to(lowerCAmelCase_						)
											lowerCAmelCase   =		image.cpu().permute(0       , 2       , 3       , 1						)[0]
											lowerCAmelCase   =		Image.fromarray(np.uinta(lowerCAmelCase_						)						).convert('''RGB'''						)
											if str(lowerCAmelCase_						).startswith('''mps'''						):
																		lowerCAmelCase   =		torch.manual_seed(lowerCAmelCase_						)
											else:
																		lowerCAmelCase   =		torch.Generator(device=lowerCAmelCase_						).manual_seed(lowerCAmelCase_						)
											lowerCAmelCase   =		{
											    '''image''': image,
											    '''prompt''': '''a cat and a frog''',
											    '''generator''': generator,
											    '''num_inference_steps''': 2,
											    '''inpaint_strength''': 1.0,
											    '''guidance_scale''': 6.0,
											    '''decode_latents''': True,
											    '''output_type''': '''numpy''',
											}
											return inputs



				def       __snake_case		( self						):







											if not hasattr(self.pipeline_class       , '''_optional_components'''						):
																		return

											lowerCAmelCase   =		self.get_dummy_components()
											lowerCAmelCase   =		self.pipeline_class(**lowerCAmelCase_						)
											pipe.to(lowerCAmelCase_						)
											pipe.set_progress_bar_config(disable=lowerCAmelCase_						)

											# set all optional components to None and update pipeline config accordingly
											for optional_component in pipe._optional_components:
																		setattr(lowerCAmelCase_       , lowerCAmelCase_       , lowerCAmelCase_						)
											pipe.register_modules(**{optional_component: None for optional_component in pipe._optional_components}						)

											lowerCAmelCase   =		self.get_dummy_inputs(lowerCAmelCase_						)
											lowerCAmelCase   =		pipe(**lowerCAmelCase_						)[0]

											with tempfile.TemporaryDirectory() as tmpdir:
																		pipe.save_pretrained(lowerCAmelCase_						)
																		lowerCAmelCase   =		self.pipeline_class.from_pretrained(lowerCAmelCase_						)
																		pipe_loaded.to(lowerCAmelCase_						)
																		pipe_loaded.set_progress_bar_config(disable=lowerCAmelCase_						)

											for optional_component in pipe._optional_components:
																		self.assertTrue(
																		    getattr(lowerCAmelCase_       , lowerCAmelCase_						) is None       , F"""`{optional_component}` did not stay set to None after loading."""       , )

											lowerCAmelCase   =		self.get_dummy_inputs(lowerCAmelCase_						)
											lowerCAmelCase   =		pipe_loaded(**lowerCAmelCase_						)[0]

											lowerCAmelCase   =		np.abs(output - output_loaded						).max()
											self.assertLess(lowerCAmelCase_       , 1E-4						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		'''cpu'''

											lowerCAmelCase   =		self.get_dummy_components()
											lowerCAmelCase   =		self.pipeline_class(**lowerCAmelCase_						)
											pipe.to(lowerCAmelCase_						)
											pipe.set_progress_bar_config(disable=lowerCAmelCase_						)

											lowerCAmelCase   =		self.get_dummy_mask_inputs(lowerCAmelCase_						)
											lowerCAmelCase   =		pipe.generate_mask(**lowerCAmelCase_						)
											lowerCAmelCase   =		mask[0, -3:, -3:]

											self.assertEqual(mask.shape       , (1, 16, 16)						)
											lowerCAmelCase   =		np.array([0] * 9						)
											lowerCAmelCase   =		np.abs(mask_slice.flatten() - expected_slice						).max()
											self.assertLessEqual(lowerCAmelCase_       , 1E-3						)
											self.assertEqual(mask[0, -3, -4]       , 0						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		'''cpu'''

											lowerCAmelCase   =		self.get_dummy_components()
											lowerCAmelCase   =		self.pipeline_class(**lowerCAmelCase_						)
											pipe.to(lowerCAmelCase_						)
											pipe.set_progress_bar_config(disable=lowerCAmelCase_						)

											lowerCAmelCase   =		self.get_dummy_inversion_inputs(lowerCAmelCase_						)
											lowerCAmelCase   =		pipe.invert(**lowerCAmelCase_						).images
											lowerCAmelCase   =		image[0, -1, -3:, -3:]

											self.assertEqual(image.shape       , (2, 32, 32, 3)						)
											lowerCAmelCase   =		np.array(
											    [0.5150, 0.5134, 0.5043, 0.5376, 0.4694, 0.51050, 0.5015, 0.4407, 0.4799]       , )
											lowerCAmelCase   =		np.abs(image_slice.flatten() - expected_slice						).max()
											self.assertLessEqual(lowerCAmelCase_       , 1E-3						)



				def       __snake_case		( self						):
											super().test_inference_batch_single_identical(expected_max_diff=5E-3						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		'''cpu'''

											lowerCAmelCase   =		self.get_dummy_components()

											lowerCAmelCase   =		{'''beta_start''': 0.00085, '''beta_end''': 0.012, '''beta_schedule''': '''scaled_linear'''}
											lowerCAmelCase   =		DPMSolverMultistepScheduler(**lowerCAmelCase_						)
											lowerCAmelCase   =		DPMSolverMultistepInverseScheduler(**lowerCAmelCase_						)

											lowerCAmelCase   =		self.pipeline_class(**lowerCAmelCase_						)
											pipe.to(lowerCAmelCase_						)
											pipe.set_progress_bar_config(disable=lowerCAmelCase_						)

											lowerCAmelCase   =		self.get_dummy_inversion_inputs(lowerCAmelCase_						)
											lowerCAmelCase   =		pipe.invert(**lowerCAmelCase_						).images
											lowerCAmelCase   =		image[0, -1, -3:, -3:]

											self.assertEqual(image.shape       , (2, 32, 32, 3)						)
											lowerCAmelCase   =		np.array(
											    [0.5150, 0.5134, 0.5043, 0.5376, 0.4694, 0.51050, 0.5015, 0.4407, 0.4799]       , )
											lowerCAmelCase   =		np.abs(image_slice.flatten() - expected_slice						).max()
											self.assertLessEqual(lowerCAmelCase_       , 1E-3						)




@require_torch_gpu
@slow
class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self						):
											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()



				@classmethod
				def       __snake_case		( cls						):
											lowerCAmelCase   =		load_image(
											    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/diffedit/fruit.png'''						)

											lowerCAmelCase   =		raw_image.convert('''RGB'''						).resize((7_68, 7_68)						)

											lowerCAmelCase   =		raw_image



				def       __snake_case		( self						):
											lowerCAmelCase   =		torch.manual_seed(0						)

											lowerCAmelCase   =		StableDiffusionDiffEditPipeline.from_pretrained(
											    '''stabilityai/stable-diffusion-2-1'''       , safety_checker=lowerCAmelCase_       , torch_dtype=torch.floataa						)
											lowerCAmelCase   =		DDIMScheduler.from_config(pipe.scheduler.config						)
											lowerCAmelCase   =		DDIMInverseScheduler.from_config(pipe.scheduler.config						)
											pipe.enable_model_cpu_offload()
											pipe.set_progress_bar_config(disable=lowerCAmelCase_						)

											lowerCAmelCase   =		'''a bowl of fruit'''
											lowerCAmelCase   =		'''a bowl of pears'''

											lowerCAmelCase   =		pipe.generate_mask(
											    image=self.raw_image       , source_prompt=lowerCAmelCase_       , target_prompt=lowerCAmelCase_       , generator=lowerCAmelCase_       , )

											lowerCAmelCase   =		pipe.invert(
											    prompt=lowerCAmelCase_       , image=self.raw_image       , inpaint_strength=0.7       , generator=lowerCAmelCase_						).latents

											lowerCAmelCase   =		pipe(
											    prompt=lowerCAmelCase_       , mask_image=lowerCAmelCase_       , image_latents=lowerCAmelCase_       , generator=lowerCAmelCase_       , negative_prompt=lowerCAmelCase_       , inpaint_strength=0.7       , output_type='''numpy'''       , ).images[0]

											lowerCAmelCase   =		(
											    np.array(
											        load_image(
											            '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
											            '''/diffedit/pears.png'''						).resize((7_68, 7_68)						)						)
											    / 2_55
											)
											assert np.abs((expected_image - image).max()						) < 5E-1






				def       __snake_case		( self						):
											lowerCAmelCase   =		torch.manual_seed(0						)

											lowerCAmelCase   =		StableDiffusionDiffEditPipeline.from_pretrained(
											    '''stabilityai/stable-diffusion-2-1'''       , safety_checker=lowerCAmelCase_       , torch_dtype=torch.floataa						)
											lowerCAmelCase   =		DPMSolverMultistepScheduler.from_config(pipe.scheduler.config						)
											lowerCAmelCase   =		DPMSolverMultistepInverseScheduler.from_config(pipe.scheduler.config						)
											pipe.enable_model_cpu_offload()
											pipe.set_progress_bar_config(disable=lowerCAmelCase_						)

											lowerCAmelCase   =		'''a bowl of fruit'''
											lowerCAmelCase   =		'''a bowl of pears'''

											lowerCAmelCase   =		pipe.generate_mask(
											    image=self.raw_image       , source_prompt=lowerCAmelCase_       , target_prompt=lowerCAmelCase_       , generator=lowerCAmelCase_       , )

											lowerCAmelCase   =		pipe.invert(
											    prompt=lowerCAmelCase_       , image=self.raw_image       , inpaint_strength=0.7       , generator=lowerCAmelCase_       , num_inference_steps=25       , ).latents

											lowerCAmelCase   =		pipe(
											    prompt=lowerCAmelCase_       , mask_image=lowerCAmelCase_       , image_latents=lowerCAmelCase_       , generator=lowerCAmelCase_       , negative_prompt=lowerCAmelCase_       , inpaint_strength=0.7       , num_inference_steps=25       , output_type='''numpy'''       , ).images[0]

											lowerCAmelCase   =		(
											    np.array(
											        load_image(
											            '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
											            '''/diffedit/pears.png'''						).resize((7_68, 7_68)						)						)
											    / 2_55
											)
											assert np.abs((expected_image - image).max()						) < 5E-1









import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCAmelCase_						=datasets.utils.logging.get_logger(__name__)


@dataclass
class 							__UpperCamelCase			( datasets.BuilderConfig  ):





				'''simple docstring'''



				__a							: Optional[datasets.Features]										=None
				__a							: str										="utf-8"
				__a							: Optional[str]										=None
				__a							: Optional[str]										=None
				__a							: bool										=True  # deprecated
				__a							: Optional[int]										=None  # deprecated
				__a							: int										=1_0 << 2_0  # 10MB
				__a							: Optional[bool]										=None




class 							__UpperCamelCase			( datasets.ArrowBasedBuilder  ):





				'''simple docstring'''



				__a							: str										=JsonConfig



				def       __snake_case		( self						):
											if self.config.block_size is not None:
																		logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''						)
																		lowerCAmelCase   =		self.config.block_size
											if self.config.use_threads is not True:
																		logger.warning(
																		    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''						)
											if self.config.newlines_in_values is not None:
																		raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''						)
											return datasets.DatasetInfo(features=self.config.features						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											if not self.config.data_files:
																		raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""						)
											lowerCAmelCase   =		dl_manager.download_and_extract(self.config.data_files						)
											if isinstance(UpperCAmelCase_       , (str, list, tuple)						):
																		lowerCAmelCase   =		data_files
																		if isinstance(UpperCAmelCase_       , UpperCAmelCase_						):
																									lowerCAmelCase   =		[files]
																		lowerCAmelCase   =		[dl_manager.iter_files(UpperCAmelCase_						) for file in files]
																		return [datasets.SplitGenerator(name=datasets.Split.TRAIN       , gen_kwargs={'''files''': files}						)]
											lowerCAmelCase   =		[]
											for split_name, files in data_files.items():
																		if isinstance(UpperCAmelCase_       , UpperCAmelCase_						):
																									lowerCAmelCase   =		[files]
																		lowerCAmelCase   =		[dl_manager.iter_files(UpperCAmelCase_						) for file in files]
																		splits.append(datasets.SplitGenerator(name=UpperCAmelCase_       , gen_kwargs={'''files''': files}						)						)
											return splits



				def       __snake_case		( self       , UpperCAmelCase_						):
											if self.config.features is not None:
																		# adding missing columns
																		for column_name in set(self.config.features						) - set(pa_table.column_names						):
																									lowerCAmelCase   =		self.config.features.arrow_schema.field(UpperCAmelCase_						).type
																									lowerCAmelCase   =		pa_table.append_column(UpperCAmelCase_       , pa.array([None] * len(UpperCAmelCase_						)       , type=UpperCAmelCase_						)						)
																		# more expensive cast to support nested structures with keys in a different order
																		# allows str <-> int/float or str to Audio for example
																		lowerCAmelCase   =		table_cast(UpperCAmelCase_       , self.config.features.arrow_schema						)
											return pa_table






				def       __snake_case		( self       , UpperCAmelCase_						):
											for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase_						)						):
																		# If the file is one json object and if we need to look at the list of items in one specific field
																		if self.config.field is not None:
																									with open(UpperCAmelCase_       , encoding=self.config.encoding       , errors=self.config.encoding_errors						) as f:
																																lowerCAmelCase   =		json.load(UpperCAmelCase_						)

																									# We keep only the field we are interested in
																									lowerCAmelCase   =		dataset[self.config.field]

																									# We accept two format: a list of dicts or a dict of lists
																									if isinstance(UpperCAmelCase_       , (list, tuple)						):
																																lowerCAmelCase   =		set().union(*[row.keys() for row in dataset]						)
																																lowerCAmelCase   =		{col: [row.get(UpperCAmelCase_						) for row in dataset] for col in keys}
																									else:
																																lowerCAmelCase   =		dataset
																									lowerCAmelCase   =		pa.Table.from_pydict(UpperCAmelCase_						)
																									yield file_idx, self._cast_table(UpperCAmelCase_						)

																		# If the file has one json object per line
																		else:
																									with open(UpperCAmelCase_       , '''rb'''						) as f:
																																lowerCAmelCase   =		0
																																# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																# Set a default minimum value of 16kB if the chunk size is really small
																																lowerCAmelCase   =		max(self.config.chunksize // 32       , 16 << 10						)
																																lowerCAmelCase   =		(
																																    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																																)
																																while True:
																																							lowerCAmelCase   =		f.read(self.config.chunksize						)
																																							if not batch:
																																														break
																																							# Finish current line
																																							try:
																																														batch += f.readline()
																																							except (AttributeError, io.UnsupportedOperation):
																																														batch += readline(UpperCAmelCase_						)
																																							# PyArrow only accepts utf-8 encoded bytes
																																							if self.config.encoding != "utf-8":
																																														lowerCAmelCase   =		batch.decode(self.config.encoding       , errors=UpperCAmelCase_						).encode('''utf-8'''						)
																																							try:
																																														while True:
																																																					try:
																																																												lowerCAmelCase   =		paj.read_json(
																																																												    io.BytesIO(UpperCAmelCase_						)       , read_options=paj.ReadOptions(block_size=UpperCAmelCase_						)						)
																																																												break
																																																					except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																												if (
																																																												    isinstance(UpperCAmelCase_       , pa.ArrowInvalid						)
																																																												    and "straddling" not in str(UpperCAmelCase_						)
																																																												    or block_size > len(UpperCAmelCase_						)
																																																												):
																																																																			raise
																																																												else:
																																																																			# Increase the block size in case it was too small.
																																																																			# The block size will be reset for the next file.
																																																																			logger.debug(
																																																																			    F"""Batch of {len(UpperCAmelCase_						)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""						)
																																																																			block_size *= 2
																																							except pa.ArrowInvalid as e:
																																														try:
																																																					with open(
																																																					    UpperCAmelCase_       , encoding=self.config.encoding       , errors=self.config.encoding_errors						) as f:
																																																												lowerCAmelCase   =		json.load(UpperCAmelCase_						)
																																														except json.JSONDecodeError:
																																																					logger.error(F"""Failed to read file '{file}' with error {type(UpperCAmelCase_						)}: {e}"""						)
																																																					raise e
																																														# If possible, parse the file as a list of json objects and exit the loop
																																														if isinstance(UpperCAmelCase_       , UpperCAmelCase_						):  # list is the only sequence type supported in JSON
																																																					try:
																																																												lowerCAmelCase   =		set().union(*[row.keys() for row in dataset]						)
																																																												lowerCAmelCase   =		{col: [row.get(UpperCAmelCase_						) for row in dataset] for col in keys}
																																																												lowerCAmelCase   =		pa.Table.from_pydict(UpperCAmelCase_						)
																																																					except (pa.ArrowInvalid, AttributeError) as e:
																																																												logger.error(F"""Failed to read file '{file}' with error {type(UpperCAmelCase_						)}: {e}"""						)
																																																												raise ValueError(F"""Not able to read records in the JSON file at {file}."""						) from None
																																																					yield file_idx, self._cast_table(UpperCAmelCase_						)
																																																					break
																																														else:
																																																					logger.error(F"""Failed to read file '{file}' with error {type(UpperCAmelCase_						)}: {e}"""						)
																																																					raise ValueError(
																																																					    F"""Not able to read records in the JSON file at {file}. """
																																																					    F"""You should probably indicate the field of the JSON file containing your records. """
																																																					    F"""This JSON file contain the following fields: {str(list(dataset.keys()						)						)}. """
																																																					    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """						) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																							yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase_						)
																																							batch_idx += 1



