




'''simple docstring'''


import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





class    __lowercase						(  A_ ):







   def __init__(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = False      ,							UpperCamelCase = False      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    List[Any]:
     super().__init__(
         UpperCamelCase      ,							split=UpperCamelCase      ,							features=UpperCamelCase      ,							cache_dir=UpperCamelCase      ,							keep_in_memory=UpperCamelCase      ,							streaming=UpperCamelCase      ,							num_proc=UpperCamelCase      ,							**UpperCamelCase      ,							)
     __a			=					field
     __a			=					path_or_paths if isinstance(UpperCamelCase      ,							UpperCamelCase      ) else {self.split: path_or_paths}
     __a			=					Json(
         cache_dir=UpperCamelCase      ,							data_files=UpperCamelCase      ,							features=UpperCamelCase      ,							field=UpperCamelCase      ,							**UpperCamelCase      ,							)







   def 		UpperCamelCase__			(	self      )				->    Optional[int]:
     # Build iterable dataset
     if self.streaming:
       __a			=					self.builder.as_streaming_dataset(split=self.split      )
     # Build regular (map-style) dataset
     else:
       __a			=					None
       __a			=					None
       __a			=					None
       __a			=					None

       self.builder.download_and_prepare(
           download_config=UpperCamelCase      ,							download_mode=UpperCamelCase      ,							verification_mode=UpperCamelCase      ,							base_path=UpperCamelCase      ,							num_proc=self.num_proc      ,							)
       __a			=					self.builder.as_dataset(
           split=self.split      ,							verification_mode=UpperCamelCase      ,							in_memory=self.keep_in_memory      )
     return dataset





class    __lowercase						:







   def __init__(	self      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    List[str]:
     if num_proc is not None and num_proc <= 0:
       raise ValueError(f"num_proc {num_proc} must be an integer > 0."      )

     __a			=					dataset
     __a			=					path_or_buf
     __a			=					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
     __a			=					num_proc
     __a			=					"""utf-8"""
     __a			=					to_json_kwargs







   def 		UpperCamelCase__			(	self      )				->    Tuple:
     __a			=					self.to_json_kwargs.pop('path_or_buf'      ,							UpperCamelCase      )
     __a			=					self.to_json_kwargs.pop('orient'      ,							'records'      )
     __a			=					self.to_json_kwargs.pop('lines'      ,							True if orient == 'records' else False      )
     __a			=					self.to_json_kwargs.pop('index'      ,							False if orient in ['split', 'table'] else True      )
     __a			=					self.to_json_kwargs.pop('compression'      ,							UpperCamelCase      )

     if compression not in [None, "infer", "gzip", "bz2", "xz"]:
       raise NotImplementedError(f"`datasets` currently does not support {compression} compression"      )

     if isinstance(self.path_or_buf      ,							(str, bytes, os.PathLike)      ):
       with fsspec.open(self.path_or_buf      ,							'wb'      ,							compression=UpperCamelCase      ) as buffer:
         __a			=					self._write(file_obj=UpperCamelCase      ,							orient=UpperCamelCase      ,							lines=UpperCamelCase      ,							index=UpperCamelCase      ,							**self.to_json_kwargs      )
     else:
       if compression:
         raise NotImplementedError(
             f"The compression parameter is not supported when writing to a buffer, but compression={compression}"
             ' was passed. Please provide a local path instead.'      )
       __a			=					self._write(
           file_obj=self.path_or_buf      ,							orient=UpperCamelCase      ,							lines=UpperCamelCase      ,							index=UpperCamelCase      ,							**self.to_json_kwargs      )
     return written







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    List[str]:
     __a			=					args

     __a			=					query_table(
         table=self.dataset.data      ,							key=slice(UpperCamelCase      ,							offset + self.batch_size      )      ,							indices=self.dataset._indices      ,							)
     __a			=					batch.to_pandas().to_json(
         path_or_buf=UpperCamelCase      ,							orient=UpperCamelCase      ,							lines=UpperCamelCase      ,							index=UpperCamelCase      ,							**UpperCamelCase      )
     if not json_str.endswith('\n'      ):
       json_str += "\n"
     return json_str.encode(self.encoding      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							**UpperCamelCase      ,							)				->    Any:
     __a			=					0

     if self.num_proc is None or self.num_proc == 1:
       for offset in logging.tqdm(
           range(0      ,							len(self.dataset      )      ,							self.batch_size      )      ,							unit='ba'      ,							disable=not logging.is_progress_bar_enabled()      ,							desc='Creating json from Arrow format'      ,							):
         __a			=					self._batch_json((offset, orient, lines, index, to_json_kwargs)      )
         written += file_obj.write(UpperCamelCase      )
     else:
       __a			=					len(self.dataset      ), self.batch_size
       with multiprocessing.Pool(self.num_proc      ) as pool:
         for json_str in logging.tqdm(
             pool.imap(
                 self._batch_json      ,							[(offset, orient, lines, index, to_json_kwargs) for offset in range(0      ,							UpperCamelCase      ,							UpperCamelCase      )]      ,							)      ,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size      ,							unit='ba'      ,							disable=not logging.is_progress_bar_enabled()      ,							desc='Creating json from Arrow format'      ,							):
           written += file_obj.write(UpperCamelCase      )

     return written









'''simple docstring'''


from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
  import PIL


UpperCAmelCase_ 		=      logging.get_logger(__name__)





class    __lowercase						(  __magic_name__ ):
   _a							=      ["""pixel_values"""]







   def __init__(	self      ,							UpperCamelCase = True      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = PILImageResampling.BILINEAR      ,							UpperCamelCase = True      ,							UpperCamelCase = 1 / 255      ,							UpperCamelCase = True      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    None:
     super().__init__(**UpperCamelCase      )
     __a			=					size if size is not None else {'shortest_edge': 384}
     __a			=					get_size_dict(UpperCamelCase      ,							default_to_square=UpperCamelCase      )

     __a			=					do_resize
     __a			=					size
     # Default value set here for backwards compatibility where the value in config is None
     __a			=					crop_pct if crop_pct is not None else 224 / 256
     __a			=					resample
     __a			=					do_rescale
     __a			=					rescale_factor
     __a			=					do_normalize
     __a			=					image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
     __a			=					image_std if image_std is not None else IMAGENET_STANDARD_STD







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase = PILImageResampling.BICUBIC      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    np.ndarray:
     __a			=					get_size_dict(UpperCamelCase      ,							default_to_square=UpperCamelCase      )
     if "shortest_edge" not in size:
       raise ValueError(f"Size dictionary must contain 'shortest_edge' key. Got {size.keys()}"      )
     __a			=					size['shortest_edge']

     if shortest_edge < 384:
       # maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
       __a			=					int(shortest_edge / crop_pct      )
       __a			=					get_resize_output_image_size(UpperCamelCase      ,							size=UpperCamelCase      ,							default_to_square=UpperCamelCase      )
       __a			=					resize(image=UpperCamelCase      ,							size=UpperCamelCase      ,							resample=UpperCamelCase      ,							data_format=UpperCamelCase      ,							**UpperCamelCase      )
       # then crop to (shortest_edge, shortest_edge)
       return center_crop(image=UpperCamelCase      ,							size=(shortest_edge, shortest_edge)      ,							data_format=UpperCamelCase      ,							**UpperCamelCase      )
     else:
       # warping (no cropping) when evaluated at 384 or larger
       return resize(
           UpperCamelCase      ,							size=(shortest_edge, shortest_edge)      ,							resample=UpperCamelCase      ,							data_format=UpperCamelCase      ,							**UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    Any:
     return rescale(UpperCamelCase      ,							scale=UpperCamelCase      ,							data_format=UpperCamelCase      ,							**UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							**UpperCamelCase      ,							)				->    np.ndarray:
     return normalize(UpperCamelCase      ,							mean=UpperCamelCase      ,							std=UpperCamelCase      ,							data_format=UpperCamelCase      ,							**UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = None      ,							UpperCamelCase = ChannelDimension.FIRST      ,							**UpperCamelCase      ,							)				->    PIL.Image.Image:
     __a			=					do_resize if do_resize is not None else self.do_resize
     __a			=					crop_pct if crop_pct is not None else self.crop_pct
     __a			=					resample if resample is not None else self.resample
     __a			=					do_rescale if do_rescale is not None else self.do_rescale
     __a			=					rescale_factor if rescale_factor is not None else self.rescale_factor
     __a			=					do_normalize if do_normalize is not None else self.do_normalize
     __a			=					image_mean if image_mean is not None else self.image_mean
     __a			=					image_std if image_std is not None else self.image_std

     __a			=					size if size is not None else self.size
     __a			=					get_size_dict(UpperCamelCase      ,							default_to_square=UpperCamelCase      )

     __a			=					make_list_of_images(UpperCamelCase      )

     if not valid_images(UpperCamelCase      ):
       raise ValueError(
           'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
           'torch.Tensor, tf.Tensor or jax.ndarray.'      )

     if do_resize and size is None or resample is None:
       raise ValueError('Size and resample must be specified if do_resize is True.'      )

     if do_resize and size["shortest_edge"] < 384 and crop_pct is None:
       raise ValueError('crop_pct must be specified if size < 384.'      )

     if do_rescale and rescale_factor is None:
       raise ValueError('Rescale factor must be specified if do_rescale is True.'      )

     if do_normalize and (image_mean is None or image_std is None):
       raise ValueError('Image mean and std must be specified if do_normalize is True.'      )

     # All transformations expect numpy arrays.
     __a			=					[to_numpy_array(UpperCamelCase      ) for image in images]

     if do_resize:
       __a			=					[self.resize(image=UpperCamelCase      ,							size=UpperCamelCase      ,							crop_pct=UpperCamelCase      ,							resample=UpperCamelCase      ) for image in images]

     if do_rescale:
       __a			=					[self.rescale(image=UpperCamelCase      ,							scale=UpperCamelCase      ) for image in images]

     if do_normalize:
       __a			=					[self.normalize(image=UpperCamelCase      ,							mean=UpperCamelCase      ,							std=UpperCamelCase      ) for image in images]

     __a			=					[to_channel_dimension_format(UpperCamelCase      ,							UpperCamelCase      ) for image in images]

     __a			=					{'pixel_values': images}
     return BatchFeature(data=UpperCamelCase      ,							tensor_type=UpperCamelCase      )



