

import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCAmelCase__ :     Optional[int]				    =    datasets.utils.logging.get_logger(__name__)







@dataclass
class        a__       (   datasets.BuilderConfig       ):







  """simple docstring"""




  UpperCAmelCase__      : Optional[datasets.Features]			       =None
  UpperCAmelCase__      : str			       ="utf-8"
  UpperCAmelCase__      : Optional[str]			       =None
  UpperCAmelCase__      : Optional[str]			       =None
  UpperCAmelCase__      : bool			       =True  # deprecated
  UpperCAmelCase__      : Optional[int]			       =None  # deprecated
  UpperCAmelCase__      : int			       =1_0 << 2_0  # 10MB
  UpperCAmelCase__      : Optional[bool]			       =None







class        a__       (   datasets.ArrowBasedBuilder       ):







  """simple docstring"""




  UpperCAmelCase__      : Tuple			       =JsonConfig


  def  _lowercase      (			self    : str		)					->Optional[int]:
        """simple docstring"""







        if self.config.block_size is not None:
              logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""		)
              SCREAMING_SNAKE_CASE :      Dict       =   self.config.block_size
        if self.config.use_threads is not True:
              logger.warning(
                  """The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore."""		)
        if self.config.newlines_in_values is not None:
              raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""		)
        return datasets.DatasetInfo(features=self.config.features		)


  def  _lowercase      (			self    : Dict				,				UpperCAmelCase__    : Optional[Any]		)					->Tuple:
        """simple docstring"""







        if not self.config.data_files:
              raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"		)
        SCREAMING_SNAKE_CASE :      List[str]       =   dl_manager.download_and_extract(self.config.data_files		)
        if isinstance(UpperCAmelCase__				,				(str, list, tuple)		):
              SCREAMING_SNAKE_CASE :      Tuple       =   data_files
              if isinstance(UpperCAmelCase__				,				UpperCAmelCase__		):
                    SCREAMING_SNAKE_CASE :      Any       =   [files]
              SCREAMING_SNAKE_CASE :      Any       =   [dl_manager.iter_files(UpperCAmelCase__		) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,				gen_kwargs={"""files""": files}		)]
        SCREAMING_SNAKE_CASE :      int       =   []
        for split_name, files in data_files.items():
              if isinstance(UpperCAmelCase__				,				UpperCAmelCase__		):
                    SCREAMING_SNAKE_CASE :      List[str]       =   [files]
              SCREAMING_SNAKE_CASE :      List[Any]       =   [dl_manager.iter_files(UpperCAmelCase__		) for file in files]
              splits.append(datasets.SplitGenerator(name=UpperCAmelCase__				,				gen_kwargs={"""files""": files}		)		)
        return splits


  def  _lowercase      (			self    : Any				,				UpperCAmelCase__    : Tuple		)					->Any:
        """simple docstring"""







        if self.config.features is not None:
              # adding missing columns
              for column_name in set(self.config.features		) - set(pa_table.column_names		):
                    SCREAMING_SNAKE_CASE :      Optional[int]       =   self.config.features.arrow_schema.field(UpperCAmelCase__		).type
                    SCREAMING_SNAKE_CASE :      Any       =   pa_table.append_column(UpperCAmelCase__				,				pa.array([None] * len(UpperCAmelCase__		)				,				type=UpperCAmelCase__		)		)
              # more expensive cast to support nested structures with keys in a different order
              # allows str <-> int/float or str to Audio for example
              SCREAMING_SNAKE_CASE :      int       =   table_cast(UpperCAmelCase__				,				self.config.features.arrow_schema		)
        return pa_table




  def  _lowercase      (			self    : Optional[Any]				,				UpperCAmelCase__    : List[str]		)					->Optional[Any]:
        """simple docstring"""







        for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase__		)		):
              # If the file is one json object and if we need to look at the list of items in one specific field
              if self.config.field is not None:
                    with open(UpperCAmelCase__				,				encoding=self.config.encoding				,				errors=self.config.encoding_errors		) as f:
                          SCREAMING_SNAKE_CASE :      Union[str, Any]       =   json.load(UpperCAmelCase__		)

                    # We keep only the field we are interested in
                    SCREAMING_SNAKE_CASE :      int       =   dataset[self.config.field]

                    # We accept two format: a list of dicts or a dict of lists
                    if isinstance(UpperCAmelCase__				,				(list, tuple)		):
                          SCREAMING_SNAKE_CASE :      List[str]       =   set().union(*[row.keys() for row in dataset]		)
                          SCREAMING_SNAKE_CASE :      Any       =   {col: [row.get(UpperCAmelCase__		) for row in dataset] for col in keys}
                    else:
                          SCREAMING_SNAKE_CASE :      Tuple       =   dataset
                    SCREAMING_SNAKE_CASE :      List[Any]       =   pa.Table.from_pydict(UpperCAmelCase__		)
                    yield file_idx, self._cast_table(UpperCAmelCase__		)

              # If the file has one json object per line
              else:
                    with open(UpperCAmelCase__				,				"""rb"""		) as f:
                          SCREAMING_SNAKE_CASE :      Optional[int]       =   0
                          # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                          # Set a default minimum value of 16kB if the chunk size is really small
                          SCREAMING_SNAKE_CASE :      int       =   max(self.config.chunksize // 3_2				,				1_6 << 1_0		)
                          SCREAMING_SNAKE_CASE :      List[str]       =   (
                              self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                          )
                          while True:
                                SCREAMING_SNAKE_CASE :      Any       =   f.read(self.config.chunksize		)
                                if not batch:
                                      break
                                # Finish current line
                                try:
                                      batch += f.readline()
                                except (AttributeError, io.UnsupportedOperation):
                                      batch += readline(UpperCAmelCase__		)
                                # PyArrow only accepts utf-8 encoded bytes
                                if self.config.encoding != "utf-8":
                                      SCREAMING_SNAKE_CASE :      Optional[Any]       =   batch.decode(self.config.encoding				,				errors=UpperCAmelCase__		).encode("""utf-8"""		)
                                try:
                                      while True:
                                            try:
                                                  SCREAMING_SNAKE_CASE :      List[Any]       =   paj.read_json(
                                                      io.BytesIO(UpperCAmelCase__		)				,				read_options=paj.ReadOptions(block_size=UpperCAmelCase__		)		)
                                                  break
                                            except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                  if (
                                                      isinstance(UpperCAmelCase__				,				pa.ArrowInvalid		)
                                                      and "straddling" not in str(UpperCAmelCase__		)
                                                      or block_size > len(UpperCAmelCase__		)
                                                  ):
                                                        raise
                                                  else:
                                                        # Increase the block size in case it was too small.
                                                        # The block size will be reset for the next file.
                                                        logger.debug(
                                                            f"Batch of {len(UpperCAmelCase__		)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."		)
                                                        block_size *= 2
                                except pa.ArrowInvalid as e:
                                      try:
                                            with open(
                                                UpperCAmelCase__				,				encoding=self.config.encoding				,				errors=self.config.encoding_errors		) as f:
                                                  SCREAMING_SNAKE_CASE :      Any       =   json.load(UpperCAmelCase__		)
                                      except json.JSONDecodeError:
                                            logger.error(f"Failed to read file \'{file}\' with error {type(UpperCAmelCase__		)}: {e}"		)
                                            raise e
                                      # If possible, parse the file as a list of json objects and exit the loop
                                      if isinstance(UpperCAmelCase__				,				UpperCAmelCase__		):  # list is the only sequence type supported in JSON
                                            try:
                                                  SCREAMING_SNAKE_CASE :      str       =   set().union(*[row.keys() for row in dataset]		)
                                                  SCREAMING_SNAKE_CASE :      List[str]       =   {col: [row.get(UpperCAmelCase__		) for row in dataset] for col in keys}
                                                  SCREAMING_SNAKE_CASE :      Any       =   pa.Table.from_pydict(UpperCAmelCase__		)
                                            except (pa.ArrowInvalid, AttributeError) as e:
                                                  logger.error(f"Failed to read file \'{file}\' with error {type(UpperCAmelCase__		)}: {e}"		)
                                                  raise ValueError(f"Not able to read records in the JSON file at {file}."		) from None
                                            yield file_idx, self._cast_table(UpperCAmelCase__		)
                                            break
                                      else:
                                            logger.error(f"Failed to read file \'{file}\' with error {type(UpperCAmelCase__		)}: {e}"		)
                                            raise ValueError(
                                                f"Not able to read records in the JSON file at {file}. "
                                                f"You should probably indicate the field of the JSON file containing your records. "
                                                f"This JSON file contain the following fields: {str(list(dataset.keys()		)		)}. "
                                                f"Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. "		) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase__		)
                                batch_idx += 1







import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD







class        a__       (   nn.Module       ):







  """simple docstring"""




  UpperCAmelCase__      : int
  UpperCAmelCase__      : int
  UpperCAmelCase__      : float			       =0.0
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : bool			       =True
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : jnp.dtype			       =jnp.floataa


  def  _lowercase      (			self    : Tuple		)					->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   []
        SCREAMING_SNAKE_CASE :      Any       =   []

        for i in range(self.num_layers		):
              SCREAMING_SNAKE_CASE :      List[Any]       =   self.in_channels if i == 0 else self.out_channels

              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   FlaxResnetBlockaD(
                  in_channels=UpperCAmelCase__				,				out_channels=self.out_channels				,				dropout_prob=self.dropout				,				dtype=self.dtype				,				)
              resnets.append(UpperCAmelCase__		)

              SCREAMING_SNAKE_CASE :      List[Any]       =   FlaxTransformeraDModel(
                  in_channels=self.out_channels				,				n_heads=self.num_attention_heads				,				d_head=self.out_channels // self.num_attention_heads				,				depth=1				,				use_linear_projection=self.use_linear_projection				,				only_cross_attention=self.only_cross_attention				,				use_memory_efficient_attention=self.use_memory_efficient_attention				,				dtype=self.dtype				,				)
              attentions.append(UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      Optional[Any]       =   resnets
        SCREAMING_SNAKE_CASE :      Optional[int]       =   attentions

        if self.add_downsample:
              SCREAMING_SNAKE_CASE :      Tuple       =   FlaxDownsampleaD(self.out_channels				,				dtype=self.dtype		)




  def __call__(			self    : int				,				UpperCAmelCase__    : List[str]				,				UpperCAmelCase__    : Union[str, Any]				,				UpperCAmelCase__    : Optional[Any]				,				UpperCAmelCase__    : List[str]=True		)					->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[int]       =   ()

        for resnet, attn in zip(self.resnets				,				self.attentions		):
              SCREAMING_SNAKE_CASE :      List[str]       =   resnet(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)
              SCREAMING_SNAKE_CASE :      List[str]       =   attn(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)
              output_states += (hidden_states,)

        if self.add_downsample:
              SCREAMING_SNAKE_CASE :      int       =   self.downsamplers_a(UpperCAmelCase__		)
              output_states += (hidden_states,)

        return hidden_states, output_states







class        a__       (   nn.Module       ):







  """simple docstring"""




  UpperCAmelCase__      : int
  UpperCAmelCase__      : int
  UpperCAmelCase__      : float			       =0.0
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : bool			       =True
  UpperCAmelCase__      : jnp.dtype			       =jnp.floataa


  def  _lowercase      (			self    : Dict		)					->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   []

        for i in range(self.num_layers		):
              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   self.in_channels if i == 0 else self.out_channels

              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   FlaxResnetBlockaD(
                  in_channels=UpperCAmelCase__				,				out_channels=self.out_channels				,				dropout_prob=self.dropout				,				dtype=self.dtype				,				)
              resnets.append(UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   resnets

        if self.add_downsample:
              SCREAMING_SNAKE_CASE :      List[str]       =   FlaxDownsampleaD(self.out_channels				,				dtype=self.dtype		)




  def __call__(			self    : Tuple				,				UpperCAmelCase__    : int				,				UpperCAmelCase__    : Optional[Any]				,				UpperCAmelCase__    : Any=True		)					->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[int]       =   ()

        for resnet in self.resnets:
              SCREAMING_SNAKE_CASE :      List[str]       =   resnet(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)
              output_states += (hidden_states,)

        if self.add_downsample:
              SCREAMING_SNAKE_CASE :      Dict       =   self.downsamplers_a(UpperCAmelCase__		)
              output_states += (hidden_states,)

        return hidden_states, output_states







class        a__       (   nn.Module       ):







  """simple docstring"""




  UpperCAmelCase__      : int
  UpperCAmelCase__      : int
  UpperCAmelCase__      : int
  UpperCAmelCase__      : float			       =0.0
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : bool			       =True
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : jnp.dtype			       =jnp.floataa


  def  _lowercase      (			self    : int		)					->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   []
        SCREAMING_SNAKE_CASE :      List[Any]       =   []

        for i in range(self.num_layers		):
              SCREAMING_SNAKE_CASE :      Any       =   self.in_channels if (i == self.num_layers - 1) else self.out_channels
              SCREAMING_SNAKE_CASE :      Any       =   self.prev_output_channel if i == 0 else self.out_channels

              SCREAMING_SNAKE_CASE :      Tuple       =   FlaxResnetBlockaD(
                  in_channels=resnet_in_channels + res_skip_channels				,				out_channels=self.out_channels				,				dropout_prob=self.dropout				,				dtype=self.dtype				,				)
              resnets.append(UpperCAmelCase__		)

              SCREAMING_SNAKE_CASE :      Tuple       =   FlaxTransformeraDModel(
                  in_channels=self.out_channels				,				n_heads=self.num_attention_heads				,				d_head=self.out_channels // self.num_attention_heads				,				depth=1				,				use_linear_projection=self.use_linear_projection				,				only_cross_attention=self.only_cross_attention				,				use_memory_efficient_attention=self.use_memory_efficient_attention				,				dtype=self.dtype				,				)
              attentions.append(UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      Optional[Any]       =   resnets
        SCREAMING_SNAKE_CASE :      List[Any]       =   attentions

        if self.add_upsample:
              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   FlaxUpsampleaD(self.out_channels				,				dtype=self.dtype		)




  def __call__(			self    : Union[str, Any]				,				UpperCAmelCase__    : str				,				UpperCAmelCase__    : List[Any]				,				UpperCAmelCase__    : Any				,				UpperCAmelCase__    : Union[str, Any]				,				UpperCAmelCase__    : Dict=True		)					->Tuple:
        """simple docstring"""







        for resnet, attn in zip(self.resnets				,				self.attentions		):
              # pop res hidden states
              SCREAMING_SNAKE_CASE :      str       =   res_hidden_states_tuple[-1]
              SCREAMING_SNAKE_CASE :      Dict       =   res_hidden_states_tuple[:-1]
              SCREAMING_SNAKE_CASE :      Union[str, Any]       =   jnp.concatenate((hidden_states, res_hidden_states)				,				axis=-1		)

              SCREAMING_SNAKE_CASE :      Tuple       =   resnet(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)
              SCREAMING_SNAKE_CASE :      Tuple       =   attn(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)

        if self.add_upsample:
              SCREAMING_SNAKE_CASE :      Tuple       =   self.upsamplers_a(UpperCAmelCase__		)

        return hidden_states







class        a__       (   nn.Module       ):







  """simple docstring"""




  UpperCAmelCase__      : int
  UpperCAmelCase__      : int
  UpperCAmelCase__      : int
  UpperCAmelCase__      : float			       =0.0
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : bool			       =True
  UpperCAmelCase__      : jnp.dtype			       =jnp.floataa


  def  _lowercase      (			self    : Union[str, Any]		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[Any]       =   []

        for i in range(self.num_layers		):
              SCREAMING_SNAKE_CASE :      Optional[Any]       =   self.in_channels if (i == self.num_layers - 1) else self.out_channels
              SCREAMING_SNAKE_CASE :      str       =   self.prev_output_channel if i == 0 else self.out_channels

              SCREAMING_SNAKE_CASE :      Optional[int]       =   FlaxResnetBlockaD(
                  in_channels=resnet_in_channels + res_skip_channels				,				out_channels=self.out_channels				,				dropout_prob=self.dropout				,				dtype=self.dtype				,				)
              resnets.append(UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      Dict       =   resnets

        if self.add_upsample:
              SCREAMING_SNAKE_CASE :      int       =   FlaxUpsampleaD(self.out_channels				,				dtype=self.dtype		)




  def __call__(			self    : Optional[Any]				,				UpperCAmelCase__    : List[str]				,				UpperCAmelCase__    : Dict				,				UpperCAmelCase__    : Tuple				,				UpperCAmelCase__    : Optional[int]=True		)					->Union[str, Any]:
        """simple docstring"""







        for resnet in self.resnets:
              # pop res hidden states
              SCREAMING_SNAKE_CASE :      Dict       =   res_hidden_states_tuple[-1]
              SCREAMING_SNAKE_CASE :      Any       =   res_hidden_states_tuple[:-1]
              SCREAMING_SNAKE_CASE :      Tuple       =   jnp.concatenate((hidden_states, res_hidden_states)				,				axis=-1		)

              SCREAMING_SNAKE_CASE :      Optional[int]       =   resnet(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)

        if self.add_upsample:
              SCREAMING_SNAKE_CASE :      Optional[int]       =   self.upsamplers_a(UpperCAmelCase__		)

        return hidden_states







class        a__       (   nn.Module       ):







  """simple docstring"""




  UpperCAmelCase__      : int
  UpperCAmelCase__      : float			       =0.0
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : int			       =1
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : bool			       =False
  UpperCAmelCase__      : jnp.dtype			       =jnp.floataa


  def  _lowercase      (			self    : Optional[Any]		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      str       =   [
            FlaxResnetBlockaD(
                in_channels=self.in_channels				,				out_channels=self.in_channels				,				dropout_prob=self.dropout				,				dtype=self.dtype				,				)
        ]

        SCREAMING_SNAKE_CASE :      List[str]       =   []

        for _ in range(self.num_layers		):
              SCREAMING_SNAKE_CASE :      Tuple       =   FlaxTransformeraDModel(
                  in_channels=self.in_channels				,				n_heads=self.num_attention_heads				,				d_head=self.in_channels // self.num_attention_heads				,				depth=1				,				use_linear_projection=self.use_linear_projection				,				use_memory_efficient_attention=self.use_memory_efficient_attention				,				dtype=self.dtype				,				)
              attentions.append(UpperCAmelCase__		)

              SCREAMING_SNAKE_CASE :      List[str]       =   FlaxResnetBlockaD(
                  in_channels=self.in_channels				,				out_channels=self.in_channels				,				dropout_prob=self.dropout				,				dtype=self.dtype				,				)
              resnets.append(UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      int       =   resnets
        SCREAMING_SNAKE_CASE :      Tuple       =   attentions




  def __call__(			self    : Union[str, Any]				,				UpperCAmelCase__    : List[str]				,				UpperCAmelCase__    : Union[str, Any]				,				UpperCAmelCase__    : Any				,				UpperCAmelCase__    : str=True		)					->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Tuple       =   self.resnets[0](UpperCAmelCase__				,				UpperCAmelCase__		)
        for attn, resnet in zip(self.attentions				,				self.resnets[1:]		):
              SCREAMING_SNAKE_CASE :      Optional[int]       =   attn(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)
              SCREAMING_SNAKE_CASE :      List[str]       =   resnet(UpperCAmelCase__				,				UpperCAmelCase__				,				deterministic=UpperCAmelCase__		)

        return hidden_states




