






"""simple docstring"""





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


A__					: Union[str, Any]=	datasets.utils.logging.get_logger(__name__)
@dataclass
class    __lowerCamelCase    (  datasets.BuilderConfig  ):
   a      :				Optional[datasets.Features] 						=None
   a      :				str 						="utf-8"
   a      :				Optional[str] 						=None
   a      :				Optional[str] 						=None
   a      :				bool 						=True  # deprecated
   a      :				Optional[int] 						=None  # deprecated
   a      :				int 						=1_0 << 2_0  # 10MB
   a      :				Optional[bool] 						=None







class    __lowerCamelCase    (  datasets.ArrowBasedBuilder  ):
   a      :				Any 						=JsonConfig

   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       int:
       if self.config.block_size is not None:
           logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'      )
           UpperCamelCase__         =    self.config.block_size
       if self.config.use_threads is not True:
           logger.warning(
               'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'      )
       if self.config.newlines_in_values is not None:
           raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'      )
       return datasets.DatasetInfo(features=self.config.features      )

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       Dict:
       if not self.config.data_files:
           raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'      )
       UpperCamelCase__         =    dl_manager.download_and_extract(self.config.data_files      )
       if isinstance(snake_case_ ,				(str, list, tuple)      ):
           UpperCamelCase__         =    data_files
           if isinstance(snake_case_ ,				snake_case_      ):
               UpperCamelCase__         =    [files]
           UpperCamelCase__         =    [dl_manager.iter_files(snake_case_      ) for file in files]
           return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,				gen_kwargs={'files': files}      )]
       UpperCamelCase__         =    []
       for split_name, files in data_files.items():
           if isinstance(snake_case_ ,				snake_case_      ):
               UpperCamelCase__         =    [files]
           UpperCamelCase__         =    [dl_manager.iter_files(snake_case_      ) for file in files]
           splits.append(datasets.SplitGenerator(name=snake_case_ ,				gen_kwargs={'files': files}      )      )
       return splits

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       pa.Table:
       if self.config.features is not None:
           # adding missing columns
           for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
               UpperCamelCase__         =    self.config.features.arrow_schema.field(snake_case_      ).type
               UpperCamelCase__         =    pa_table.append_column(snake_case_ ,				pa.array([None] * len(snake_case_      ) ,				type=snake_case_      )      )
           # more expensive cast to support nested structures with keys in a different order
           # allows str <-> int/float or str to Audio for example
           UpperCamelCase__         =    table_cast(snake_case_ ,				self.config.features.arrow_schema      )
       return pa_table







   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       str:
       for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case_      )      ):
           # If the file is one json object and if we need to look at the list of items in one specific field
           if self.config.field is not None:
               with open(snake_case_ ,				encoding=self.config.encoding ,				errors=self.config.encoding_errors      ) as f:
                   UpperCamelCase__         =    json.load(snake_case_      )

               # We keep only the field we are interested in
               UpperCamelCase__         =    dataset[self.config.field]

               # We accept two format: a list of dicts or a dict of lists
               if isinstance(snake_case_ ,				(list, tuple)      ):
                   UpperCamelCase__         =    set().union(*[row.keys() for row in dataset]      )
                   UpperCamelCase__         =    {col: [row.get(snake_case_      ) for row in dataset] for col in keys}
               else:
                   UpperCamelCase__         =    dataset
               UpperCamelCase__         =    pa.Table.from_pydict(snake_case_      )
               yield file_idx, self._cast_table(snake_case_      )

           # If the file has one json object per line
           else:
               with open(snake_case_ ,				'rb'      ) as f:
                   UpperCamelCase__         =    0
                   # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                   # Set a default minimum value of 16kB if the chunk size is really small
                   UpperCamelCase__         =    max(self.config.chunksize // 32 ,				16 << 10      )
                   UpperCamelCase__         =    (
                       self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
                   )
                   while True:
                       UpperCamelCase__         =    f.read(self.config.chunksize      )
                       if not batch:
                           break
                       # Finish current line
                       try:
                           batch += f.readline()
                       except (AttributeError, io.UnsupportedOperation):
                           batch += readline(snake_case_      )
                       # PyArrow only accepts utf-8 encoded bytes
                       if self.config.encoding != "utf-8":
                           UpperCamelCase__         =    batch.decode(self.config.encoding ,				errors=snake_case_      ).encode('utf-8'      )
                       try:
                           while True:
                               try:
                                   UpperCamelCase__         =    paj.read_json(
                                       io.BytesIO(snake_case_      ) ,				read_options=paj.ReadOptions(block_size=snake_case_      )      )
                                   break
                               except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                   if (
                                       isinstance(snake_case_ ,				pa.ArrowInvalid      )
                                       and "straddling" not in str(snake_case_      )
                                       or block_size > len(snake_case_      )
                                   ):
                                       raise
                                   else:
                                       # Increase the block size in case it was too small.
                                       # The block size will be reset for the next file.
                                       logger.debug(
                                           F'Batch of {len(snake_case_      )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'      )
                                       block_size *= 2
                       except pa.ArrowInvalid as e:
                           try:
                               with open(
                                   snake_case_ ,				encoding=self.config.encoding ,				errors=self.config.encoding_errors      ) as f:
                                   UpperCamelCase__         =    json.load(snake_case_      )
                           except json.JSONDecodeError:
                               logger.error(F'Failed to read file \'{file}\' with error {type(snake_case_      )}: {e}'      )
                               raise e
                           # If possible, parse the file as a list of json objects and exit the loop
                           if isinstance(snake_case_ ,				snake_case_      ):  # list is the only sequence type supported in JSON
                               try:
                                   UpperCamelCase__         =    set().union(*[row.keys() for row in dataset]      )
                                   UpperCamelCase__         =    {col: [row.get(snake_case_      ) for row in dataset] for col in keys}
                                   UpperCamelCase__         =    pa.Table.from_pydict(snake_case_      )
                               except (pa.ArrowInvalid, AttributeError) as e:
                                   logger.error(F'Failed to read file \'{file}\' with error {type(snake_case_      )}: {e}'      )
                                   raise ValueError(F'Not able to read records in the JSON file at {file}.'      ) from None
                               yield file_idx, self._cast_table(snake_case_      )
                               break
                           else:
                               logger.error(F'Failed to read file \'{file}\' with error {type(snake_case_      )}: {e}'      )
                               raise ValueError(
                                   F'Not able to read records in the JSON file at {file}. '
                                   F'You should probably indicate the field of the JSON file containing your records. '
                                   F'This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. '
                                   F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                       yield (file_idx, batch_idx), self._cast_table(snake_case_      )
                       batch_idx += 1












"""simple docstring"""





from copy import deepcopy
class    __lowerCamelCase    :

   def __init__(					self ,				snake_case_ = None ,				snake_case_ = None      )		->       None:

       if arr is None and size is not None:
           UpperCamelCase__         =    size
           UpperCamelCase__         =    [0] * size
       elif arr is not None:
           self.init(snake_case_      )
       else:
           raise ValueError('Either arr or size must be specified'      )

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       None:
       UpperCamelCase__         =    len(snake_case_      )
       UpperCamelCase__         =    deepcopy(snake_case_      )
       for i in range(1 ,				self.size      ):
           UpperCamelCase__         =    self.next_(snake_case_      )
           if j < self.size:
               self.tree[j] += self.tree[i]

   def 						SCREAMING_SNAKE_CASE__ (					self      )		->       list[int]:
       UpperCamelCase__         =    self.tree[:]
       for i in range(self.size - 1 ,				0 ,				-1      ):
           UpperCamelCase__         =    self.next_(snake_case_      )
           if j < self.size:
               arr[j] -= arr[i]
       return arr

   @staticmethod
   def 						SCREAMING_SNAKE_CASE__ (					snake_case_      )		->       int:
       return index + (index & (-index))

   @staticmethod
   def 						SCREAMING_SNAKE_CASE__ (					snake_case_      )		->       int:
       return index - (index & (-index))

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_      )		->       None:
       if index == 0:
           self.tree[0] += value
           return
       while index < self.size:
           self.tree[index] += value
           UpperCamelCase__         =    self.next_(snake_case_      )

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_      )		->       None:
       self.add(snake_case_ ,				value - self.get(snake_case_      )      )

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       int:
       if right == 0:
           return 0
       UpperCamelCase__         =    self.tree[0]
       right -= 1  # make right inclusive
       while right > 0:
           result += self.tree[right]
           UpperCamelCase__         =    self.prev(snake_case_      )
       return result

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_      )		->       int:
       return self.prefix(snake_case_      ) - self.prefix(snake_case_      )

   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       int:
       return self.query(snake_case_ ,				index + 1      )







   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_      )		->       int:
       value -= self.tree[0]
       if value < 0:
           return -1

       UpperCamelCase__         =    1  # Largest power of 2 <= size
       while j * 2 < self.size:
           j *= 2

       UpperCamelCase__         =    0

       while j > 0:
           if i + j < self.size and self.tree[i + j] <= value:
               value -= self.tree[i + j]
               i += j
           j //= 2
       return i


if __name__ == "__main__":
  import doctest

  doctest.testmod()




