





'''simple docstring'''


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_SCREAMING_SNAKE_CASE	 =							datasets.utils.logging.get_logger(__name__)

@dataclass
class    lowerCAmelCase_		(		datasets.BuilderConfig		):
 __lowerCamelCase						:							Optional[datasets.Features]         =					None
 __lowerCamelCase						:							str         =					"utf-8"
 __lowerCamelCase						:							Optional[str]         =					None
 __lowerCamelCase						:							Optional[str]         =					None
 __lowerCamelCase						:							bool         =					True  # deprecated
 __lowerCamelCase						:							Optional[int]         =					None  # deprecated
 __lowerCamelCase						:							int         =					10 << 20  # 10MB
 __lowerCamelCase						:							Optional[bool]         =					None







class    lowerCAmelCase_		(		datasets.ArrowBasedBuilder		):
 __lowerCamelCase						:							Optional[int]         =					JsonConfig




 def 							_snake_case   (	self		)    ->							Union[str, Any]:
     if self.config.block_size is not None:
         logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"		)
         _lowerCAmelCase				=      self.config.block_size
     if self.config.use_threads is not True:
         logger.warning(
             "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."		)
     if self.config.newlines_in_values is not None:
         raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"		)
     return datasets.DatasetInfo(features=self.config.features		)




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							List[str]:
     if not self.config.data_files:
         raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''		)
     _lowerCAmelCase				=      dl_manager.download_and_extract(self.config.data_files		)
     if isinstance(_lowerCAmelCase			,     (str, list, tuple)		):
         _lowerCAmelCase				=      data_files
         if isinstance(_lowerCAmelCase			,     _lowerCAmelCase		):
             _lowerCAmelCase				=      [files]
         _lowerCAmelCase				=      [dl_manager.iter_files(_lowerCAmelCase		) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,     gen_kwargs={"files": files}		)]
     _lowerCAmelCase				=      []
     for split_name, files in data_files.items():
         if isinstance(_lowerCAmelCase			,     _lowerCAmelCase		):
             _lowerCAmelCase				=      [files]
         _lowerCAmelCase				=      [dl_manager.iter_files(_lowerCAmelCase		) for file in files]
         splits.append(datasets.SplitGenerator(name=_lowerCAmelCase			,     gen_kwargs={"files": files}		)		)
     return splits




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							pa.Table:
     if self.config.features is not None:
         # adding missing columns
         for column_name in set(self.config.features		) - set(pa_table.column_names		):
             _lowerCAmelCase				=      self.config.features.arrow_schema.field(_lowerCAmelCase		).type
             _lowerCAmelCase				=      pa_table.append_column(_lowerCAmelCase			,     pa.array([None] * len(_lowerCAmelCase		)			,     type=_lowerCAmelCase		)		)
         # more expensive cast to support nested structures with keys in a different order
         # allows str <-> int/float or str to Audio for example
         _lowerCAmelCase				=      table_cast(_lowerCAmelCase			,     self.config.features.arrow_schema		)
     return pa_table





 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							List[Any]:
     for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase		)		):
         # If the file is one json object and if we need to look at the list of items in one specific field
         if self.config.field is not None:
             with open(_lowerCAmelCase			,     encoding=self.config.encoding			,     errors=self.config.encoding_errors		) as f:
                 _lowerCAmelCase				=      json.load(_lowerCAmelCase		)

             # We keep only the field we are interested in
             _lowerCAmelCase				=      dataset[self.config.field]

             # We accept two format: a list of dicts or a dict of lists
             if isinstance(_lowerCAmelCase			,     (list, tuple)		):
                 _lowerCAmelCase				=      set().union(*[row.keys() for row in dataset]		)
                 _lowerCAmelCase				=      {col: [row.get(_lowerCAmelCase		) for row in dataset] for col in keys}
             else:
                 _lowerCAmelCase				=      dataset
             _lowerCAmelCase				=      pa.Table.from_pydict(_lowerCAmelCase		)
             yield file_idx, self._cast_table(_lowerCAmelCase		)

         # If the file has one json object per line
         else:
             with open(_lowerCAmelCase			,     "rb"		) as f:
                 _lowerCAmelCase				=      0
                 # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                 # Set a default minimum value of 16kB if the chunk size is really small
                 _lowerCAmelCase				=      max(self.config.chunksize // 32			,     16 << 10		)
                 _lowerCAmelCase				=      (
                     self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
                 )
                 while True:
                     _lowerCAmelCase				=      f.read(self.config.chunksize		)
                     if not batch:
                         break
                     # Finish current line
                     try:
                         batch += f.readline()
                     except (AttributeError, io.UnsupportedOperation):
                         batch += readline(_lowerCAmelCase		)
                     # PyArrow only accepts utf-8 encoded bytes
                     if self.config.encoding != "utf-8":
                         _lowerCAmelCase				=      batch.decode(self.config.encoding			,     errors=_lowerCAmelCase		).encode("utf-8"		)
                     try:
                         while True:
                             try:
                                 _lowerCAmelCase				=      paj.read_json(
                                     io.BytesIO(_lowerCAmelCase		)			,     read_options=paj.ReadOptions(block_size=_lowerCAmelCase		)		)
                                 break
                             except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                 if (
                                     isinstance(_lowerCAmelCase			,     pa.ArrowInvalid		)
                                     and "straddling" not in str(_lowerCAmelCase		)
                                     or block_size > len(_lowerCAmelCase		)
                                 ):
                                     raise
                                 else:
                                     # Increase the block size in case it was too small.
                                     # The block size will be reset for the next file.
                                     logger.debug(
                                         f'''Batch of {len(_lowerCAmelCase		)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''		)
                                     block_size *= 2
                     except pa.ArrowInvalid as e:
                         try:
                             with open(
                                 _lowerCAmelCase			,     encoding=self.config.encoding			,     errors=self.config.encoding_errors		) as f:
                                 _lowerCAmelCase				=      json.load(_lowerCAmelCase		)
                         except json.JSONDecodeError:
                             logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase		)}: {e}'''		)
                             raise e
                         # If possible, parse the file as a list of json objects and exit the loop
                         if isinstance(_lowerCAmelCase			,     _lowerCAmelCase		):  # list is the only sequence type supported in JSON
                             try:
                                 _lowerCAmelCase				=      set().union(*[row.keys() for row in dataset]		)
                                 _lowerCAmelCase				=      {col: [row.get(_lowerCAmelCase		) for row in dataset] for col in keys}
                                 _lowerCAmelCase				=      pa.Table.from_pydict(_lowerCAmelCase		)
                             except (pa.ArrowInvalid, AttributeError) as e:
                                 logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase		)}: {e}'''		)
                                 raise ValueError(f'''Not able to read records in the JSON file at {file}.'''		) from None
                             yield file_idx, self._cast_table(_lowerCAmelCase		)
                             break
                         else:
                             logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase		)}: {e}'''		)
                             raise ValueError(
                                 f'''Not able to read records in the JSON file at {file}. '''
                                 f'''You should probably indicate the field of the JSON file containing your records. '''
                                 f'''This JSON file contain the following fields: {str(list(dataset.keys()		)		)}. '''
                                 f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''		) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                     yield (file_idx, batch_idx), self._cast_table(_lowerCAmelCase		)
                     batch_idx += 1









'''simple docstring'''


from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
       from ..models.auto import AutoTokenizer

class    lowerCAmelCase_		:




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							Tuple:
     raise NotImplementedError()





 def 							_snake_case   (	self		)    ->							Optional[int]:
     raise NotImplementedError()







class    lowerCAmelCase_		(		__magic_name__		):




 def __init__(	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False			,     **_lowerCAmelCase		)    ->							str:
     _lowerCAmelCase				=      tokenizer
     _lowerCAmelCase				=      skip_prompt
     _lowerCAmelCase				=      decode_kwargs

     # variables used in the streaming process
     _lowerCAmelCase				=      []
     _lowerCAmelCase				=      0
     _lowerCAmelCase				=      True




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							List[Any]:
     if len(value.shape		) > 1 and value.shape[0] > 1:
         raise ValueError("TextStreamer only supports batch size 1"		)
     elif len(value.shape		) > 1:
         _lowerCAmelCase				=      value[0]

     if self.skip_prompt and self.next_tokens_are_prompt:
         _lowerCAmelCase				=      False
         return

     # Add the new token to the cache and decodes the entire thing.
     self.token_cache.extend(value.tolist()		)
     _lowerCAmelCase				=      self.tokenizer.decode(self.token_cache			,     **self.decode_kwargs		)

     # After the symbol for a new line, we flush the cache.
     if text.endswith("\n"		):
         _lowerCAmelCase				=      text[self.print_len :]
         _lowerCAmelCase				=      []
         _lowerCAmelCase				=      0
     # If the last token is a CJK character, we print the characters.
     elif len(_lowerCAmelCase		) > 0 and self._is_chinese_char(ord(text[-1]		)		):
         _lowerCAmelCase				=      text[self.print_len :]
         self.print_len += len(_lowerCAmelCase		)
     # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
     # which may change with the subsequent token -- there are probably smarter ways to do this!)
     else:
         _lowerCAmelCase				=      text[self.print_len : text.rfind(" "		) + 1]
         self.print_len += len(_lowerCAmelCase		)

     self.on_finalized_text(_lowerCAmelCase		)




 def 							_snake_case   (	self		)    ->							Union[str, Any]:
     # Flush the cache, if it exists
     if len(self.token_cache		) > 0:
         _lowerCAmelCase				=      self.tokenizer.decode(self.token_cache			,     **self.decode_kwargs		)
         _lowerCAmelCase				=      text[self.print_len :]
         _lowerCAmelCase				=      []
         _lowerCAmelCase				=      0
     else:
         _lowerCAmelCase				=      ""

     _lowerCAmelCase				=      True
     self.on_finalized_text(_lowerCAmelCase			,     stream_end=_lowerCAmelCase		)




 def 							_snake_case   (	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False		)    ->							List[Any]:
     print(_lowerCAmelCase			,     flush=_lowerCAmelCase			,     end="" if not stream_end else None		)





 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							Tuple:
     # This defines a "chinese character" as anything in the CJK Unicode block:
     #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
     #
     # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
     # despite its name. The modern Korean Hangul alphabet is a different block,
     # as is Japanese Hiragana and Katakana. Those alphabets are used to write
     # space-separated words, so they are not treated specially and handled
     # like the all of the other languages.
     if (
         (cp >= 0X4E_00 and cp <= 0X9F_FF)
         or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
         or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
         or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
         or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
         or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
         or (cp >= 0XF9_00 and cp <= 0XFA_FF)
         or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
     ):  #
         return True

     return False







class    lowerCAmelCase_		(		__magic_name__		):




 def __init__(	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False			,     _lowerCAmelCase = None			,     **_lowerCAmelCase		)    ->							Dict:
     super().__init__(_lowerCAmelCase			,     _lowerCAmelCase			,     **_lowerCAmelCase		)
     _lowerCAmelCase				=      Queue()
     _lowerCAmelCase				=      None
     _lowerCAmelCase				=      timeout




 def 							_snake_case   (	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False		)    ->							Any:
     self.text_queue.put(_lowerCAmelCase			,     timeout=self.timeout		)
     if stream_end:
         self.text_queue.put(self.stop_signal			,     timeout=self.timeout		)




 def __iter__(	self		)    ->							int:
     return self





 def 							_snake_case   (	self		)    ->							Tuple:
     _lowerCAmelCase				=      self.text_queue.get(timeout=self.timeout		)
     if value == self.stop_signal:
         raise StopIteration()
     else:
         return value


