





'''simple docstring'''


import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_SCREAMING_SNAKE_CASE	 =							datasets.utils.logging.get_logger(__name__)

@dataclass
class    lowerCAmelCase_							(		datasets.BuilderConfig		):
 __lowerCamelCase						:							int         =					10_000
 __lowerCamelCase						:							Optional[List[str]]         =					None
 __lowerCamelCase						:							Optional[datasets.Features]         =					None







class    lowerCAmelCase_							(		datasets.ArrowBasedBuilder		):
 __lowerCamelCase						:							str         =					ParquetConfig




 def 							_snake_case   (	self		)    ->							Optional[Any]:
     return datasets.DatasetInfo(features=self.config.features		)




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							int:
     if not self.config.data_files:
         raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''		)
     _lowerCAmelCase				=      dl_manager.download_and_extract(self.config.data_files		)
     if isinstance(_lowerCAmelCase			,     (str, list, tuple)		):
         _lowerCAmelCase				=      data_files
         if isinstance(_lowerCAmelCase			,     _lowerCAmelCase		):
             _lowerCAmelCase				=      [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _lowerCAmelCase				=      [dl_manager.iter_files(_lowerCAmelCase		) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,     gen_kwargs={"files": files}		)]
     _lowerCAmelCase				=      []
     for split_name, files in data_files.items():
         if isinstance(_lowerCAmelCase			,     _lowerCAmelCase		):
             _lowerCAmelCase				=      [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _lowerCAmelCase				=      [dl_manager.iter_files(_lowerCAmelCase		) for file in files]
         # Infer features is they are stoed in the arrow schema
         if self.info.features is None:
             for file in itertools.chain.from_iterable(_lowerCAmelCase		):
                 with open(_lowerCAmelCase			,     "rb"		) as f:
                     _lowerCAmelCase				=      datasets.Features.from_arrow_schema(pq.read_schema(_lowerCAmelCase		)		)
                 break
         splits.append(datasets.SplitGenerator(name=_lowerCAmelCase			,     gen_kwargs={"files": files}		)		)
     return splits




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							pa.Table:
     if self.info.features is not None:
         # more expensive cast to support nested features with keys in a different order
         # allows str <-> int/float or str to Audio for example
         _lowerCAmelCase				=      table_cast(_lowerCAmelCase			,     self.info.features.arrow_schema		)
     return pa_table





 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							List[str]:
     _lowerCAmelCase				=      self.info.features.arrow_schema if self.info.features is not None else None
     if self.info.features is not None and self.config.columns is not None:
         if sorted(field.name for field in schema		) != sorted(self.config.columns		):
             raise ValueError(
                 f'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''		)
     for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase		)		):
         with open(_lowerCAmelCase			,     "rb"		) as f:
             _lowerCAmelCase				=      pq.ParquetFile(_lowerCAmelCase		)
             try:
                 for batch_idx, record_batch in enumerate(
                     parquet_file.iter_batches(batch_size=self.config.batch_size			,     columns=self.config.columns		)		):
                     _lowerCAmelCase				=      pa.Table.from_batches([record_batch]		)
                     # Uncomment for debugging (will print the Arrow table size and elements)
                     # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                     # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                     yield f'''{file_idx}_{batch_idx}''', self._cast_table(_lowerCAmelCase		)
             except ValueError as e:
                 logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase		)}: {e}'''		)
                 raise









'''simple docstring'''


from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
       from ..models.auto import AutoTokenizer

class    lowerCAmelCase_		:




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							Tuple:
     raise NotImplementedError()





 def 							_snake_case   (	self		)    ->							Optional[int]:
     raise NotImplementedError()







class    lowerCAmelCase_		(		__magic_name__		):




 def __init__(	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False			,     **_lowerCAmelCase		)    ->							str:
     _lowerCAmelCase				=      tokenizer
     _lowerCAmelCase				=      skip_prompt
     _lowerCAmelCase				=      decode_kwargs

     # variables used in the streaming process
     _lowerCAmelCase				=      []
     _lowerCAmelCase				=      0
     _lowerCAmelCase				=      True




 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							List[Any]:
     if len(value.shape		) > 1 and value.shape[0] > 1:
         raise ValueError("TextStreamer only supports batch size 1"		)
     elif len(value.shape		) > 1:
         _lowerCAmelCase				=      value[0]

     if self.skip_prompt and self.next_tokens_are_prompt:
         _lowerCAmelCase				=      False
         return

     # Add the new token to the cache and decodes the entire thing.
     self.token_cache.extend(value.tolist()		)
     _lowerCAmelCase				=      self.tokenizer.decode(self.token_cache			,     **self.decode_kwargs		)

     # After the symbol for a new line, we flush the cache.
     if text.endswith("\n"		):
         _lowerCAmelCase				=      text[self.print_len :]
         _lowerCAmelCase				=      []
         _lowerCAmelCase				=      0
     # If the last token is a CJK character, we print the characters.
     elif len(_lowerCAmelCase		) > 0 and self._is_chinese_char(ord(text[-1]		)		):
         _lowerCAmelCase				=      text[self.print_len :]
         self.print_len += len(_lowerCAmelCase		)
     # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
     # which may change with the subsequent token -- there are probably smarter ways to do this!)
     else:
         _lowerCAmelCase				=      text[self.print_len : text.rfind(" "		) + 1]
         self.print_len += len(_lowerCAmelCase		)

     self.on_finalized_text(_lowerCAmelCase		)




 def 							_snake_case   (	self		)    ->							Union[str, Any]:
     # Flush the cache, if it exists
     if len(self.token_cache		) > 0:
         _lowerCAmelCase				=      self.tokenizer.decode(self.token_cache			,     **self.decode_kwargs		)
         _lowerCAmelCase				=      text[self.print_len :]
         _lowerCAmelCase				=      []
         _lowerCAmelCase				=      0
     else:
         _lowerCAmelCase				=      ""

     _lowerCAmelCase				=      True
     self.on_finalized_text(_lowerCAmelCase			,     stream_end=_lowerCAmelCase		)




 def 							_snake_case   (	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False		)    ->							List[Any]:
     print(_lowerCAmelCase			,     flush=_lowerCAmelCase			,     end="" if not stream_end else None		)





 def 							_snake_case   (	self			,     _lowerCAmelCase		)    ->							Tuple:
     # This defines a "chinese character" as anything in the CJK Unicode block:
     #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
     #
     # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
     # despite its name. The modern Korean Hangul alphabet is a different block,
     # as is Japanese Hiragana and Katakana. Those alphabets are used to write
     # space-separated words, so they are not treated specially and handled
     # like the all of the other languages.
     if (
         (cp >= 0X4E_00 and cp <= 0X9F_FF)
         or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
         or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
         or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
         or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
         or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
         or (cp >= 0XF9_00 and cp <= 0XFA_FF)
         or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
     ):  #
         return True

     return False







class    lowerCAmelCase_		(		__magic_name__		):




 def __init__(	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False			,     _lowerCAmelCase = None			,     **_lowerCAmelCase		)    ->							Dict:
     super().__init__(_lowerCAmelCase			,     _lowerCAmelCase			,     **_lowerCAmelCase		)
     _lowerCAmelCase				=      Queue()
     _lowerCAmelCase				=      None
     _lowerCAmelCase				=      timeout




 def 							_snake_case   (	self			,     _lowerCAmelCase			,     _lowerCAmelCase = False		)    ->							Any:
     self.text_queue.put(_lowerCAmelCase			,     timeout=self.timeout		)
     if stream_end:
         self.text_queue.put(self.stop_signal			,     timeout=self.timeout		)




 def __iter__(	self		)    ->							int:
     return self





 def 							_snake_case   (	self		)    ->							Tuple:
     _lowerCAmelCase				=      self.text_queue.get(timeout=self.timeout		)
     if value == self.stop_signal:
         raise StopIteration()
     else:
         return value


