





import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader



def     _lowerCAmelCase		(  _lowerCAmelCase     ):
   '''simple docstring'''


   A_     :				str			      =	np.inf

   def set_batch_size(_lowerCAmelCase     ) -> None:
      nonlocal batch_size
      if isinstance(_lowerCAmelCase  ,_lowerCAmelCase     ):
         A_     :				List[str]			      =	min(_lowerCAmelCase  ,config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS     )
      elif isinstance(_lowerCAmelCase  ,_lowerCAmelCase     ):
         A_     :				Optional[int]			      =	min(_lowerCAmelCase  ,config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS     )
      elif isinstance(_lowerCAmelCase  ,_lowerCAmelCase     ) and feature.dtype == "binary":
         A_     :				Tuple			      =	min(_lowerCAmelCase  ,config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS     )

   _visit(_lowerCAmelCase  ,_lowerCAmelCase     )

   return None if batch_size is np.inf else batch_size

class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__      ,       a__ = None      ,       a__ = None      ,       a__ = None      ,       a__ = False      ,       a__ = False      ,       a__ = None      ,       **a__      ,       ):
         super().__init__(
             a__      ,       split=a__      ,       features=a__      ,       cache_dir=a__      ,       keep_in_memory=a__      ,       streaming=a__      ,       num_proc=a__      ,       **a__      ,       )
         A_     :				List[str]			      =	path_or_paths if isinstance(a__      ,       a__       ) else {self.split: path_or_paths}
         A_     :				Dict			      =	_PACKAGED_DATASETS_MODULES["""parquet"""][1]
         A_     :				Tuple			      =	Parquet(
             cache_dir=a__      ,       data_files=a__      ,       features=a__      ,       hash=a__      ,       **a__      ,       )






      def 		_lowerCamelCase      (		self       ):
         # Build iterable dataset
         if self.streaming:
            A_     :				Dict			      =	self.builder.as_streaming_dataset(split=self.split       )
         # Build regular (map-style) dataset
         else:
            A_     :				List[Any]			      =	None
            A_     :				Tuple			      =	None
            A_     :				Dict			      =	None
            A_     :				List[Any]			      =	None

            self.builder.download_and_prepare(
                download_config=a__      ,       download_mode=a__      ,       verification_mode=a__      ,       base_path=a__      ,       num_proc=self.num_proc      ,       )
            A_     :				Dict			      =	self.builder.as_dataset(
                split=self.split      ,       verification_mode=a__      ,       in_memory=self.keep_in_memory       )
         return dataset






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__      ,       a__ = None      ,       **a__      ,       ):
         A_     :				Tuple			      =	dataset
         A_     :				Optional[Any]			      =	path_or_buf
         A_     :				Any			      =	batch_size or get_writer_batch_size(dataset.features       )
         A_     :				Dict			      =	parquet_writer_kwargs
      def 		_lowerCamelCase      (		self       ):
         A_     :				Union[str, Any]			      =	self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

         if isinstance(self.path_or_buf      ,       (str, bytes, os.PathLike)       ):
            with open(self.path_or_buf      ,       """wb+"""       ) as buffer:
               A_     :				Optional[Any]			      =	self._write(file_obj=a__      ,       batch_size=a__      ,       **self.parquet_writer_kwargs       )
         else:
            A_     :				Optional[Any]			      =	self._write(file_obj=self.path_or_buf      ,       batch_size=a__      ,       **self.parquet_writer_kwargs       )
         return written






      def 		_lowerCamelCase      (		self      ,       a__      ,       a__      ,       **a__       ):
         A_     :				Tuple			      =	0
         A_     :				Dict			      =	parquet_writer_kwargs.pop("""path_or_buf"""      ,       a__       )
         A_     :				Optional[Any]			      =	self.dataset.features.arrow_schema

         A_     :				Dict			      =	pq.ParquetWriter(a__      ,       schema=a__      ,       **a__       )

         for offset in logging.tqdm(
             range(0      ,       len(self.dataset       )      ,       a__       )      ,       unit="""ba"""      ,       disable=not logging.is_progress_bar_enabled()      ,       desc="""Creating parquet from Arrow format"""      ,       ):
            A_     :				List[Any]			      =	query_table(
                table=self.dataset._data      ,       key=slice(a__      ,       offset + batch_size       )      ,       indices=self.dataset._indices if self.dataset._indices is not None else None      ,       )
            writer.write_table(a__       )
            written += batch.nbytes
         writer.close()
         return written













# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


_lowerCAmelCase 				=     """3"""

print("""Python version:""", sys.version)
print("""transformers version:""", transformers.__version__)

try:
      import torch

      print("""Torch version:""", torch.__version__)
      print("""Cuda available:""", torch.cuda.is_available())
      print("""Cuda version:""", torch.version.cuda)
      print("""CuDNN version:""", torch.backends.cudnn.version())
      print("""Number of GPUs available:""", torch.cuda.device_count())
      print("""NCCL version:""", torch.cuda.nccl.version())
except ImportError:
      print("""Torch version:""", None)

try:
      import deepspeed

      print("""DeepSpeed version:""", deepspeed.__version__)
except ImportError:
      print("""DeepSpeed version:""", None)

try:
      import tensorflow as tf

      print("""TensorFlow version:""", tf.__version__)
      print("""TF GPUs available:""", bool(tf.config.list_physical_devices("""GPU""")))
      print("""Number of TF GPUs available:""", len(tf.config.list_physical_devices("""GPU""")))
except ImportError:
      print("""TensorFlow version:""", None)






