






"""simple docstring"""





import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


UpperCamelCase_	:				str										= datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
    import pyspark


@dataclass
class      _lowercase (     datasets.BuilderConfig ):
      _a   :     Optional[datasets.Features]								=   None



def     __lowercase					(    a :				"pyspark.sql.DataFrame"       ,		a :				List[int]       ,		)					->					Any:
      import pyspark

      def generate_fn():
            __snake_case      :    Optional[Any]  			=df.select('''*'''       ,		pyspark.sql.functions.spark_partition_id().alias('''part_id''' ) )
            for partition_id in partition_order:
                  __snake_case      :    Union[str, Any]  			=df_with_partition_id.select('''*''' ).where(f'''part_id = {partition_id}''' ).drop('''part_id''' )
                  __snake_case      :    Tuple  			=partition_df.collect()
                  __snake_case      :    Optional[Any]  			=0
                  for row in rows:
                        yield f'''{partition_id}_{row_id}''', row.asDict()
                        row_id += 1

      return generate_fn


class      _lowercase (     _BaseExamplesIterable ):


      def __init__(    self				:	Union[str, Any]							,   a				:	"pyspark.sql.DataFrame"							,   a				:	Any=None							,   ):





            """simple docstring"""







            __snake_case      :    Tuple  			=df
            __snake_case      :    List[str]  			=partition_order or range(self.df.rdd.getNumPartitions()						)
            __snake_case      :    Optional[int]  			=_generate_iterable_examples(self.df							,   self.partition_order						)


      def __iter__(    self				:	Tuple						):





            """simple docstring"""







            yield from self.generate_examples_fn()


      def _UpperCamelCase						(    self				:	List[str]							,   a				:	np.random.Generator						):





            """simple docstring"""







            __snake_case      :    Dict  			=list(range(self.df.rdd.getNumPartitions()						)						)
            generator.shuffle(a						)
            return SparkExamplesIterable(self.df							,   partition_order=a						)


      def _UpperCamelCase						(    self				:	Dict							,   a				:	int							,   a				:	int						):





            """simple docstring"""







            __snake_case      :    Tuple  			=self.split_shard_indices_by_worker(a							,   a						)
            return SparkExamplesIterable(self.df							,   partition_order=a						)







      @property
      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            return len(self.partition_order						)



class      _lowercase (     datasets.DatasetBuilder ):
      _a   :     List[str]								=   SparkConfig


      def __init__(    self				:	Tuple							,   a				:	"pyspark.sql.DataFrame"							,   a				:	str = None							,   a				:	str = None							,   **a				:	Union[str, Any]							,   ):





            """simple docstring"""







            import pyspark

            __snake_case      :    List[str]  			=pyspark.sql.SparkSession.builder.getOrCreate()
            __snake_case      :    Tuple  			=df
            __snake_case      :    Optional[Any]  			=working_dir

            super().__init__(
                cache_dir=a							,   config_name=str(self.df.semanticHash()						)							,   **a							,   )


      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            def create_cache_and_write_probe(a				:	Any						):
                  # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
                  # already exist.
                  os.makedirs(self._cache_dir							,   exist_ok=a						)
                  __snake_case      :    Tuple  			=os.path.join(self._cache_dir							,   '''fs_test''' + uuid.uuida().hex						)
                  # Opening the file in append mode will create a new file unless it already exists, in which case it will not
                  # change the file contents.
                  open(a							,   '''a'''						)
                  return [probe_file]

            if self._spark.conf.get('''spark.master'''							,   ''''''						).startswith('''local'''						):
                  return

            # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
            # accessible to the driver.
            # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
            if self._cache_dir:
                  __snake_case      :    List[Any]  			=(
                      self._spark.sparkContext.parallelize(range(1						)							,   1						).mapPartitions(a						).collect()
                  )
                  if os.path.isfile(probe[0]						):
                        return

            raise ValueError(
                '''When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir'''						)


      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            return datasets.DatasetInfo(features=self.config.features						)


      def _UpperCamelCase						(    self				:	Tuple							,   a				:	datasets.download.download_manager.DownloadManager						):





            """simple docstring"""







            return [datasets.SplitGenerator(name=datasets.Split.TRAIN						)]


      def _UpperCamelCase						(    self				:	Optional[int]							,   a				:	List[str]						):





            """simple docstring"""







            import pyspark

            def get_arrow_batch_size(a				:	Optional[Any]						):
                  for batch in it:
                        yield pa.RecordBatch.from_pydict({'''batch_bytes''': [batch.nbytes]}						)

            __snake_case      :    List[Any]  			=self.df.count()
            __snake_case      :    List[str]  			=df_num_rows if df_num_rows <= 1_0_0 else 1_0_0
            # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
            __snake_case      :    Optional[int]  			=(
                self.df.limit(a						)
                .repartition(1						)
                .mapInArrow(a							,   '''batch_bytes: long'''						)
                .agg(pyspark.sql.functions.sum('''batch_bytes'''						).alias('''sample_bytes'''						)						)
                .collect()[0]
                .sample_bytes
                / sample_num_rows
            )
            __snake_case      :    str  			=approx_bytes_per_row * df_num_rows
            if approx_total_size > max_shard_size:
                  # Make sure there is at least one row per partition.
                  __snake_case      :    int  			=min(a							,   int(approx_total_size / max_shard_size						)						)
                  __snake_case      :    Optional[Any]  			=self.df.repartition(a						)


      def _UpperCamelCase						(    self				:	Optional[Any]							,   a				:	str							,   a				:	str							,   a				:	int							,   ):





            """simple docstring"""







            import pyspark

            __snake_case      :    Dict  			=ParquetWriter if file_format == '''parquet''' else ArrowWriter
            __snake_case      :    List[str]  			=os.path.join(self._working_dir							,   os.path.basename(a						)						) if self._working_dir else fpath
            __snake_case      :    Optional[Any]  			=file_format == '''parquet'''

            # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
            # pickling the SparkContext.
            __snake_case      :    int  			=self.config.features
            __snake_case      :    Optional[Any]  			=self._writer_batch_size
            __snake_case      :    Union[str, Any]  			=self._fs.storage_options

            def write_arrow(a				:	List[Any]						):
                  # Within the same SparkContext, no two task attempts will share the same attempt ID.
                  __snake_case      :    Union[str, Any]  			=pyspark.TaskContext().taskAttemptId()
                  __snake_case      :    Dict  			=next(a							,   a						)
                  if first_batch is None:
                        # Some partitions might not receive any data.
                        return pa.RecordBatch.from_arrays(
                            [[task_id], [0], [0]]							,   names=['''task_id''', '''num_examples''', '''num_bytes''']							,   )
                  __snake_case      :    Tuple  			=0
                  __snake_case      :    Tuple  			=writer_class(
                      features=a							,   path=working_fpath.replace('''SSSSS'''							,   f'''{shard_id:05d}'''						).replace('''TTTTT'''							,   f'''{task_id:05d}'''						)							,   writer_batch_size=a							,   storage_options=a							,   embed_local_files=a							,   )
                  __snake_case      :    Optional[Any]  			=pa.Table.from_batches([first_batch]						)
                  writer.write_table(a						)
                  for batch in it:
                        if max_shard_size is not None and writer._num_bytes >= max_shard_size:
                              __snake_case      :    Union[str, Any]  			=writer.finalize()
                              writer.close()
                              yield pa.RecordBatch.from_arrays(
                                  [[task_id], [num_examples], [num_bytes]]							,   names=['''task_id''', '''num_examples''', '''num_bytes''']							,   )
                              shard_id += 1
                              __snake_case      :    List[Any]  			=writer_class(
                                  features=writer._features							,   path=working_fpath.replace('''SSSSS'''							,   f'''{shard_id:05d}'''						).replace('''TTTTT'''							,   f'''{task_id:05d}'''						)							,   writer_batch_size=a							,   storage_options=a							,   embed_local_files=a							,   )
                        __snake_case      :    int  			=pa.Table.from_batches([batch]						)
                        writer.write_table(a						)

                  if writer._num_bytes > 0:
                        __snake_case      :    Dict  			=writer.finalize()
                        writer.close()
                        yield pa.RecordBatch.from_arrays(
                            [[task_id], [num_examples], [num_bytes]]							,   names=['''task_id''', '''num_examples''', '''num_bytes''']							,   )

                  if working_fpath != fpath:
                        for file in os.listdir(os.path.dirname(a						)						):
                              __snake_case      :    List[str]  			=os.path.join(os.path.dirname(a						)							,   os.path.basename(a						)						)
                              shutil.move(a							,   a						)

            __snake_case      :    int  			=(
                self.df.mapInArrow(a							,   '''task_id: long, num_examples: long, num_bytes: long'''						)
                .groupBy('''task_id'''						)
                .agg(
                    pyspark.sql.functions.sum('''num_examples'''						).alias('''total_num_examples'''						)							,   pyspark.sql.functions.sum('''num_bytes'''						).alias('''total_num_bytes'''						)							,   pyspark.sql.functions.count('''num_bytes'''						).alias('''num_shards'''						)							,   pyspark.sql.functions.collect_list('''num_examples'''						).alias('''shard_lengths'''						)							,   )
                .collect()
            )
            for row in stats:
                  yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)


      def _UpperCamelCase						(    self				:	str							,   a				:	"datasets.SplitGenerator"							,   a				:	str = "arrow"							,   a				:	Optional[Union[str, int]] = None							,   a				:	Optional[int] = None							,   **a				:	Union[str, Any]							,   ):





            """simple docstring"""







            self._validate_cache_dir()

            __snake_case      :    Tuple  			=convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE						)
            self._repartition_df_if_needed(a						)
            __snake_case      :    Union[str, Any]  			=not is_remote_filesystem(self._fs						)
            __snake_case      :    str  			=os.path.join if is_local else posixpath.join

            __snake_case      :    Optional[Any]  			='''-TTTTT-SSSSS-of-NNNNN'''
            __snake_case      :    Optional[int]  			=f'''{self.name}-{split_generator.name}{SUFFIX}.{file_format}'''
            __snake_case      :    str  			=path_join(self._output_dir							,   a						)

            __snake_case      :    Dict  			=0
            __snake_case      :    Dict  			=0
            __snake_case      :    Dict  			=0
            __snake_case      :    str  			=[]
            __snake_case      :    List[str]  			=[]

            for task_id, content in self._prepare_split_single(a							,   a							,   a						):
                  (
                      __snake_case				
                  )      :    Any  			=content
                  if num_bytes > 0:
                        total_num_examples += num_examples
                        total_num_bytes += num_bytes
                        total_shards += num_shards
                        task_id_and_num_shards.append((task_id, num_shards)						)
                        all_shard_lengths.extend(a						)

            __snake_case      :    Dict  			=total_num_examples
            __snake_case      :    List[str]  			=total_num_bytes

            # should rename everything at the end
            logger.debug(f'''Renaming {total_shards} shards.'''						)
            if total_shards > 1:
                  __snake_case      :    Union[str, Any]  			=all_shard_lengths

                  # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
                  # pickling error due to pickling the SparkContext.
                  __snake_case      :    Union[str, Any]  			=self._fs

                  # use the -SSSSS-of-NNNNN pattern
                  def _rename_shard(
                      a				:	int							,   a				:	int							,   a				:	int							,   ):
                        rename(
                            a							,   fpath.replace('''SSSSS'''							,   f'''{shard_id:05d}'''						).replace('''TTTTT'''							,   f'''{task_id:05d}'''						)							,   fpath.replace('''TTTTT-SSSSS'''							,   f'''{global_shard_id:05d}'''						).replace('''NNNNN'''							,   f'''{total_shards:05d}'''						)							,   )

                  __snake_case      :    Optional[Any]  			=[]
                  __snake_case      :    Any  			=0
                  for i in range(len(a						)						):
                        __snake_case      :    Optional[int]  			=task_id_and_num_shards[i]
                        for shard_id in range(a						):
                              args.append([task_id, shard_id, global_shard_id]						)
                              global_shard_id += 1
                  self._spark.sparkContext.parallelize(a							,   len(a						)						).map(lambda a						: _rename_shard(*a						)						).collect()
            else:
                  # don't use any pattern
                  __snake_case      :    Tuple  			=0
                  __snake_case      :    Dict  			=task_id_and_num_shards[0][0]
                  self._rename(
                      fpath.replace('''SSSSS'''							,   f'''{shard_id:05d}'''						).replace('''TTTTT'''							,   f'''{task_id:05d}'''						)							,   fpath.replace(a							,   ''''''						)							,   )







      def _UpperCamelCase						(    self				:	List[Any]							,   a				:	"datasets.SplitGenerator"							,   ):





            """simple docstring"""







            return SparkExamplesIterable(self.df						)










"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase_	:				Any										= {"""configuration_reformer""": ["""REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ReformerConfig"""]}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase_	:				List[str]										= ["""ReformerTokenizer"""]

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase_	:				List[str]										= ["""ReformerTokenizerFast"""]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase_	:				Optional[int]										= [
        """REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """ReformerAttention""",
        """ReformerForMaskedLM""",
        """ReformerForQuestionAnswering""",
        """ReformerForSequenceClassification""",
        """ReformerLayer""",
        """ReformerModel""",
        """ReformerModelWithLMHead""",
        """ReformerPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig

    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_reformer import ReformerTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_reformer_fast import ReformerTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_reformer import (
            REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
            ReformerAttention,
            ReformerForMaskedLM,
            ReformerForQuestionAnswering,
            ReformerForSequenceClassification,
            ReformerLayer,
            ReformerModel,
            ReformerModelWithLMHead,
            ReformerPreTrainedModel,
        )

else:
    import sys

    UpperCamelCase_	:				Tuple										= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


