


from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def  SCREAMING_SNAKE_CASE_		(    __A				:    Union[str, Any]						,					__A				:    List[str]       )			->	Optional[int]:

       """simple docstring"""

       a_      :	Dict							=						[]
       for part_id in partition_order:
              a_      :	Optional[int]							=						df.where(F"""SPARK_PARTITION_ID() = {part_id}"""       ).collect()
              for row_idx, row in enumerate(__A       ):
                     expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict())       )
       return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE_		(    )			->	str:

       """simple docstring"""

       a_      :	Any							=						pyspark.sql.SparkSession.builder.master('local[*]'       ).appName('pyspark'       ).getOrCreate()
       a_      :	Optional[int]							=						spark.range(1_00       ).repartition(1       )
       a_      :	List[str]							=						Spark(__A       )
       # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
       # that each partition can hold 2 rows.
       spark_builder._repartition_df_if_needed(max_shard_size=16       )
       # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
       assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE_		(    )			->	Tuple:

       """simple docstring"""

       a_      :	Optional[Any]							=						pyspark.sql.SparkSession.builder.master('local[*]'       ).appName('pyspark'       ).getOrCreate()
       a_      :	Tuple							=						spark.range(10       ).repartition(2       )
       a_      :	List[Any]							=						[1, 0]
       a_      :	Tuple							=						_generate_iterable_examples(__A						,					__A       )  # Reverse the partitions.
       a_      :	Union[str, Any]							=						_get_expected_row_ids_and_row_dicts_for_partition_order(__A						,					__A       )

       for i, (row_id, row_dict) in enumerate(generate_fn()       ):
              a_							,	a_      :	List[Any]							=						expected_row_ids_and_row_dicts[i]
              assert row_id == expected_row_id
              assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE_		(    )			->	str:

       """simple docstring"""

       a_      :	Dict							=						pyspark.sql.SparkSession.builder.master('local[*]'       ).appName('pyspark'       ).getOrCreate()
       a_      :	Dict							=						spark.range(10       ).repartition(1       )
       a_      :	List[Any]							=						SparkExamplesIterable(__A       )
       assert it.n_shards == 1
       for i, (row_id, row_dict) in enumerate(__A       ):
              assert row_id == F"""0_{i}"""
              assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE_		(    )			->	Optional[int]:

       """simple docstring"""

       a_      :	Any							=						pyspark.sql.SparkSession.builder.master('local[*]'       ).appName('pyspark'       ).getOrCreate()
       a_      :	Tuple							=						spark.range(30       ).repartition(3       )
       # Mock the generator so that shuffle reverses the partition indices.
       with patch('numpy.random.Generator'       ) as generator_mock:
              a_      :	int							=						lambda __A       : x.reverse()
              a_      :	Optional[Any]							=						_get_expected_row_ids_and_row_dicts_for_partition_order(__A						,					[2, 1, 0]       )

              a_      :	Tuple							=						SparkExamplesIterable(__A       ).shuffle_data_sources(__A       )
              assert shuffled_it.n_shards == 3
              for i, (row_id, row_dict) in enumerate(__A       ):
                     a_							,	a_      :	Optional[int]							=						expected_row_ids_and_row_dicts[i]
                     assert row_id == expected_row_id
                     assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE_		(    )			->	Tuple:

       """simple docstring"""

       a_      :	Union[str, Any]							=						pyspark.sql.SparkSession.builder.master('local[*]'       ).appName('pyspark'       ).getOrCreate()
       a_      :	Optional[Any]							=						spark.range(20       ).repartition(4       )

       # Partitions 0 and 2
       a_      :	Optional[Any]							=						SparkExamplesIterable(__A       ).shard_data_sources(worker_id=0						,					num_workers=2       )
       assert shard_it_a.n_shards == 2
       a_      :	int							=						_get_expected_row_ids_and_row_dicts_for_partition_order(__A						,					[0, 2]       )
       for i, (row_id, row_dict) in enumerate(__A       ):
              a_							,	a_      :	Optional[int]							=						expected_row_ids_and_row_dicts_a[i]
              assert row_id == expected_row_id
              assert row_dict == expected_row_dict

       # Partitions 1 and 3
       a_      :	str							=						SparkExamplesIterable(__A       ).shard_data_sources(worker_id=1						,					num_workers=2       )
       assert shard_it_a.n_shards == 2
       a_      :	Tuple							=						_get_expected_row_ids_and_row_dicts_for_partition_order(__A						,					[1, 3]       )
       for i, (row_id, row_dict) in enumerate(__A       ):
              a_							,	a_      :	List[str]							=						expected_row_ids_and_row_dicts_a[i]
              assert row_id == expected_row_id
              assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE_		(    )			->	Dict:

       """simple docstring"""

       a_      :	List[str]							=						pyspark.sql.SparkSession.builder.master('local[*]'       ).appName('pyspark'       ).getOrCreate()
       a_      :	str							=						spark.range(1_00       ).repartition(1       )
       a_      :	Tuple							=						Spark(__A       )
       # Choose a small max_shard_size for maximum partitioning.
       spark_builder._repartition_df_if_needed(max_shard_size=1       )
       # The new number of partitions should not be greater than the number of rows.
       assert spark_builder.df.rdd.getNumPartitions() == 1_00






from typing import List, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_   :				Dict						      = logging.get_logger(__name__)

UpperCAmelCase_   :				int						      = {
    'huggingface/autoformer-tourism-monthly': 'https://huggingface.co/huggingface/autoformer-tourism-monthly/resolve/main/config.json',
}
class 				SCREAMING_SNAKE_CASE__						(       lowercase__    ):
      snake_case__       :  Union[str, Any]       =		'''autoformer'''
      snake_case__       :  Any       =		{
          '''hidden_size''': '''d_model''',
          '''num_attention_heads''': '''encoder_attention_heads''',
          '''num_hidden_layers''': '''encoder_layers''',
      }
      def __init__( self       : Optional[int]  ,							SCREAMING_SNAKE_CASE__       : Optional[int] = None  ,							SCREAMING_SNAKE_CASE__       : Optional[int] = None  ,							SCREAMING_SNAKE_CASE__       : str = "student_t"  ,							SCREAMING_SNAKE_CASE__       : str = "nll"  ,							SCREAMING_SNAKE_CASE__       : int = 1  ,							SCREAMING_SNAKE_CASE__       : List[int] = [1, 2, 3, 4, 5, 6, 7]  ,							SCREAMING_SNAKE_CASE__       : bool = True  ,							SCREAMING_SNAKE_CASE__       : int = 0  ,							SCREAMING_SNAKE_CASE__       : int = 0  ,							SCREAMING_SNAKE_CASE__       : int = 0  ,							SCREAMING_SNAKE_CASE__       : int = 0  ,							SCREAMING_SNAKE_CASE__       : Optional[List[int]] = None  ,							SCREAMING_SNAKE_CASE__       : Optional[List[int]] = None  ,							SCREAMING_SNAKE_CASE__       : int = 6_4  ,							SCREAMING_SNAKE_CASE__       : int = 2  ,							SCREAMING_SNAKE_CASE__       : int = 2  ,							SCREAMING_SNAKE_CASE__       : int = 2  ,							SCREAMING_SNAKE_CASE__       : int = 2  ,							SCREAMING_SNAKE_CASE__       : int = 3_2  ,							SCREAMING_SNAKE_CASE__       : int = 3_2  ,							SCREAMING_SNAKE_CASE__       : str = "gelu"  ,							SCREAMING_SNAKE_CASE__       : float = 0.1  ,							SCREAMING_SNAKE_CASE__       : float = 0.1  ,							SCREAMING_SNAKE_CASE__       : float = 0.1  ,							SCREAMING_SNAKE_CASE__       : float = 0.1  ,							SCREAMING_SNAKE_CASE__       : float = 0.1  ,							SCREAMING_SNAKE_CASE__       : int = 1_0_0  ,							SCREAMING_SNAKE_CASE__       : float = 0.02  ,							SCREAMING_SNAKE_CASE__       : bool = True  ,							SCREAMING_SNAKE_CASE__       : List[str]=True  ,							SCREAMING_SNAKE_CASE__       : int = 1_0  ,							SCREAMING_SNAKE_CASE__       : int = 2_5  ,							SCREAMING_SNAKE_CASE__       : int = 3  ,							**SCREAMING_SNAKE_CASE__       : int  ,							)	->    Union[str, Any]:
             # time series specific configuration
             a_      :	Optional[int]							=						prediction_length
             a_      :	Tuple							=						context_length if context_length is not None else prediction_length
             a_      :	Tuple							=						distribution_output
             a_      :	int							=						loss
             a_      :	Tuple							=						input_size
             a_      :	int							=						num_time_features
             a_      :	Dict							=						lags_sequence
             a_      :	List[str]							=						scaling
             a_      :	Tuple							=						num_dynamic_real_features
             a_      :	Dict							=						num_static_real_features
             a_      :	Dict							=						num_static_categorical_features
             if cardinality is not None and num_static_categorical_features > 0:
                    if len(SCREAMING_SNAKE_CASE__  ) != num_static_categorical_features:
                           raise ValueError(
                               'The cardinality should be a list of the same length as `num_static_categorical_features`'  )
                    a_      :	Any							=						cardinality
             else:
                    a_      :	Tuple							=						[0]
             if embedding_dimension is not None and num_static_categorical_features > 0:
                    if len(SCREAMING_SNAKE_CASE__  ) != num_static_categorical_features:
                           raise ValueError(
                               'The embedding dimension should be a list of the same length as `num_static_categorical_features`'  )
                    a_      :	Tuple							=						embedding_dimension
             else:
                    a_      :	Optional[Any]							=						[min(5_0  ,							(cat + 1) // 2  ) for cat in self.cardinality]
             a_      :	Optional[Any]							=						num_parallel_samples

             # Transformer architecture configuration
             a_      :	Dict							=						input_size * len(self.lags_sequence  ) + self._number_of_features
             a_      :	Tuple							=						d_model
             a_      :	int							=						encoder_attention_heads
             a_      :	Optional[int]							=						decoder_attention_heads
             a_      :	str							=						encoder_ffn_dim
             a_      :	List[str]							=						decoder_ffn_dim
             a_      :	Any							=						encoder_layers
             a_      :	Any							=						decoder_layers

             a_      :	List[Any]							=						dropout
             a_      :	Tuple							=						attention_dropout
             a_      :	Union[str, Any]							=						activation_dropout
             a_      :	List[Any]							=						encoder_layerdrop
             a_      :	List[str]							=						decoder_layerdrop

             a_      :	Optional[Any]							=						activation_function
             a_      :	Union[str, Any]							=						init_std

             a_      :	List[str]							=						use_cache

             # Autoformer
             a_      :	Tuple							=						label_length
             a_      :	List[Any]							=						moving_average
             a_      :	Optional[Any]							=						autocorrelation_factor

             super().__init__(is_encoder_decoder=SCREAMING_SNAKE_CASE__  ,							**SCREAMING_SNAKE_CASE__  )

      @property
      def  SCREAMING_SNAKE_CASE							( self       : List[Any]  )	->    int:
             return (
                 sum(self.embedding_dimension  )
                 + self.num_dynamic_real_features
                 + self.num_time_features
                 + self.num_static_real_features
                 + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
             )


