
'''simple docstring'''






from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def 		A_       (	_lowerCAmelCase   :    List[Any]				,	_lowerCAmelCase   :    List[str]							):



      """simple docstring"""




      _lowerCamelCase				:   Dict								=   []
      for part_id in partition_order:
            _lowerCamelCase				:   Union[str, Any]								=   df.where(F'SPARK_PARTITION_ID() = {part_id}'							).collect()
            for row_idx, row in enumerate(_lowerCAmelCase							):
                  expected_row_ids_and_row_dicts.append((F'{part_id}_{row_idx}', row.asDict())							)
      return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def 		A_       (	):



      """simple docstring"""




      _lowerCamelCase				:   List[str]								=   pyspark.sql.SparkSession.builder.master("local[*]"							).appName("pyspark"							).getOrCreate()
      _lowerCamelCase				:   Optional[int]								=   spark.range(100							).repartition(1							)
      _lowerCamelCase				:   Tuple								=   Spark(_lowerCAmelCase							)
      # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
      # that each partition can hold 2 rows.
      spark_builder._repartition_df_if_needed(max_shard_size=16							)
      # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
      assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def 		A_       (	):



      """simple docstring"""




      _lowerCamelCase				:   Optional[Any]								=   pyspark.sql.SparkSession.builder.master("local[*]"							).appName("pyspark"							).getOrCreate()
      _lowerCamelCase				:   Dict								=   spark.range(10							).repartition(2							)
      _lowerCamelCase				:   str								=   [1, 0]
      _lowerCamelCase				:   int								=   _generate_iterable_examples(_lowerCAmelCase				,	_lowerCAmelCase							)  # Reverse the partitions.
      _lowerCamelCase				:   Union[str, Any]								=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase				,	_lowerCAmelCase							)

      for i, (row_id, row_dict) in enumerate(generate_fn()							):
            _lowerCamelCase						,		_lowerCamelCase				:   Dict								=   expected_row_ids_and_row_dicts[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		A_       (	):



      """simple docstring"""




      _lowerCamelCase				:   Optional[int]								=   pyspark.sql.SparkSession.builder.master("local[*]"							).appName("pyspark"							).getOrCreate()
      _lowerCamelCase				:   Tuple								=   spark.range(10							).repartition(1							)
      _lowerCamelCase				:   int								=   SparkExamplesIterable(_lowerCAmelCase							)
      assert it.n_shards == 1
      for i, (row_id, row_dict) in enumerate(_lowerCAmelCase							):
            assert row_id == F'0_{i}'
            assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def 		A_       (	):



      """simple docstring"""




      _lowerCamelCase				:   List[Any]								=   pyspark.sql.SparkSession.builder.master("local[*]"							).appName("pyspark"							).getOrCreate()
      _lowerCamelCase				:   Optional[int]								=   spark.range(30							).repartition(3							)
      # Mock the generator so that shuffle reverses the partition indices.
      with patch("numpy.random.Generator"							) as generator_mock:
            _lowerCamelCase				:   Tuple								=   lambda _lowerCAmelCase							: x.reverse()
            _lowerCamelCase				:   Optional[int]								=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase				,	[2, 1, 0]							)

            _lowerCamelCase				:   Tuple								=   SparkExamplesIterable(_lowerCAmelCase							).shuffle_data_sources(_lowerCAmelCase							)
            assert shuffled_it.n_shards == 3
            for i, (row_id, row_dict) in enumerate(_lowerCAmelCase							):
                  _lowerCamelCase						,		_lowerCamelCase				:   Optional[Any]								=   expected_row_ids_and_row_dicts[i]
                  assert row_id == expected_row_id
                  assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		A_       (	):



      """simple docstring"""




      _lowerCamelCase				:   List[str]								=   pyspark.sql.SparkSession.builder.master("local[*]"							).appName("pyspark"							).getOrCreate()
      _lowerCamelCase				:   Tuple								=   spark.range(20							).repartition(4							)

      # Partitions 0 and 2
      _lowerCamelCase				:   Optional[int]								=   SparkExamplesIterable(_lowerCAmelCase							).shard_data_sources(worker_id=0				,	num_workers=2							)
      assert shard_it_a.n_shards == 2
      _lowerCamelCase				:   List[Any]								=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase				,	[0, 2]							)
      for i, (row_id, row_dict) in enumerate(_lowerCAmelCase							):
            _lowerCamelCase						,		_lowerCamelCase				:   Optional[Any]								=   expected_row_ids_and_row_dicts_a[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict

      # Partitions 1 and 3
      _lowerCamelCase				:   Dict								=   SparkExamplesIterable(_lowerCAmelCase							).shard_data_sources(worker_id=1				,	num_workers=2							)
      assert shard_it_a.n_shards == 2
      _lowerCamelCase				:   int								=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase				,	[1, 3]							)
      for i, (row_id, row_dict) in enumerate(_lowerCAmelCase							):
            _lowerCamelCase						,		_lowerCamelCase				:   Optional[Any]								=   expected_row_ids_and_row_dicts_a[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		A_       (	):



      """simple docstring"""




      _lowerCamelCase				:   Optional[int]								=   pyspark.sql.SparkSession.builder.master("local[*]"							).appName("pyspark"							).getOrCreate()
      _lowerCamelCase				:   Dict								=   spark.range(100							).repartition(1							)
      _lowerCamelCase				:   List[str]								=   Spark(_lowerCAmelCase							)
      # Choose a small max_shard_size for maximum partitioning.
      spark_builder._repartition_df_if_needed(max_shard_size=1							)
      # The new number of partitions should not be greater than the number of rows.
      assert spark_builder.df.rdd.getNumPartitions() == 100

'''simple docstring'''






import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
 import jax.numpy as jnp

 from transformers.models.albert.modeling_flax_albert import (
     FlaxAlbertForMaskedLM,
     FlaxAlbertForMultipleChoice,
     FlaxAlbertForPreTraining,
     FlaxAlbertForQuestionAnswering,
     FlaxAlbertForSequenceClassification,
     FlaxAlbertForTokenClassification,
     FlaxAlbertModel,
 )

class  UpperCAmelCase__						(       unittest.TestCase ):



    def __init__(	self      :       Union[str, Any],__A      :       Union[str, Any],__A      :       Optional[int]=1_3,__A      :       Optional[Any]=7,__A      :       int=True,__A      :       Any=True,__A      :       Dict=True,__A      :       List[Any]=True,__A      :       Optional[Any]=9_9,__A      :       List[str]=3_2,__A      :       int=5,__A      :       str=4,__A      :       int=3_7,__A      :       Optional[Any]="gelu",__A      :       Union[str, Any]=0.1,__A      :       str=0.1,__A      :       Any=5_1_2,__A      :       Any=1_6,__A      :       Optional[int]=2,__A      :       int=0.02,__A      :       Optional[int]=4,):
          _lowerCamelCase				:   Union[str, Any]								=   parent
          _lowerCamelCase				:   Tuple								=   batch_size
          _lowerCamelCase				:   List[str]								=   seq_length
          _lowerCamelCase				:   Optional[int]								=   is_training
          _lowerCamelCase				:   str								=   use_attention_mask
          _lowerCamelCase				:   str								=   use_token_type_ids
          _lowerCamelCase				:   List[str]								=   use_labels
          _lowerCamelCase				:   Tuple								=   vocab_size
          _lowerCamelCase				:   Dict								=   hidden_size
          _lowerCamelCase				:   str								=   num_hidden_layers
          _lowerCamelCase				:   List[str]								=   num_attention_heads
          _lowerCamelCase				:   Optional[Any]								=   intermediate_size
          _lowerCamelCase				:   Optional[int]								=   hidden_act
          _lowerCamelCase				:   Any								=   hidden_dropout_prob
          _lowerCamelCase				:   List[Any]								=   attention_probs_dropout_prob
          _lowerCamelCase				:   Tuple								=   max_position_embeddings
          _lowerCamelCase				:   str								=   type_vocab_size
          _lowerCamelCase				:   str								=   type_sequence_label_size
          _lowerCamelCase				:   int								=   initializer_range
          _lowerCamelCase				:   List[Any]								=   num_choices



    def   lowerCamelCase_  (	self      :       Optional[int]     ):
          _lowerCamelCase				:   List[Any]								=   ids_tensor([self.batch_size, self.seq_length],self.vocab_size     )

          _lowerCamelCase				:   Union[str, Any]								=   None
          if self.use_attention_mask:
                _lowerCamelCase				:   Tuple								=   random_attention_mask([self.batch_size, self.seq_length]     )

          _lowerCamelCase				:   Any								=   None
          if self.use_token_type_ids:
                _lowerCamelCase				:   int								=   ids_tensor([self.batch_size, self.seq_length],self.type_vocab_size     )

          _lowerCamelCase				:   int								=   AlbertConfig(
              vocab_size=self.vocab_size,hidden_size=self.hidden_size,num_hidden_layers=self.num_hidden_layers,num_attention_heads=self.num_attention_heads,intermediate_size=self.intermediate_size,hidden_act=self.hidden_act,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,is_decoder=__A,initializer_range=self.initializer_range,)

          return config, input_ids, token_type_ids, attention_mask



    def   lowerCamelCase_  (	self      :       Dict     ):
          _lowerCamelCase				:   str								=   self.prepare_config_and_inputs()
          _lowerCamelCase						,		_lowerCamelCase						,		_lowerCamelCase						,		_lowerCamelCase				:   Union[str, Any]								=   config_and_inputs
          _lowerCamelCase				:   Any								=   {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
          return config, inputs_dict


@require_flax
class  UpperCAmelCase__						(       A						,	unittest.TestCase ):
    lowerCAmelCase_				    =		(
        (
            FlaxAlbertModel,
            FlaxAlbertForPreTraining,
            FlaxAlbertForMaskedLM,
            FlaxAlbertForMultipleChoice,
            FlaxAlbertForQuestionAnswering,
            FlaxAlbertForSequenceClassification,
            FlaxAlbertForTokenClassification,
            FlaxAlbertForQuestionAnswering,
        )
        if is_flax_available()
        else ()
    )



    def   lowerCamelCase_  (	self      :       Any     ):
          _lowerCamelCase				:   List[str]								=   FlaxAlbertModelTester(self     )



    @slow
    def   lowerCamelCase_  (	self      :       Union[str, Any]     ):
          for model_class_name in self.all_model_classes:
                _lowerCamelCase				:   str								=   model_class_name.from_pretrained("albert-base-v2"     )
                _lowerCamelCase				:   Any								=   model(np.ones((1, 1)     )     )
                self.assertIsNotNone(__A     )


@require_flax
class  UpperCAmelCase__						(       unittest.TestCase ):



    @slow
    def   lowerCamelCase_  (	self      :       Tuple     ):
          _lowerCamelCase				:   Optional[int]								=   FlaxAlbertModel.from_pretrained("albert-base-v2"     )
          _lowerCamelCase				:   int								=   np.array([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]     )
          _lowerCamelCase				:   Union[str, Any]								=   np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]     )
          _lowerCamelCase				:   Optional[Any]								=   model(__A,attention_mask=__A     )[0]
          _lowerCamelCase				:   Dict								=   (1, 1_1, 7_6_8)
          self.assertEqual(output.shape,__A     )
          _lowerCamelCase				:   Tuple								=   np.array(
              [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]     )

          self.assertTrue(jnp.allclose(output[:, 1:4, 1:4],__A,atol=1e-4     )     )