






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowerCamelCase      :    List[str]   		=   {
    """configuration_funnel""": ["""FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """FunnelConfig"""],
    """convert_funnel_original_tf_checkpoint_to_pytorch""": [],
    """tokenization_funnel""": ["""FunnelTokenizer"""],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    str   		=   ["""FunnelTokenizerFast"""]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    List[Any]   		=   [
	    """FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """FunnelBaseModel""",
	    """FunnelForMaskedLM""",
	    """FunnelForMultipleChoice""",
	    """FunnelForPreTraining""",
	    """FunnelForQuestionAnswering""",
	    """FunnelForSequenceClassification""",
	    """FunnelForTokenClassification""",
	    """FunnelModel""",
	    """FunnelPreTrainedModel""",
	    """load_tf_weights_in_funnel""",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    Dict   		=   [
	    """TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """TFFunnelBaseModel""",
	    """TFFunnelForMaskedLM""",
	    """TFFunnelForMultipleChoice""",
	    """TFFunnelForPreTraining""",
	    """TFFunnelForQuestionAnswering""",
	    """TFFunnelForSequenceClassification""",
	    """TFFunnelForTokenClassification""",
	    """TFFunnelModel""",
	    """TFFunnelPreTrainedModel""",
	]


if TYPE_CHECKING:
	from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
	from .tokenization_funnel import FunnelTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_funnel_fast import FunnelTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_funnel import (
		    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		    FunnelPreTrainedModel,
		    load_tf_weights_in_funnel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_funnel import (
		    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFFunnelBaseModel,
		    TFFunnelForMaskedLM,
		    TFFunnelForMultipleChoice,
		    TFFunnelForPreTraining,
		    TFFunnelForQuestionAnswering,
		    TFFunnelForSequenceClassification,
		    TFFunnelForTokenClassification,
		    TFFunnelModel,
		    TFFunnelPreTrainedModel,
		)

else:
	import sys

	__lowerCamelCase      :    Dict   		=   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)











from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def   SCREAMING_SNAKE_CASE       (				snake_case_      :							List[Any]  ,						snake_case_      :							Union[str, Any]     ):
	snake_case__				:   Optional[int]     =						[]
	for part_id in partition_order:
		snake_case__				:   List[Any]     =						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''     ).collect()
		for row_idx, row in enumerate(snake_case_     ):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())     )
	return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Tuple     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Union[str, Any]     =						spark.range(100     ).repartition(1     )
	snake_case__				:   Any     =						Spark(snake_case_     )
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16     )
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Dict     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Optional[Any]     =						spark.range(10     ).repartition(2     )
	snake_case__				:   Optional[Any]     =						[1, 0]
	snake_case__				:   Dict     =						_generate_iterable_examples(snake_case_  ,						snake_case_     )  # Reverse the partitions.
	snake_case__				:   Tuple     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						snake_case_     )

	for i, (row_id, row_dict) in enumerate(generate_fn()     ):
		snake_case__,		snake_case__				:   Tuple     =						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Optional[int]     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Optional[int]     =						spark.range(10     ).repartition(1     )
	snake_case__				:   Union[str, Any]     =						SparkExamplesIterable(snake_case_     )
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(snake_case_     ):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Optional[int]     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   str     =						spark.range(30     ).repartition(3     )
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("numpy.random.Generator"     ) as generator_mock:
		snake_case__				:   Union[str, Any]     =						lambda snake_case_     : x.reverse()
		snake_case__				:   Optional[int]     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						[2, 1, 0]     )

		snake_case__				:   List[Any]     =						SparkExamplesIterable(snake_case_     ).shuffle_data_sources(snake_case_     )
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(snake_case_     ):
			snake_case__,		snake_case__				:   Optional[Any]     =						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Any     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Tuple     =						spark.range(20     ).repartition(4     )

	# Partitions 0 and 2
	snake_case__				:   List[Any]     =						SparkExamplesIterable(snake_case_     ).shard_data_sources(worker_id=0  ,						num_workers=2     )
	assert shard_it_a.n_shards == 2
	snake_case__				:   List[str]     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						[0, 2]     )
	for i, (row_id, row_dict) in enumerate(snake_case_     ):
		snake_case__,		snake_case__				:   Optional[int]     =						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	snake_case__				:   Any     =						SparkExamplesIterable(snake_case_     ).shard_data_sources(worker_id=1  ,						num_workers=2     )
	assert shard_it_a.n_shards == 2
	snake_case__				:   List[Any]     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						[1, 3]     )
	for i, (row_id, row_dict) in enumerate(snake_case_     ):
		snake_case__,		snake_case__				:   Optional[Any]     =						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Dict     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Tuple     =						spark.range(100     ).repartition(1     )
	snake_case__				:   Union[str, Any]     =						Spark(snake_case_     )
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1     )
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 100



