



'''simple docstring'''



from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)



def   __lowerCAmelCase					(							lowerCamelCase   :						Dict				,		lowerCamelCase   :						Tuple					):






	'''simple docstring'''




	__lowerCAmelCase												=  []
	for part_id in partition_order:
		__lowerCAmelCase												=  df.where(f'''SPARK_PARTITION_ID() = {part_id}'''					).collect()
		for row_idx, row in enumerate(lowerCamelCase					):
			expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict())					)
	return expected_row_ids_and_row_dicts



@require_not_windows
@require_dill_gt_0_3_2
def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  pyspark.sql.SparkSession.builder.master("local[*]"					).appName("pyspark"					).getOrCreate()
	__lowerCAmelCase												=  spark.range(1_00					).repartition(1					)
	__lowerCAmelCase												=  Spark(lowerCamelCase					)
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16					)
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50



@require_not_windows
@require_dill_gt_0_3_2
def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  pyspark.sql.SparkSession.builder.master("local[*]"					).appName("pyspark"					).getOrCreate()
	__lowerCAmelCase												=  spark.range(10					).repartition(2					)
	__lowerCAmelCase												=  [1, 0]
	__lowerCAmelCase												=  _generate_iterable_examples(lowerCamelCase				,		lowerCamelCase					)  # Reverse the partitions.
	__lowerCAmelCase												=  _get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase				,		lowerCamelCase					)

	for i, (row_id, row_dict) in enumerate(generate_fn()					):
		__lowerCAmelCase							,       __lowerCAmelCase												=  expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  pyspark.sql.SparkSession.builder.master("local[*]"					).appName("pyspark"					).getOrCreate()
	__lowerCAmelCase												=  spark.range(10					).repartition(1					)
	__lowerCAmelCase												=  SparkExamplesIterable(lowerCamelCase					)
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(lowerCamelCase					):
		assert row_id == f'''0_{i}'''
		assert row_dict == {"id": i}



@require_not_windows
@require_dill_gt_0_3_2
def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  pyspark.sql.SparkSession.builder.master("local[*]"					).appName("pyspark"					).getOrCreate()
	__lowerCAmelCase												=  spark.range(30					).repartition(3					)
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("numpy.random.Generator"					) as generator_mock:
		__lowerCAmelCase												=  lambda lowerCamelCase					: x.reverse()
		__lowerCAmelCase												=  _get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase				,		[2, 1, 0]					)

		__lowerCAmelCase												=  SparkExamplesIterable(lowerCamelCase					).shuffle_data_sources(lowerCamelCase					)
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(lowerCamelCase					):
			__lowerCAmelCase							,       __lowerCAmelCase												=  expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  pyspark.sql.SparkSession.builder.master("local[*]"					).appName("pyspark"					).getOrCreate()
	__lowerCAmelCase												=  spark.range(20					).repartition(4					)

	# Partitions 0 and 2
	__lowerCAmelCase												=  SparkExamplesIterable(lowerCamelCase					).shard_data_sources(worker_id=0				,		num_workers=2					)
	assert shard_it_a.n_shards == 2
	__lowerCAmelCase												=  _get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase				,		[0, 2]					)
	for i, (row_id, row_dict) in enumerate(lowerCamelCase					):
		__lowerCAmelCase							,       __lowerCAmelCase												=  expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	__lowerCAmelCase												=  SparkExamplesIterable(lowerCamelCase					).shard_data_sources(worker_id=1				,		num_workers=2					)
	assert shard_it_a.n_shards == 2
	__lowerCAmelCase												=  _get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase				,		[1, 3]					)
	for i, (row_id, row_dict) in enumerate(lowerCamelCase					):
		__lowerCAmelCase							,       __lowerCAmelCase												=  expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  pyspark.sql.SparkSession.builder.master("local[*]"					).appName("pyspark"					).getOrCreate()
	__lowerCAmelCase												=  spark.range(1_00					).repartition(1					)
	__lowerCAmelCase												=  Spark(lowerCamelCase					)
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1					)
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00




'''simple docstring'''



from argparse import ArgumentParser

from . import BaseTransformersCLICommand



def   __lowerCAmelCase					(							lowerCamelCase   :						List[str]					):






	'''simple docstring'''




	return DownloadCommand(args.model				,		args.cache_dir				,		args.force				,		args.trust_remote_code					)






class 					UpperCAmelCase__	(		UpperCamelCase__ ):



					@staticmethod
					def        UpperCAmelCase_ (    UpperCamelCase     )     ->			Tuple:
						__lowerCAmelCase												=  parser.add_parser("download"     )
						download_parser.add_argument(
						    "--cache-dir"			,				type=UpperCamelCase			,				default=UpperCamelCase			,				help="Path to location to store the models"     )
						download_parser.add_argument(
						    "--force"			,				action="store_true"			,				help="Force the model to be download even if already in cache-dir"     )
						download_parser.add_argument(
						    "--trust-remote-code"			,				action="store_true"			,				help="Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you've reviewed the code as it will execute on your local machine"			,				)
						download_parser.add_argument("model"			,				type=UpperCamelCase			,				help="Name of the model to download"     )
						download_parser.set_defaults(func=UpperCamelCase     )



					def __init__(    self			,				UpperCamelCase			,				UpperCamelCase			,				UpperCamelCase			,				UpperCamelCase     )     ->			List[str]:
						__lowerCAmelCase												=  model
						__lowerCAmelCase												=  cache
						__lowerCAmelCase												=  force
						__lowerCAmelCase												=  trust_remote_code



					def        UpperCAmelCase_ (    self     )     ->			Any:
						from ..models.auto import AutoModel, AutoTokenizer

						AutoModel.from_pretrained(
						    self._model			,				cache_dir=self._cache			,				force_download=self._force			,				trust_remote_code=self._trust_remote_code     )
						AutoTokenizer.from_pretrained(
						    self._model			,				cache_dir=self._cache			,				force_download=self._force			,				trust_remote_code=self._trust_remote_code     )