






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
	import torch

if is_vision_available():
	from PIL import Image

	from transformers import MobileViTImageProcessor



class 							SCREAMING_SNAKE_CASE__		(       unittest.TestCase		):





					"""simple docstring"""






					def __init__(      self  :						Dict							,	__A  :						Optional[Any]							,	__A  :						Union[str, Any]=7							,	__A  :						Any=3							,	__A  :						List[str]=1_8							,	__A  :						Any=3_0							,	__A  :						Tuple=4_0_0							,	__A  :						Any=True							,	__A  :						Tuple=None							,	__A  :						Optional[Any]=True							,	__A  :						str=None							,	__A  :						str=True							,	):
						snake_case__				:   Optional[int]     =						size if size is not None else {"shortest_edge": 2_0}
						snake_case__				:   int     =						crop_size if crop_size is not None else {"height": 1_8, "width": 1_8}
						snake_case__				:   Union[str, Any]     =						parent
						snake_case__				:   Any     =						batch_size
						snake_case__				:   Dict     =						num_channels
						snake_case__				:   List[str]     =						image_size
						snake_case__				:   Optional[int]     =						min_resolution
						snake_case__				:   List[str]     =						max_resolution
						snake_case__				:   str     =						do_resize
						snake_case__				:   int     =						size
						snake_case__				:   Optional[int]     =						do_center_crop
						snake_case__				:   List[Any]     =						crop_size
						snake_case__				:   Optional[Any]     =						do_flip_channel_order






					def  _lowercase  (      self  :						Union[str, Any] ):
						return {
						    "do_resize": self.do_resize,
						    "size": self.size,
						    "do_center_crop": self.do_center_crop,
						    "crop_size": self.crop_size,
						    "do_flip_channel_order": self.do_flip_channel_order,
						}



@require_torch
@require_vision
class 							SCREAMING_SNAKE_CASE__		(       UpperCamelCase_				,     unittest.TestCase		):





					"""simple docstring"""






					a_			=	MobileViTImageProcessor if is_vision_available() else None



					def  _lowercase  (      self  :						List[str] ):
						snake_case__				:   Dict     =						MobileViTImageProcessingTester(self )



					@property
					def  _lowercase  (      self  :						List[str] ):
						return self.image_processor_tester.prepare_image_processor_dict()



					def  _lowercase  (      self  :						str ):
						snake_case__				:   Optional[int]     =						self.image_processing_class(**self.image_processor_dict )
						self.assertTrue(hasattr(__A							,	"do_resize" ) )
						self.assertTrue(hasattr(__A							,	"size" ) )
						self.assertTrue(hasattr(__A							,	"do_center_crop" ) )
						self.assertTrue(hasattr(__A							,	"center_crop" ) )
						self.assertTrue(hasattr(__A							,	"do_flip_channel_order" ) )



					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   Any     =						self.image_processing_class.from_dict(self.image_processor_dict )
						self.assertEqual(image_processor.size							,	{"shortest_edge": 2_0} )
						self.assertEqual(image_processor.crop_size							,	{"height": 1_8, "width": 1_8} )

						snake_case__				:   Dict     =						self.image_processing_class.from_dict(self.image_processor_dict							,	size=4_2							,	crop_size=8_4 )
						self.assertEqual(image_processor.size							,	{"shortest_edge": 4_2} )
						self.assertEqual(image_processor.crop_size							,	{"height": 8_4, "width": 8_4} )



					def  _lowercase  (      self  :						Union[str, Any] ):
						pass



					def  _lowercase  (      self  :						Optional[int] ):
						# Initialize image_processing
						snake_case__				:   Any     =						self.image_processing_class(**self.image_processor_dict )
						# create random PIL images
						snake_case__				:   Tuple     =						prepare_image_inputs(self.image_processor_tester							,	equal_resolution=__A )
						for image in image_inputs:
							self.assertIsInstance(__A							,	Image.Image )

						# Test not batched input
						snake_case__				:   List[Any]     =						image_processing(image_inputs[0]							,	return_tensors="pt" ).pixel_values
						self.assertEqual(
						    encoded_images.shape							,	(
						        1,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )							,	)

						# Test batched
						snake_case__				:   Optional[int]     =						image_processing(__A							,	return_tensors="pt" ).pixel_values
						self.assertEqual(
						    encoded_images.shape							,	(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )							,	)



					def  _lowercase  (      self  :						str ):
						# Initialize image_processing
						snake_case__				:   List[str]     =						self.image_processing_class(**self.image_processor_dict )
						# create random numpy tensors
						snake_case__				:   Tuple     =						prepare_image_inputs(self.image_processor_tester							,	equal_resolution=__A							,	numpify=__A )
						for image in image_inputs:
							self.assertIsInstance(__A							,	np.ndarray )

						# Test not batched input
						snake_case__				:   Tuple     =						image_processing(image_inputs[0]							,	return_tensors="pt" ).pixel_values
						self.assertEqual(
						    encoded_images.shape							,	(
						        1,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )							,	)

						# Test batched
						snake_case__				:   Union[str, Any]     =						image_processing(__A							,	return_tensors="pt" ).pixel_values
						self.assertEqual(
						    encoded_images.shape							,	(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )							,	)






					def  _lowercase  (      self  :						int ):
						# Initialize image_processing
						snake_case__				:   Optional[Any]     =						self.image_processing_class(**self.image_processor_dict )
						# create random PyTorch tensors
						snake_case__				:   Any     =						prepare_image_inputs(self.image_processor_tester							,	equal_resolution=__A							,	torchify=__A )
						for image in image_inputs:
							self.assertIsInstance(__A							,	torch.Tensor )

						# Test not batched input
						snake_case__				:   int     =						image_processing(image_inputs[0]							,	return_tensors="pt" ).pixel_values
						self.assertEqual(
						    encoded_images.shape							,	(
						        1,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )							,	)

						# Test batched
						snake_case__				:   Dict     =						image_processing(__A							,	return_tensors="pt" ).pixel_values
						self.assertEqual(
						    encoded_images.shape							,	(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )							,	)











from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def   SCREAMING_SNAKE_CASE       (				snake_case_      :							List[Any]  ,						snake_case_      :							Union[str, Any]     ):
	snake_case__				:   Optional[int]     =						[]
	for part_id in partition_order:
		snake_case__				:   List[Any]     =						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''     ).collect()
		for row_idx, row in enumerate(snake_case_     ):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())     )
	return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Tuple     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Union[str, Any]     =						spark.range(100     ).repartition(1     )
	snake_case__				:   Any     =						Spark(snake_case_     )
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16     )
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Dict     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Optional[Any]     =						spark.range(10     ).repartition(2     )
	snake_case__				:   Optional[Any]     =						[1, 0]
	snake_case__				:   Dict     =						_generate_iterable_examples(snake_case_  ,						snake_case_     )  # Reverse the partitions.
	snake_case__				:   Tuple     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						snake_case_     )

	for i, (row_id, row_dict) in enumerate(generate_fn()     ):
		snake_case__,		snake_case__				:   Tuple     =						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Optional[int]     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Optional[int]     =						spark.range(10     ).repartition(1     )
	snake_case__				:   Union[str, Any]     =						SparkExamplesIterable(snake_case_     )
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(snake_case_     ):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Optional[int]     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   str     =						spark.range(30     ).repartition(3     )
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("numpy.random.Generator"     ) as generator_mock:
		snake_case__				:   Union[str, Any]     =						lambda snake_case_     : x.reverse()
		snake_case__				:   Optional[int]     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						[2, 1, 0]     )

		snake_case__				:   List[Any]     =						SparkExamplesIterable(snake_case_     ).shuffle_data_sources(snake_case_     )
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(snake_case_     ):
			snake_case__,		snake_case__				:   Optional[Any]     =						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Any     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Tuple     =						spark.range(20     ).repartition(4     )

	# Partitions 0 and 2
	snake_case__				:   List[Any]     =						SparkExamplesIterable(snake_case_     ).shard_data_sources(worker_id=0  ,						num_workers=2     )
	assert shard_it_a.n_shards == 2
	snake_case__				:   List[str]     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						[0, 2]     )
	for i, (row_id, row_dict) in enumerate(snake_case_     ):
		snake_case__,		snake_case__				:   Optional[int]     =						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	snake_case__				:   Any     =						SparkExamplesIterable(snake_case_     ).shard_data_sources(worker_id=1  ,						num_workers=2     )
	assert shard_it_a.n_shards == 2
	snake_case__				:   List[Any]     =						_get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_  ,						[1, 3]     )
	for i, (row_id, row_dict) in enumerate(snake_case_     ):
		snake_case__,		snake_case__				:   Optional[Any]     =						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def   SCREAMING_SNAKE_CASE       (				):
	snake_case__				:   Dict     =						pyspark.sql.SparkSession.builder.master("local[*]"     ).appName("pyspark"     ).getOrCreate()
	snake_case__				:   Tuple     =						spark.range(100     ).repartition(1     )
	snake_case__				:   Union[str, Any]     =						Spark(snake_case_     )
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1     )
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 100



