


"""simple docstring"""

from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)


def lowerCAmelCase_      (				UpperCamelCase__	: Tuple				,   UpperCamelCase__	: Dict ):






	"""simple docstring"""







	__lowercase									=	[]
	for part_id in partition_order:
		__lowercase									=	df.where(f'''SPARK_PARTITION_ID() = {part_id}''' ).collect()
		for row_idx, row in enumerate(UpperCamelCase__ ):
			expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict()) )
	return expected_row_ids_and_row_dicts


@require_not_windows
@require_dill_gt_0_3_2
def lowerCAmelCase_      (				):






	"""simple docstring"""







	__lowercase									=	pyspark.sql.SparkSession.builder.master("""local[*]""" ).appName("""pyspark""" ).getOrCreate()
	__lowercase									=	spark.range(100 ).repartition(1 )
	__lowercase									=	Spark(UpperCamelCase__ )
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16 )
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50


@require_not_windows
@require_dill_gt_0_3_2
def lowerCAmelCase_      (				):






	"""simple docstring"""







	__lowercase									=	pyspark.sql.SparkSession.builder.master("""local[*]""" ).appName("""pyspark""" ).getOrCreate()
	__lowercase									=	spark.range(10 ).repartition(2 )
	__lowercase									=	[1, 0]
	__lowercase									=	_generate_iterable_examples(UpperCamelCase__				,   UpperCamelCase__ )  # Reverse the partitions.
	__lowercase									=	_get_expected_row_ids_and_row_dicts_for_partition_order(UpperCamelCase__				,   UpperCamelCase__ )

	for i, (row_id, row_dict) in enumerate(generate_fn() ):
		__lowercase							,		__lowercase									=	expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def lowerCAmelCase_      (				):






	"""simple docstring"""







	__lowercase									=	pyspark.sql.SparkSession.builder.master("""local[*]""" ).appName("""pyspark""" ).getOrCreate()
	__lowercase									=	spark.range(10 ).repartition(1 )
	__lowercase									=	SparkExamplesIterable(UpperCamelCase__ )
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(UpperCamelCase__ ):
		assert row_id == f'''0_{i}'''
		assert row_dict == {"id": i}


@require_not_windows
@require_dill_gt_0_3_2
def lowerCAmelCase_      (				):






	"""simple docstring"""







	__lowercase									=	pyspark.sql.SparkSession.builder.master("""local[*]""" ).appName("""pyspark""" ).getOrCreate()
	__lowercase									=	spark.range(30 ).repartition(3 )
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("""numpy.random.Generator""" ) as generator_mock:
		__lowercase									=	lambda UpperCamelCase__ : x.reverse()
		__lowercase									=	_get_expected_row_ids_and_row_dicts_for_partition_order(UpperCamelCase__				,   [2, 1, 0] )

		__lowercase									=	SparkExamplesIterable(UpperCamelCase__ ).shuffle_data_sources(UpperCamelCase__ )
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(UpperCamelCase__ ):
			__lowercase							,		__lowercase									=	expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def lowerCAmelCase_      (				):






	"""simple docstring"""







	__lowercase									=	pyspark.sql.SparkSession.builder.master("""local[*]""" ).appName("""pyspark""" ).getOrCreate()
	__lowercase									=	spark.range(20 ).repartition(4 )

	# Partitions 0 and 2
	__lowercase									=	SparkExamplesIterable(UpperCamelCase__ ).shard_data_sources(worker_id=0				,   num_workers=2 )
	assert shard_it_a.n_shards == 2
	__lowercase									=	_get_expected_row_ids_and_row_dicts_for_partition_order(UpperCamelCase__				,   [0, 2] )
	for i, (row_id, row_dict) in enumerate(UpperCamelCase__ ):
		__lowercase							,		__lowercase									=	expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	__lowercase									=	SparkExamplesIterable(UpperCamelCase__ ).shard_data_sources(worker_id=1				,   num_workers=2 )
	assert shard_it_a.n_shards == 2
	__lowercase									=	_get_expected_row_ids_and_row_dicts_for_partition_order(UpperCamelCase__				,   [1, 3] )
	for i, (row_id, row_dict) in enumerate(UpperCamelCase__ ):
		__lowercase							,		__lowercase									=	expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def lowerCAmelCase_      (				):






	"""simple docstring"""







	__lowercase									=	pyspark.sql.SparkSession.builder.master("""local[*]""" ).appName("""pyspark""" ).getOrCreate()
	__lowercase									=	spark.range(100 ).repartition(1 )
	__lowercase									=	Spark(UpperCamelCase__ )
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1 )
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 100




"""simple docstring"""

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
       import torch

if is_vision_available():
       from PIL import Image

       from transformers import ChineseCLIPImageProcessor





class    lowerCamelCase__						(		unittest.TestCase    ):
      def __init__(		self	:	List[Any]      ,       A_	:	List[str]      ,       A_	:	str=7      ,       A_	:	Optional[int]=3      ,       A_	:	str=1_8      ,       A_	:	Tuple=3_0      ,       A_	:	Optional[Any]=4_0_0      ,       A_	:	Union[str, Any]=True      ,       A_	:	str=None      ,       A_	:	str=True      ,       A_	:	Optional[Any]=None      ,       A_	:	List[Any]=True      ,       A_	:	Union[str, Any]=[0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]      ,       A_	:	str=[0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]      ,       A_	:	List[str]=True      ,       ):






       '''simple docstring'''


       __lowercase									=	size if size is not None else {"""height""": 2_2_4, """width""": 2_2_4}
       __lowercase									=	crop_size if crop_size is not None else {"""height""": 1_8, """width""": 1_8}
       __lowercase									=	parent
       __lowercase									=	batch_size
       __lowercase									=	num_channels
       __lowercase									=	image_size
       __lowercase									=	min_resolution
       __lowercase									=	max_resolution
       __lowercase									=	do_resize
       __lowercase									=	size
       __lowercase									=	do_center_crop
       __lowercase									=	crop_size
       __lowercase									=	do_normalize
       __lowercase									=	image_mean
       __lowercase									=	image_std
       __lowercase									=	do_convert_rgb
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       return {
           "do_resize": self.do_resize,
           "size": self.size,
           "do_center_crop": self.do_center_crop,
           "crop_size": self.crop_size,
           "do_normalize": self.do_normalize,
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_convert_rgb": self.do_convert_rgb,
       }






      def SCREAMING_SNAKE_CASE_	(		self	:	str      ,       A_	:	List[Any]=False      ,       A_	:	List[Any]=False      ,       A_	:	Optional[Any]=False							):






       '''simple docstring'''


       assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

       if equal_resolution:
        __lowercase									=	[]
        for i in range(self.batch_size							):
         image_inputs.append(
             np.random.randint(
                 2_5_5      ,       size=(self.num_channels, self.max_resolution, self.max_resolution)      ,       dtype=np.uinta							)							)
       else:
        __lowercase									=	[]
        for i in range(self.batch_size							):
         __lowercase							,		__lowercase									=	np.random.choice(np.arange(self.min_resolution      ,       self.max_resolution							)      ,       2							)
         image_inputs.append(np.random.randint(2_5_5      ,       size=(self.num_channels, width, height)      ,       dtype=np.uinta							)							)

       if not numpify and not torchify:
        # PIL expects the channel dimension as last dimension
        __lowercase									=	[Image.fromarray(np.moveaxis(A_      ,       0      ,       -1							)							) for x in image_inputs]

       if torchify:
        __lowercase									=	[torch.from_numpy(A_							) for x in image_inputs]

       return image_inputs






@require_torch
@require_vision
class    lowerCamelCase__						(		_a			,   unittest.TestCase    ):
      a :     Union[str, Any]											=		ChineseCLIPImageProcessor if is_vision_available() else None
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	ChineseCLIPImageProcessingTester(self      ,       do_center_crop=A_							)
      @property
      def SCREAMING_SNAKE_CASE_	(		self	:	Union[str, Any]							):






       '''simple docstring'''


       return self.image_processor_tester.prepare_image_processor_dict()
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class(**self.image_processor_dict							)
       self.assertTrue(hasattr(A_      ,       """do_resize"""							)							)
       self.assertTrue(hasattr(A_      ,       """size"""							)							)
       self.assertTrue(hasattr(A_      ,       """do_center_crop"""							)							)
       self.assertTrue(hasattr(A_      ,       """center_crop"""							)							)
       self.assertTrue(hasattr(A_      ,       """do_normalize"""							)							)
       self.assertTrue(hasattr(A_      ,       """image_mean"""							)							)
       self.assertTrue(hasattr(A_      ,       """image_std"""							)							)
       self.assertTrue(hasattr(A_      ,       """do_convert_rgb"""							)							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class.from_dict(self.image_processor_dict							)
       self.assertEqual(image_processor.size      ,       {"""height""": 2_2_4, """width""": 2_2_4}							)
       self.assertEqual(image_processor.crop_size      ,       {"""height""": 1_8, """width""": 1_8}							)

       __lowercase									=	self.image_processing_class.from_dict(self.image_processor_dict      ,       size=4_2      ,       crop_size=8_4							)
       self.assertEqual(image_processor.size      ,       {"""shortest_edge""": 4_2}							)
       self.assertEqual(image_processor.crop_size      ,       {"""height""": 8_4, """width""": 8_4}							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       pass
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class(**self.image_processor_dict							)
       # create random PIL images
       __lowercase									=	self.image_processor_tester.prepare_inputs(equal_resolution=A_							)
       for image in image_inputs:
        self.assertIsInstance(A_      ,       Image.Image							)

       # Test not batched input
       __lowercase									=	image_processing(image_inputs[0]      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )

       # Test batched
       __lowercase									=	image_processing(A_      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )
      def SCREAMING_SNAKE_CASE_	(		self	:	Any							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class(**self.image_processor_dict							)
       # create random numpy tensors
       __lowercase									=	self.image_processor_tester.prepare_inputs(equal_resolution=A_      ,       numpify=A_							)
       for image in image_inputs:
        self.assertIsInstance(A_      ,       np.ndarray							)

       # Test not batched input
       __lowercase									=	image_processing(image_inputs[0]      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )

       # Test batched
       __lowercase									=	image_processing(A_      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )






      def SCREAMING_SNAKE_CASE_	(		self	:	Dict							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class(**self.image_processor_dict							)
       # create random PyTorch tensors
       __lowercase									=	self.image_processor_tester.prepare_inputs(equal_resolution=A_      ,       torchify=A_							)
       for image in image_inputs:
        self.assertIsInstance(A_      ,       torch.Tensor							)

       # Test not batched input
       __lowercase									=	image_processing(image_inputs[0]      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )

       # Test batched
       __lowercase									=	image_processing(A_      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )






@require_torch
@require_vision
class    lowerCamelCase__						(		_a			,   unittest.TestCase    ):
      a :     Optional[Any]											=		ChineseCLIPImageProcessor if is_vision_available() else None
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	ChineseCLIPImageProcessingTester(self      ,       num_channels=4      ,       do_center_crop=A_							)
       __lowercase									=	3
      @property
      def SCREAMING_SNAKE_CASE_	(		self	:	str							):






       '''simple docstring'''


       return self.image_processor_tester.prepare_image_processor_dict()
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class(**self.image_processor_dict							)
       self.assertTrue(hasattr(A_      ,       """do_resize"""							)							)
       self.assertTrue(hasattr(A_      ,       """size"""							)							)
       self.assertTrue(hasattr(A_      ,       """do_center_crop"""							)							)
       self.assertTrue(hasattr(A_      ,       """center_crop"""							)							)
       self.assertTrue(hasattr(A_      ,       """do_normalize"""							)							)
       self.assertTrue(hasattr(A_      ,       """image_mean"""							)							)
       self.assertTrue(hasattr(A_      ,       """image_std"""							)							)
       self.assertTrue(hasattr(A_      ,       """do_convert_rgb"""							)							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Union[str, Any]							):






       '''simple docstring'''


       pass






      def SCREAMING_SNAKE_CASE_	(		self	:	str							):






       '''simple docstring'''


       __lowercase									=	self.image_processing_class(**self.image_processor_dict							)
       # create random PIL images
       __lowercase									=	self.image_processor_tester.prepare_inputs(equal_resolution=A_							)
       for image in image_inputs:
        self.assertIsInstance(A_      ,       Image.Image							)

       # Test not batched input
       __lowercase									=	image_processing(image_inputs[0]      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               1,
               self.expected_encoded_image_num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )

       # Test batched
       __lowercase									=	image_processing(A_      ,       return_tensors="""pt"""							).pixel_values
       self.assertEqual(
           encoded_images.shape      ,       (
               self.image_processor_tester.batch_size,
               self.expected_encoded_image_num_channels,
               self.image_processor_tester.crop_size["""height"""],
               self.image_processor_tester.crop_size["""width"""],
           )      ,       )
