



"""simple docstring"""

import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[int]			):  # picklable for multiprocessing



					"""simple docstring"""





					return x.sum()







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Optional[Any]			):  # picklable for multiprocessing



					"""simple docstring"""





					return i + 1







@dataclass
class    __lowercase     :







						"""simple docstring"""







						_A		:							int
						_A		:							str







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					{}
											snake_case_					:		Any 					=					[]
											snake_case_					:		Any 					=					1
											snake_case_					:		List[str] 					=					[1, 2]
											snake_case_					:		Dict 					=					{"""a""": 1, """b""": 2}
											snake_case_					:		Tuple 					=					{"""a""": [1, 2], """b""": [3, 4]}
											snake_case_					:		Optional[Any] 					=					{"""a""": {"""1""": 1}, """b""": 2}
											snake_case_					:		List[str] 					=					{"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
											snake_case_					:		List[str] 					=					{}
											snake_case_					:		List[Any] 					=					[]
											snake_case_					:		Optional[int] 					=					2
											snake_case_					:		List[Any] 					=					[2, 3]
											snake_case_					:		List[str] 					=					{"""a""": 2, """b""": 3}
											snake_case_					:		Tuple 					=					{"""a""": [2, 3], """b""": [4, 5]}
											snake_case_					:		Optional[Any] 					=					{"""a""": {"""1""": 2}, """b""": 3}
											snake_case_					:		Optional[int] 					=					{"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__       )   ,		lowercase__       )

											snake_case_					:		Optional[Any] 					=					2
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )

											snake_case_					:		List[str] 					=					{"""a""": np.eye(2       ), """b""": np.zeros(3       ), """c""": np.ones(2       )}
											snake_case_					:		List[str] 					=					{"""a""": 2, """b""": 0, """c""": 2}
											snake_case_					:		Any 					=					{
											    """a""": np.eye(2       ).astype(lowercase__       ),
											    """b""": np.zeros(3       ).astype(lowercase__       ),
											    """c""": np.ones(2       ).astype(lowercase__       ),
											}
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		map_numpy=lowercase__       )   ,		lowercase__       )
											self.assertEqual(
											    {k: v.tolist() for k, v in map_nested(lowercase__   ,		lowercase__   ,		map_numpy=lowercase__       ).items()}   ,		{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}   ,		)
											self.assertEqual(map_nested(lowercase__   ,		lowercase__   ,		map_numpy=lowercase__   ,		num_proc=lowercase__       )   ,		lowercase__       )
											self.assertEqual(
											    {k: v.tolist() for k, v in map_nested(lowercase__   ,		lowercase__   ,		map_numpy=lowercase__   ,		num_proc=lowercase__       ).items()}   ,		{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}   ,		)
											with self.assertRaises(lowercase__       ):  # can't pickle a local lambda
																map_nested(lambda lowercase__       : x + 1   ,		lowercase__   ,		num_proc=lowercase__       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[str] 					=					{"""a""": 1, """b""": 2}
											snake_case_					:		Dict 					=					{"""a""": 3, """b""": 4}
											snake_case_					:		Dict 					=					{"""a""": 5, """b""": 6}
											snake_case_					:		List[Any] 					=					sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))]       )
											self.assertEqual(sorted(zip_dict(lowercase__   ,		lowercase__   ,		lowercase__       )       )   ,		lowercase__       )







						def 						__UpperCamelCase							(self       ):







											class    __lowercase     :







																	"""simple docstring"""







																	_A		:							str							       =		"""bar"""
											snake_case_					:		Dict 					=					Foo()
											self.assertEqual(foo.my_attr   ,		"""bar"""       )
											with temporary_assignment(lowercase__   ,		"""my_attr"""   ,		"""BAR"""       ):
																self.assertEqual(foo.my_attr   ,		"""BAR"""       )
											self.assertEqual(foo.my_attr   ,		"""bar"""       )







@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc"""				,    [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (1_6, 1_6, 1_6),
        (1_6, 1_7, 1_6),
        (1_7, 1_6, 1_6),
    ]				,    )
def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Tuple				,    SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  Optional[Any]			):



					"""simple docstring"""





					with patch("""datasets.utils.py_utils._single_map_nested"""			) as mock_single_map_nested, patch(
					    """datasets.parallel.parallel.Pool"""			) as mock_multiprocessing_pool:
										snake_case_					:		Optional[Any] 					=					{f'{i}': i for i in range(SCREAMING_SNAKE_CASE__			)}
										snake_case_					:		Union[str, Any] 					=					map_nested(lambda SCREAMING_SNAKE_CASE__			: x + 1_0				,    SCREAMING_SNAKE_CASE__				,    num_proc=SCREAMING_SNAKE_CASE__				,    parallel_min_length=1_6			)
										if expected_num_proc == 1:
															assert mock_single_map_nested.called
															assert not mock_multiprocessing_pool.called
										else:
															assert not mock_single_map_nested.called
															assert mock_multiprocessing_pool.called
															assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						@require_tf
						def 						__UpperCamelCase							(self       ):
											import tensorflow as tf
											from tensorflow.keras import layers

											snake_case_					:		Tuple 					=					layers.Dense(2       )

											def gen_random_output():
																snake_case_					:		List[Any] 					=					tf.random.uniform((1, 3)       )
																return model(lowercase__       ).numpy()

											with temp_seed(42   ,		set_tensorflow=lowercase__       ):
																snake_case_					:		Tuple 					=					gen_random_output()
											with temp_seed(42   ,		set_tensorflow=lowercase__       ):
																snake_case_					:		str 					=					gen_random_output()
											snake_case_					:		Optional[int] 					=					gen_random_output()

											np.testing.assert_equal(lowercase__   ,		lowercase__       )
											self.assertGreater(np.abs(outa - outa       ).sum()   ,		0       )







						@require_torch
						def 						__UpperCamelCase							(self       ):
											import torch

											def gen_random_output():
																snake_case_					:		Any 					=					torch.nn.Linear(3   ,		2       )
																snake_case_					:		Optional[int] 					=					torch.rand(1   ,		3       )
																return model(lowercase__       ).detach().numpy()

											with temp_seed(42   ,		set_pytorch=lowercase__       ):
																snake_case_					:		Optional[Any] 					=					gen_random_output()
											with temp_seed(42   ,		set_pytorch=lowercase__       ):
																snake_case_					:		Optional[Any] 					=					gen_random_output()
											snake_case_					:		Tuple 					=					gen_random_output()

											np.testing.assert_equal(lowercase__   ,		lowercase__       )
											self.assertGreater(np.abs(outa - outa       ).sum()   ,		0       )







						def 						__UpperCamelCase							(self       ):
											def gen_random_output():
																return np.random.rand(1   ,		3       )

											with temp_seed(42       ):
																snake_case_					:		List[Any] 					=					gen_random_output()
											with temp_seed(42       ):
																snake_case_					:		Union[str, Any] 					=					gen_random_output()
											snake_case_					:		Optional[Any] 					=					gen_random_output()

											np.testing.assert_equal(lowercase__   ,		lowercase__       )
											self.assertGreater(np.abs(outa - outa       ).sum()   ,		0       )







@pytest.mark.parametrize("""input_data"""				,    [{}]			)
def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  str			):



					"""simple docstring"""





					snake_case_					:		Union[str, Any] 					=					NestedDataStructure(SCREAMING_SNAKE_CASE__			).data
					assert output_data == input_data







@pytest.mark.parametrize(
    """data, expected_output"""				,    [
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ]				,    )
def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Tuple				,    SCREAMING_SNAKE_CASE__ :  List[Any]			):



					"""simple docstring"""





					snake_case_					:		Dict 					=					NestedDataStructure(SCREAMING_SNAKE_CASE__			).flatten()
					assert output == expected_output







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					snake_case_					:		Union[str, Any] 					=					A(x=1				,    y="""foobar"""			)
					snake_case_					:		List[Any] 					=					{"""x""": 1, """y""": """foobar"""}
					assert asdict(SCREAMING_SNAKE_CASE__			) == expected_output

					snake_case_					:		Tuple 					=					{"""a""": {"""b""": A(x=1_0				,    y="""foo"""			)}, """c""": [A(x=2_0				,    y="""bar"""			)]}
					snake_case_					:		Union[str, Any] 					=					{"""a""": {"""b""": {"""x""": 1_0, """y""": """foo"""}}, """c""": [{"""x""": 2_0, """y""": """bar"""}]}
					assert asdict(SCREAMING_SNAKE_CASE__			) == expected_output

					with pytest.raises(SCREAMING_SNAKE_CASE__			):
										asdict([1, A(x=1_0				,    y="""foo"""			)]			)







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  str			):



					"""simple docstring"""





					return text.split()







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Dict			):



					"""simple docstring"""





					yield (time.time(), content)
					time.sleep(2			)
					yield (time.time(), content)







def 							SCREAMING_SNAKE_CASE__      (		):



					"""simple docstring"""





					with Pool(2			) as pool:
										snake_case_					:		Optional[int] 					=					list(iflatmap_unordered(SCREAMING_SNAKE_CASE__				,    _split_text				,    kwargs_iterable=[{"""text""": """hello there"""}] * 1_0			)			)
										assert out.count("""hello"""			) == 1_0
										assert out.count("""there"""			) == 1_0
										assert len(SCREAMING_SNAKE_CASE__			) == 2_0

					# check multiprocess from pathos (uses dill for pickling)
					with multiprocess.Pool(2			) as pool:
										snake_case_					:		Union[str, Any] 					=					list(iflatmap_unordered(SCREAMING_SNAKE_CASE__				,    _split_text				,    kwargs_iterable=[{"""text""": """hello there"""}] * 1_0			)			)
										assert out.count("""hello"""			) == 1_0
										assert out.count("""there"""			) == 1_0
										assert len(SCREAMING_SNAKE_CASE__			) == 2_0

					# check that we get items as fast as possible
					with Pool(2			) as pool:
										snake_case_					:		Optional[int] 					=					[]
										for yield_time, content in iflatmap_unordered(
										    SCREAMING_SNAKE_CASE__				,    _aseconds_generator_of_aitems_with_timing				,    kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]			):
															assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
															out.append(SCREAMING_SNAKE_CASE__			)
										assert out.count("""a"""			) == 2
										assert out.count("""b"""			) == 2
										assert len(SCREAMING_SNAKE_CASE__			) == 4









"""simple docstring"""

from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
			from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


a_			   =					logging.get_logger(__name__)

a_			   =					{
    '''microsoft/deberta-v2-xlarge''': '''https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json''',
    '''microsoft/deberta-v2-xxlarge''': '''https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json''',
    '''microsoft/deberta-v2-xlarge-mnli''': (
        '''https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json'''
    ),
    '''microsoft/deberta-v2-xxlarge-mnli''': (
        '''https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json'''
    ),
}







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							Dict							       =		"""deberta-v2"""







						def __init__(self   ,		lowercase__=12_81_00   ,		lowercase__=15_36   ,		lowercase__=24   ,		lowercase__=24   ,		lowercase__=61_44   ,		lowercase__="gelu"   ,		lowercase__=0.1   ,		lowercase__=0.1   ,		lowercase__=5_12   ,		lowercase__=0   ,		lowercase__=0.02   ,		lowercase__=1e-7   ,		lowercase__=False   ,		lowercase__=-1   ,		lowercase__=0   ,		lowercase__=True   ,		lowercase__=None   ,		lowercase__=0   ,		lowercase__="gelu"   ,		**lowercase__   ,		):
											super().__init__(**lowercase__       )

											snake_case_					:		Union[str, Any] 					=					hidden_size
											snake_case_					:		str 					=					num_hidden_layers
											snake_case_					:		Tuple 					=					num_attention_heads
											snake_case_					:		Dict 					=					intermediate_size
											snake_case_					:		Optional[int] 					=					hidden_act
											snake_case_					:		Union[str, Any] 					=					hidden_dropout_prob
											snake_case_					:		Any 					=					attention_probs_dropout_prob
											snake_case_					:		List[Any] 					=					max_position_embeddings
											snake_case_					:		Union[str, Any] 					=					type_vocab_size
											snake_case_					:		Union[str, Any] 					=					initializer_range
											snake_case_					:		List[Any] 					=					relative_attention
											snake_case_					:		Dict 					=					max_relative_positions
											snake_case_					:		Optional[int] 					=					pad_token_id
											snake_case_					:		List[str] 					=					position_biased_input

											# Backwards compatibility
											if type(lowercase__       ) == str:
																snake_case_					:		Union[str, Any] 					=					[x.strip() for x in pos_att_type.lower().split("""|"""       )]

											snake_case_					:		Optional[int] 					=					pos_att_type
											snake_case_					:		List[str] 					=					vocab_size
											snake_case_					:		Tuple 					=					layer_norm_eps

											snake_case_					:		List[Any] 					=					kwargs.get("""pooler_hidden_size"""   ,		lowercase__       )
											snake_case_					:		List[str] 					=					pooler_dropout
											snake_case_					:		int 					=					pooler_hidden_act







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						@property
						def 						__UpperCamelCase							(self       ):
											if self.task == "multiple-choice":
																snake_case_					:		List[Any] 					=					{0: """batch""", 1: """choice""", 2: """sequence"""}
											else:
																snake_case_					:		int 					=					{0: """batch""", 1: """sequence"""}
											if self._config.type_vocab_size > 0:
																return OrderedDict(
																    [("""input_ids""", dynamic_axis), ("""attention_mask""", dynamic_axis), ("""token_type_ids""", dynamic_axis)]       )
											else:
																return OrderedDict([("""input_ids""", dynamic_axis), ("""attention_mask""", dynamic_axis)]       )







						@property
						def 						__UpperCamelCase							(self       ):
											return 12







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__ = -1   ,		lowercase__ = -1   ,		lowercase__ = -1   ,		lowercase__ = False   ,		lowercase__ = None   ,		lowercase__ = 3   ,		lowercase__ = 40   ,		lowercase__ = 40   ,		lowercase__ = None   ,		):
											snake_case_					:		str 					=					super().generate_dummy_inputs(preprocessor=lowercase__   ,		framework=lowercase__       )
											if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
																del dummy_inputs["token_type_ids"]
											return dummy_inputs




