



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			return (torch.arange(state.num_processes    ) + 1.0 + (state.num_processes * state.process_index)).to(state.device    )




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			A							:					List[str]              =						create_tensor(_lowerCamelCase    )
			A							:					Any              =						gather(_lowerCamelCase    )
			assert gathered_tensor.tolist() == list(range(1 ,    state.num_processes**2 + 1    )    )




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			A							:					List[str]              =						[state.process_index]
			A							:					Optional[int]              =						gather_object(_lowerCamelCase    )
			assert len(_lowerCamelCase    ) == state.num_processes, f"""{gathered_obj}, {len(_lowerCamelCase    )} != {state.num_processes}"""
			assert gathered_obj == list(range(state.num_processes    )    ), f"""{gathered_obj} != {list(range(state.num_processes    )    )}"""




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			A							:					List[Any]              =						create_tensor(_lowerCamelCase    )
			A							:					int              =						broadcast(_lowerCamelCase    )
			assert broadcasted_tensor.shape == torch.Size([state.num_processes]    )
			assert broadcasted_tensor.tolist() == list(range(1 ,    state.num_processes + 1    )    )




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			# We need to pad the tensor with one more element if we are the main process
			# to ensure that we can pad
			if state.is_main_process:
						A							:					Dict              =						torch.arange(state.num_processes + 1    ).to(state.device    )
			else:
						A							:					Dict              =						torch.arange(state.num_processes    ).to(state.device    )
			A							:					int              =						pad_across_processes(_lowerCamelCase    )
			assert padded_tensor.shape == torch.Size([state.num_processes + 1]    )
			if not state.is_main_process:
						assert padded_tensor.tolist() == list(range(0 ,    state.num_processes    )    ) + [0]




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			# For now runs on only two processes
			if state.num_processes != 2:
						return
			A							:					Any              =						create_tensor(_lowerCamelCase    )
			A							:					int              =						reduce(_lowerCamelCase ,    "sum"    )
			A							:					str              =						torch.tensor([4.0, 6]    ).to(state.device    )
			assert torch.allclose(_lowerCamelCase ,    _lowerCamelCase    ), f"""{reduced_tensor} != {truth_tensor}"""




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			# For now runs on only two processes
			if state.num_processes != 2:
						return
			A							:					Any              =						create_tensor(_lowerCamelCase    )
			A							:					Any              =						reduce(_lowerCamelCase ,    "mean"    )
			A							:					Union[str, Any]              =						torch.tensor([2.0, 3]    ).to(state.device    )
			assert torch.allclose(_lowerCamelCase ,    _lowerCamelCase    ), f"""{reduced_tensor} != {truth_tensor}"""




def 	UpperCAmelCase   (						_lowerCamelCase    ):
			# For xla_spawn (TPUs)
			main()




def 	UpperCAmelCase   (						):
			A							:					List[Any]              =						PartialState()
			state.print(f"""State: {state}"""    )
			state.print("testing gather"    )
			test_gather(_lowerCamelCase    )
			state.print("testing gather_object"    )
			test_gather_object(_lowerCamelCase    )
			state.print("testing broadcast"    )
			test_broadcast(_lowerCamelCase    )
			state.print("testing pad_across_processes"    )
			test_pad_across_processes(_lowerCamelCase    )
			state.print("testing reduce_sum"    )
			test_reduce_sum(_lowerCamelCase    )
			state.print("testing reduce_mean"    )
			test_reduce_mean(_lowerCamelCase    )


if __name__ == "__main__":
							main()




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
							import torch

							from transformers import XLMRobertaModel

@require_sentencepiece
@require_tokenizers
@require_torch
class 	lowerCamelCase_    ( unittest.TestCase						):






						'''simple docstring'''







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Any )      ->				Union[str, Any]:
									A							:					Union[str, Any]              =						XLMRobertaModel.from_pretrained("xlm-roberta-base" )
									A							:					Tuple              =						torch.tensor([[0, 5_81, 1_02_69, 83, 9_99_42, 1_36, 6_07_42, 23, 70, 8_05_83, 1_82_76, 2]] )
									# The dog is cute and lives in the garden house

									A							:					Tuple              =						torch.Size((1, 12, 7_68) )  # batch_size, sequence_length, embedding_vector_dim
									A							:					List[str]              =						torch.tensor(
									    [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]] )
									#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
									#  xlmr.eval()
									#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
									with torch.no_grad():
												A							:					Tuple              =						model(__lowerCamelCase )["last_hidden_state"].detach()
									self.assertEqual(output.shape	,	__lowerCamelCase )
									# compare the actual values for a slice of last dim
									self.assertTrue(torch.allclose(output[:, :, -1]	,	__lowerCamelCase	,	atol=1e-3 ) )







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int] )      ->				int:
									A							:					str              =						XLMRobertaModel.from_pretrained("xlm-roberta-large" )
									A							:					List[Any]              =						torch.tensor([[0, 5_81, 1_02_69, 83, 9_99_42, 1_36, 6_07_42, 23, 70, 8_05_83, 1_82_76, 2]] )
									# The dog is cute and lives in the garden house

									A							:					Optional[Any]              =						torch.Size((1, 12, 10_24) )  # batch_size, sequence_length, embedding_vector_dim
									A							:					List[Any]              =						torch.tensor(
									    [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]] )
									#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
									#  xlmr.eval()
									#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
									with torch.no_grad():
												A							:					Optional[int]              =						model(__lowerCamelCase )["last_hidden_state"].detach()
									self.assertEqual(output.shape	,	__lowerCamelCase )
									# compare the actual values for a slice of last dim
									self.assertTrue(torch.allclose(output[:, :, -1]	,	__lowerCamelCase	,	atol=1e-3 ) )