
'''simple docstring'''







# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce



def 			_a		(	_SCREAMING_SNAKE_CASE		:   int  ):
						return (torch.arange(state.num_processes  ) + 1.0 + (state.num_processes * state.process_index)).to(state.device  )



def 			_a		(	_SCREAMING_SNAKE_CASE		:   Optional[Any]  ):
						_SCREAMING_SNAKE_CASE				= create_tensor(_SCREAMING_SNAKE_CASE  )
						_SCREAMING_SNAKE_CASE				= gather(_SCREAMING_SNAKE_CASE  )
						assert gathered_tensor.tolist() == list(range(1	,			state.num_processes**2 + 1  )  )



def 			_a		(	_SCREAMING_SNAKE_CASE		:   List[str]  ):
						_SCREAMING_SNAKE_CASE				= [state.process_index]
						_SCREAMING_SNAKE_CASE				= gather_object(_SCREAMING_SNAKE_CASE  )
						assert len(_SCREAMING_SNAKE_CASE  ) == state.num_processes, F'{gathered_obj}, {len(_SCREAMING_SNAKE_CASE  )} != {state.num_processes}'
						assert gathered_obj == list(range(state.num_processes  )  ), F'{gathered_obj} != {list(range(state.num_processes  )  )}'



def 			_a		(	_SCREAMING_SNAKE_CASE		:   List[str]  ):
						_SCREAMING_SNAKE_CASE				= create_tensor(_SCREAMING_SNAKE_CASE  )
						_SCREAMING_SNAKE_CASE				= broadcast(_SCREAMING_SNAKE_CASE  )
						assert broadcasted_tensor.shape == torch.Size([state.num_processes]  )
						assert broadcasted_tensor.tolist() == list(range(1	,			state.num_processes + 1  )  )



def 			_a		(	_SCREAMING_SNAKE_CASE		:   Any  ):
						# We need to pad the tensor with one more element if we are the main process
						# to ensure that we can pad
						if state.is_main_process:
												_SCREAMING_SNAKE_CASE				= torch.arange(state.num_processes + 1  ).to(state.device  )
						else:
												_SCREAMING_SNAKE_CASE				= torch.arange(state.num_processes  ).to(state.device  )
						_SCREAMING_SNAKE_CASE				= pad_across_processes(_SCREAMING_SNAKE_CASE  )
						assert padded_tensor.shape == torch.Size([state.num_processes + 1]  )
						if not state.is_main_process:
												assert padded_tensor.tolist() == list(range(0	,			state.num_processes  )  ) + [0]



def 			_a		(	_SCREAMING_SNAKE_CASE		:   Union[str, Any]  ):
						# For now runs on only two processes
						if state.num_processes != 2:
												return
						_SCREAMING_SNAKE_CASE				= create_tensor(_SCREAMING_SNAKE_CASE  )
						_SCREAMING_SNAKE_CASE				= reduce(_SCREAMING_SNAKE_CASE	,			"sum"  )
						_SCREAMING_SNAKE_CASE				= torch.tensor([4.0, 6]  ).to(state.device  )
						assert torch.allclose(_SCREAMING_SNAKE_CASE	,			_SCREAMING_SNAKE_CASE  ), F'{reduced_tensor} != {truth_tensor}'



def 			_a		(	_SCREAMING_SNAKE_CASE		:   Any  ):
						# For now runs on only two processes
						if state.num_processes != 2:
												return
						_SCREAMING_SNAKE_CASE				= create_tensor(_SCREAMING_SNAKE_CASE  )
						_SCREAMING_SNAKE_CASE				= reduce(_SCREAMING_SNAKE_CASE	,			"mean"  )
						_SCREAMING_SNAKE_CASE				= torch.tensor([2.0, 3]  ).to(state.device  )
						assert torch.allclose(_SCREAMING_SNAKE_CASE	,			_SCREAMING_SNAKE_CASE  ), F'{reduced_tensor} != {truth_tensor}'



def 			_a		(	_SCREAMING_SNAKE_CASE		:   int  ):
						# For xla_spawn (TPUs)
						main()



def 			_a		(	):
						_SCREAMING_SNAKE_CASE				= PartialState()
						state.print(F'State: {state}'  )
						state.print("testing gather"  )
						test_gather(_SCREAMING_SNAKE_CASE  )
						state.print("testing gather_object"  )
						test_gather_object(_SCREAMING_SNAKE_CASE  )
						state.print("testing broadcast"  )
						test_broadcast(_SCREAMING_SNAKE_CASE  )
						state.print("testing pad_across_processes"  )
						test_pad_across_processes(_SCREAMING_SNAKE_CASE  )
						state.print("testing reduce_sum"  )
						test_reduce_sum(_SCREAMING_SNAKE_CASE  )
						state.print("testing reduce_mean"  )
						test_reduce_mean(_SCREAMING_SNAKE_CASE  )


if __name__ == "__main__":
						main()

'''simple docstring'''







import os
import unittest

from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 			lowerCAmelCase (				__UpperCAmelCase		,     unittest.TestCase			):
		a							:					int       =					LayoutLMTokenizer
		a							:					Optional[int]       =					LayoutLMTokenizerFast
		a							:					Optional[int]       =					True
		a							:					Any       =					True




		def     lowercase       (				self      ):
								super().setUp()

								_SCREAMING_SNAKE_CASE				= [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								_SCREAMING_SNAKE_CASE				= os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]      )
								with open(self.vocab_file					,						"w"					,						encoding="utf-8"      ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )




		def     lowercase       (				self					,						**UpperCamelCase      ):
								return LayoutLMTokenizer.from_pretrained(self.tmpdirname					,						**UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= "UNwant\u00E9d,running"
								_SCREAMING_SNAKE_CASE				= "unwanted, running"
								return input_text, output_text




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= self.tokenizer_class(self.vocab_file      )

								_SCREAMING_SNAKE_CASE				= tokenizer.tokenize("UNwant\u00E9d,running"      )
								self.assertListEqual(UpperCamelCase					,						["un", "##want", "##ed", ",", "runn", "##ing"]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase      )					,						[7, 4, 5, 10, 8, 9]      )




		def     lowercase       (				self      ):
								pass