





import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    MODEL_FOR_PRETRAINING_MAPPING,
	    MegatronBertForCausalLM,
	    MegatronBertForMaskedLM,
	    MegatronBertForMultipleChoice,
	    MegatronBertForNextSentencePrediction,
	    MegatronBertForPreTraining,
	    MegatronBertForQuestionAnswering,
	    MegatronBertForSequenceClassification,
	    MegatronBertForTokenClassification,
	    MegatronBertModel,
	)




class   UpperCamelCase_       :


							def __init__(							self    :List[Any]							,				__A    :List[Any]							,				__A    :Optional[Any]=13							,				__A    :Optional[int]=7							,				__A    :Tuple=True							,				__A    :Union[str, Any]=True							,				__A    :List[Any]=True							,				__A    :str=True							,				__A    :Tuple=99							,				__A    :Union[str, Any]=64							,				__A    :Tuple=32							,				__A    :Optional[int]=5							,				__A    :int=4							,				__A    :Optional[int]=37							,				__A    :str="gelu"							,				__A    :List[str]=0.1							,				__A    :str=0.1							,				__A    :List[Any]=512							,				__A    :str=16							,				__A    :int=2							,				__A    :List[str]=0.0_2							,				__A    :List[Any]=3							,				__A    :Optional[int]=4							,				__A    :Tuple=None							,				)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      parent
									SCREAMING_SNAKE_CASE__			=      batch_size
									SCREAMING_SNAKE_CASE__			=      seq_length
									SCREAMING_SNAKE_CASE__			=      is_training
									SCREAMING_SNAKE_CASE__			=      use_input_mask
									SCREAMING_SNAKE_CASE__			=      use_token_type_ids
									SCREAMING_SNAKE_CASE__			=      use_labels
									SCREAMING_SNAKE_CASE__			=      vocab_size
									SCREAMING_SNAKE_CASE__			=      hidden_size
									SCREAMING_SNAKE_CASE__			=      embedding_size
									SCREAMING_SNAKE_CASE__			=      num_hidden_layers
									SCREAMING_SNAKE_CASE__			=      num_attention_heads
									SCREAMING_SNAKE_CASE__			=      intermediate_size
									SCREAMING_SNAKE_CASE__			=      hidden_act
									SCREAMING_SNAKE_CASE__			=      hidden_dropout_prob
									SCREAMING_SNAKE_CASE__			=      attention_probs_dropout_prob
									SCREAMING_SNAKE_CASE__			=      max_position_embeddings
									SCREAMING_SNAKE_CASE__			=      type_vocab_size
									SCREAMING_SNAKE_CASE__			=      type_sequence_label_size
									SCREAMING_SNAKE_CASE__			=      initializer_range
									SCREAMING_SNAKE_CASE__			=      num_labels
									SCREAMING_SNAKE_CASE__			=      num_choices
									SCREAMING_SNAKE_CASE__			=      scope


							def      _snake_case  (							self    :int		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size, self.seq_length]							,				self.vocab_size		)

									SCREAMING_SNAKE_CASE__			=      None
									if self.use_input_mask:
											SCREAMING_SNAKE_CASE__			=      random_attention_mask([self.batch_size, self.seq_length]		)

									SCREAMING_SNAKE_CASE__			=      None
									if self.use_token_type_ids:
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size, self.seq_length]							,				self.type_vocab_size		)

									SCREAMING_SNAKE_CASE__			=      None
									SCREAMING_SNAKE_CASE__			=      None
									SCREAMING_SNAKE_CASE__			=      None
									if self.use_labels:
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size]							,				self.type_sequence_label_size		)
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size, self.seq_length]							,				self.num_labels		)
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size]							,				self.num_choices		)

									SCREAMING_SNAKE_CASE__			=      self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def      _snake_case  (							self    :int		)   ->   List[str]:



									"""simple docstring"""



									return MegatronBertConfig(
									    vocab_size=self.vocab_size							,				hidden_size=self.hidden_size							,				num_hidden_layers=self.num_hidden_layers							,				num_attention_heads=self.num_attention_heads							,				intermediate_size=self.intermediate_size							,				embedding_size=self.embedding_size							,				hidden_act=self.hidden_act							,				hidden_dropout_prob=self.hidden_dropout_prob							,				attention_probs_dropout_prob=self.attention_probs_dropout_prob							,				max_position_embeddings=self.max_position_embeddings							,				type_vocab_size=self.type_vocab_size							,				is_decoder=__A							,				initializer_range=self.initializer_range							,				)


							def      _snake_case  (							self    :List[str]							,				__A    :Optional[int]							,				__A    :List[Any]							,				__A    :List[Any]							,				__A    :Any							,				__A    :List[Any]							,				__A    :Any							,				__A    :Union[str, Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertModel(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				token_type_ids=__A		)
									SCREAMING_SNAKE_CASE__			=      model(__A							,				token_type_ids=__A		)
									SCREAMING_SNAKE_CASE__			=      model(__A		)

									self.parent.assertEqual(result.last_hidden_state.shape							,				(self.batch_size, self.seq_length, self.hidden_size)		)
									self.parent.assertEqual(result.pooler_output.shape							,				(self.batch_size, self.hidden_size)		)


							def      _snake_case  (							self    :Any							,				__A    :Union[str, Any]							,				__A    :List[str]							,				__A    :Union[str, Any]							,				__A    :Any							,				__A    :Tuple							,				__A    :int							,				__A    :int		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertForMaskedLM(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A		)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.seq_length, self.vocab_size)		)


							def      _snake_case  (							self    :str							,				__A    :Tuple							,				__A    :List[Any]							,				__A    :Any							,				__A    :Optional[Any]							,				__A    :int							,				__A    :Union[str, Any]							,				__A    :Union[str, Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertForCausalLM(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A		)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.seq_length, self.vocab_size)		)


							def      _snake_case  (							self    :Optional[int]							,				__A    :Optional[Any]							,				__A    :Dict							,				__A    :List[str]							,				__A    :List[str]							,				__A    :List[Any]							,				__A    :Optional[int]							,				__A    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertForNextSentencePrediction(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A							,				)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, 2)		)


							def      _snake_case  (							self    :Optional[int]							,				__A    :Optional[Any]							,				__A    :str							,				__A    :int							,				__A    :List[str]							,				__A    :Optional[Any]							,				__A    :List[Any]							,				__A    :List[str]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertForPreTraining(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A							,				next_sentence_label=__A							,				)
									self.parent.assertEqual(result.prediction_logits.shape							,				(self.batch_size, self.seq_length, self.vocab_size)		)
									self.parent.assertEqual(result.seq_relationship_logits.shape							,				(self.batch_size, 2)		)


							def      _snake_case  (							self    :Dict							,				__A    :List[Any]							,				__A    :int							,				__A    :Any							,				__A    :Optional[int]							,				__A    :int							,				__A    :Union[str, Any]							,				__A    :int		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertForQuestionAnswering(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				token_type_ids=__A							,				start_positions=__A							,				end_positions=__A							,				)
									self.parent.assertEqual(result.start_logits.shape							,				(self.batch_size, self.seq_length)		)
									self.parent.assertEqual(result.end_logits.shape							,				(self.batch_size, self.seq_length)		)


							def      _snake_case  (							self    :str							,				__A    :Union[str, Any]							,				__A    :Any							,				__A    :Optional[Any]							,				__A    :Union[str, Any]							,				__A    :str							,				__A    :int							,				__A    :Union[str, Any]		)   ->   Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.num_labels
									SCREAMING_SNAKE_CASE__			=      MegatronBertForSequenceClassification(__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A		)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.num_labels)		)


							def      _snake_case  (							self    :List[Any]							,				__A    :Tuple							,				__A    :Union[str, Any]							,				__A    :Optional[Any]							,				__A    :List[Any]							,				__A    :Union[str, Any]							,				__A    :List[str]							,				__A    :List[str]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.num_labels
									SCREAMING_SNAKE_CASE__			=      MegatronBertForTokenClassification(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A		)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.seq_length, self.num_labels)		)


							def      _snake_case  (							self    :Dict							,				__A    :str							,				__A    :List[str]							,				__A    :Dict							,				__A    :Union[str, Any]							,				__A    :List[str]							,				__A    :List[str]							,				__A    :Dict		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.num_choices
									SCREAMING_SNAKE_CASE__			=      MegatronBertForMultipleChoice(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      input_ids.unsqueeze(1		).expand(-1							,				self.num_choices							,				-1		).contiguous()
									SCREAMING_SNAKE_CASE__			=      token_type_ids.unsqueeze(1		).expand(-1							,				self.num_choices							,				-1		).contiguous()
									SCREAMING_SNAKE_CASE__			=      input_mask.unsqueeze(1		).expand(-1							,				self.num_choices							,				-1		).contiguous()
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				token_type_ids=__A							,				labels=__A							,				)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.num_choices)		)


							def      _snake_case  (							self    :Dict		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.prepare_config_and_inputs()
									(
									    (
									    SCREAMING_SNAKE_CASE__
									)	,						(
									    SCREAMING_SNAKE_CASE__
									)	,						(
									    SCREAMING_SNAKE_CASE__
									)	,						(
									    SCREAMING_SNAKE_CASE__
									)	,						(
									    SCREAMING_SNAKE_CASE__
									)	,						(
									    SCREAMING_SNAKE_CASE__
									)	,						(
									    SCREAMING_SNAKE_CASE__
									)	,						
									)			=      config_and_inputs
									SCREAMING_SNAKE_CASE__			=      {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
									return config, inputs_dict




@require_torch
class   UpperCamelCase_       (     UpperCamelCase__				,  UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					(
							    (
							        MegatronBertModel,
							        MegatronBertForMaskedLM,
							        MegatronBertForCausalLM,
							        MegatronBertForMultipleChoice,
							        MegatronBertForNextSentencePrediction,
							        MegatronBertForPreTraining,
							        MegatronBertForQuestionAnswering,
							        MegatronBertForSequenceClassification,
							        MegatronBertForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase_           =					(
							    {
							        "feature-extraction": MegatronBertModel,
							        "fill-mask": MegatronBertForMaskedLM,
							        "question-answering": MegatronBertForQuestionAnswering,
							        "text-classification": MegatronBertForSequenceClassification,
							        "text-generation": MegatronBertForCausalLM,
							        "token-classification": MegatronBertForTokenClassification,
							        "zero-shot": MegatronBertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase_           =					True
							# test_resize_embeddings = False
							lowerCamelCase_           =					False


							def      _snake_case  (							self    :Optional[Any]							,				__A    :Union[str, Any]							,				__A    :List[Any]							,				__A    :Optional[int]=False		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      super()._prepare_for_class(__A							,				__A							,				return_labels=__A		)

									if return_labels:
											if model_class in get_values(__A		):
													SCREAMING_SNAKE_CASE__			=      torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)							,				dtype=torch.long							,				device=__A		)
													SCREAMING_SNAKE_CASE__			=      torch.zeros(
													    self.model_tester.batch_size							,				dtype=torch.long							,				device=__A		)
									return inputs_dict


							def      _snake_case  (							self    :Optional[int]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MegatronBertModelTester(self		)
									SCREAMING_SNAKE_CASE__			=      ConfigTester(self							,				config_class=__A							,				hidden_size=37		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   Dict:



									"""simple docstring"""



									self.config_tester.run_common_tests()


							def      _snake_case  (							self    :Tuple		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_model(*__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__A		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__A		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_pretraining(*__A		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_question_answering(*__A		)


							def      _snake_case  (							self    :Optional[int]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__A		)


							def      _snake_case  (							self    :Dict		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_megatron_bert_for_token_classification(*__A		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Tuple    ):
		return torch.tensor(
		    UpperCamelCase__					,   dtype=torch.long					,   device=UpperCamelCase__					,   )


_lowerCamelCase   				=			1e-4




@require_torch
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase_       (     unittest.TestCase    ):


							@slow
							@unittest.skip("""Model is not available."""		)
							def      _snake_case  (							self    :str		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      """nvidia/megatron-bert-uncased-345m"""
									if "MYDIR" in os.environ:
											SCREAMING_SNAKE_CASE__			=      os.path.join(os.environ["""MYDIR"""]							,				__A		)
									SCREAMING_SNAKE_CASE__			=      MegatronBertModel.from_pretrained(__A		)
									model.to(__A		)
									model.half()
									SCREAMING_SNAKE_CASE__			=      _long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]]		)
									with torch.no_grad():
											SCREAMING_SNAKE_CASE__			=      model(__A		)[0]
									SCREAMING_SNAKE_CASE__			=      torch.Size((1, 9, 1024)		)
									self.assertEqual(output.shape							,				__A		)

									SCREAMING_SNAKE_CASE__			=      [-0.6_0_4_0, -0.2_5_1_7, -0.1_0_2_5, 0.3_4_2_0, -0.6_7_5_8, -0.0_0_1_7, -0.1_0_8_9, -0.1_9_9_0, 0.5_7_2_8]
									for ii in range(3		):
											for jj in range(3		):
													SCREAMING_SNAKE_CASE__			=      output[0, ii, jj]
													SCREAMING_SNAKE_CASE__			=      expected[3 * ii + jj]
													SCREAMING_SNAKE_CASE__			=      """ii={} jj={} a={} b={}""".format(__A							,				__A							,				__A							,				__A		)
													self.assertTrue(math.isclose(__A							,				__A							,				rel_tol=__A							,				abs_tol=__A		)							,				msg=__A		)






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import Accelerator
from accelerate.test_utils import execute_subprocess_async, require_multi_gpu
from accelerate.utils import patch_environment




class   UpperCamelCase_       (     unittest.TestCase    ):


							def      _snake_case  (							self    :Any		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      inspect.getfile(accelerate.test_utils		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_script.py"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(
									    mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_distributed_data_loop.py"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_ops.py"""]		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[Any]		)   ->   Tuple:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices.'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.test_file_path]
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Tuple		)   ->   Optional[Any]:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices.'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.operation_file_path]
									print(f'''Command: {cmd}'''		)
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', inspect.getfile(self.__class__		)]
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[int]		)   ->   str:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices, using 2 devices only'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.data_loop_file_path]
									with patch_environment(omp_num_threads=1							,				cuda_visible_devices="""0,1"""		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


if __name__ == "__main__":
	_lowerCamelCase   				=			Accelerator()
	_lowerCamelCase   				=			(accelerator.state.process_index + 2, 10)
	_lowerCamelCase   				=			torch.randint(0, 10, shape).to(accelerator.device)

	_lowerCamelCase   				=			''

	_lowerCamelCase   				=			accelerator.pad_across_processes(tensor)
	if tensora.shape[0] != accelerator.state.num_processes + 1:
		error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
	if not torch.equal(tensora[: accelerator.state.process_index + 2], tensor):
		error_msg += "Tensors have different values."
	if not torch.all(tensora[accelerator.state.process_index + 2 :] == 0):
		error_msg += "Padding was not done with the right value (0)."

	_lowerCamelCase   				=			accelerator.pad_across_processes(tensor, pad_first=True)
	if tensora.shape[0] != accelerator.state.num_processes + 1:
		error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
	_lowerCamelCase   				=			accelerator.state.num_processes - accelerator.state.process_index - 1
	if not torch.equal(tensora[index:], tensor):
		error_msg += "Tensors have different values."
	if not torch.all(tensora[:index] == 0):
		error_msg += "Padding was not done with the right value (0)."

	# Raise error at the end to make sure we don't stop at the first failure.
	if len(error_msg) > 0:
		raise ValueError(error_msg)