import inspect
import unittest

from datasets import load_dataset
from packaging import version

from transformers import BeitConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import (
        MODEL_MAPPING,
        BeitForImageClassification,
        BeitForMaskedImageModeling,
        BeitForSemanticSegmentation,
        BeitModel,
    )
    from transformers.models.beit.modeling_beit import BEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    import PIL
    from PIL import Image

    from transformers import BeitImageProcessor


class 			lowerCamelCase_  :






    '''simple docstring'''






    def __init__(			self				,				__lowercase				,				__lowercase=100				,				__lowercase=13				,				__lowercase=30				,				__lowercase=2				,				__lowercase=3				,				__lowercase=True				,				__lowercase=True				,				__lowercase=32				,				__lowercase=4				,				__lowercase=4				,				__lowercase=37				,				__lowercase="gelu"				,				__lowercase=0.1				,				__lowercase=0.1				,				__lowercase=10				,				__lowercase=0.02				,				__lowercase=3				,				__lowercase=None				,				__lowercase=[0, 1, 2, 3]				,				) ->			Optional[int]:
        __UpperCamelCase  :Tuple    						=	parent
        __UpperCamelCase  :Optional[int]    						=	100
        __UpperCamelCase  :str    						=	batch_size
        __UpperCamelCase  :Union[str, Any]    						=	image_size
        __UpperCamelCase  :Optional[Any]    						=	patch_size
        __UpperCamelCase  :Dict    						=	num_channels
        __UpperCamelCase  :Optional[int]    						=	is_training
        __UpperCamelCase  :List[Any]    						=	use_labels
        __UpperCamelCase  :Union[str, Any]    						=	hidden_size
        __UpperCamelCase  :List[str]    						=	num_hidden_layers
        __UpperCamelCase  :Any    						=	num_attention_heads
        __UpperCamelCase  :List[str]    						=	intermediate_size
        __UpperCamelCase  :List[Any]    						=	hidden_act
        __UpperCamelCase  :List[str]    						=	hidden_dropout_prob
        __UpperCamelCase  :int    						=	attention_probs_dropout_prob
        __UpperCamelCase  :Union[str, Any]    						=	type_sequence_label_size
        __UpperCamelCase  :Optional[Any]    						=	initializer_range
        __UpperCamelCase  :Tuple    						=	scope
        __UpperCamelCase  :Union[str, Any]    						=	out_indices
        __UpperCamelCase  :Optional[int]    						=	num_labels

        # in BeiT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
        __UpperCamelCase  :Optional[Any]    						=	(image_size // patch_size) ** 2
        __UpperCamelCase  :str    						=	num_patches + 1







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        __UpperCamelCase  :int    						=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        __UpperCamelCase  :str    						=	None
        __UpperCamelCase  :Union[str, Any]    						=	None
        if self.use_labels:
            __UpperCamelCase  :Any    						=	ids_tensor([self.batch_size]				,				self.type_sequence_label_size)
            __UpperCamelCase  :str    						=	ids_tensor([self.batch_size, self.image_size, self.image_size]				,				self.num_labels)

        __UpperCamelCase  :Tuple    						=	self.get_config()

        return config, pixel_values, labels, pixel_labels







    def     UpperCamelCase__     (			self) ->			Tuple:
        return BeitConfig(
            vocab_size=self.vocab_size				,				image_size=self.image_size				,				patch_size=self.patch_size				,				num_channels=self.num_channels				,				hidden_size=self.hidden_size				,				num_hidden_layers=self.num_hidden_layers				,				num_attention_heads=self.num_attention_heads				,				intermediate_size=self.intermediate_size				,				hidden_act=self.hidden_act				,				hidden_dropout_prob=self.hidden_dropout_prob				,				attention_probs_dropout_prob=self.attention_probs_dropout_prob				,				is_decoder=__lowercase				,				initializer_range=self.initializer_range				,				out_indices=self.out_indices				,				)







    def     UpperCamelCase__     (			self				,				__lowercase				,				__lowercase				,				__lowercase				,				__lowercase) ->			Optional[int]:
        __UpperCamelCase  :int    						=	BeitModel(config=__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase  :Tuple    						=	model(__lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape				,				(self.batch_size, self.seq_length, self.hidden_size))







    def     UpperCamelCase__     (			self				,				__lowercase				,				__lowercase				,				__lowercase				,				__lowercase) ->			Union[str, Any]:
        __UpperCamelCase  :Optional[Any]    						=	BeitForMaskedImageModeling(config=__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase  :Tuple    						=	model(__lowercase)
        self.parent.assertEqual(result.logits.shape				,				(self.batch_size, self.seq_length - 1, self.vocab_size))







    def     UpperCamelCase__     (			self				,				__lowercase				,				__lowercase				,				__lowercase				,				__lowercase) ->			List[Any]:
        __UpperCamelCase  :Union[str, Any]    						=	self.type_sequence_label_size
        __UpperCamelCase  :Dict    						=	BeitForImageClassification(__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase  :List[Any]    						=	model(__lowercase				,				labels=__lowercase)
        self.parent.assertEqual(result.logits.shape				,				(self.batch_size, self.type_sequence_label_size))

        # test greyscale images
        __UpperCamelCase  :Optional[Any]    						=	1
        __UpperCamelCase  :Tuple    						=	BeitForImageClassification(__lowercase)
        model.to(__lowercase)
        model.eval()

        __UpperCamelCase  :List[Any]    						=	floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        __UpperCamelCase  :Union[str, Any]    						=	model(__lowercase				,				labels=__lowercase)
        self.parent.assertEqual(result.logits.shape				,				(self.batch_size, self.type_sequence_label_size))







    def     UpperCamelCase__     (			self				,				__lowercase				,				__lowercase				,				__lowercase				,				__lowercase) ->			Optional[int]:
        __UpperCamelCase  :Tuple    						=	self.num_labels
        __UpperCamelCase  :int    						=	BeitForSemanticSegmentation(__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase  :Union[str, Any]    						=	model(__lowercase)
        self.parent.assertEqual(
            result.logits.shape				,				(self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2))
        __UpperCamelCase  :Optional[int]    						=	model(__lowercase				,				labels=__lowercase)
        self.parent.assertEqual(
            result.logits.shape				,				(self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2))







    def     UpperCamelCase__     (			self) ->			List[str]:
        __UpperCamelCase  :Tuple    						=	self.prepare_config_and_inputs()
        __UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase  :Union[str, Any]    						=	config_and_inputs
        __UpperCamelCase  :Dict    						=	{'''pixel_values''': pixel_values}
        return config, inputs_dict


@require_torch
class 			lowerCamelCase_  (     UpperCAmelCase_		,			UpperCAmelCase_		,			unittest.TestCase				):






    '''simple docstring'''






    a__			:		Dict      			=   (
        (BeitModel, BeitForImageClassification, BeitForMaskedImageModeling, BeitForSemanticSegmentation)
        if is_torch_available()
        else ()
    )
    a__			:		str      			=   (
        {
            """feature-extraction""": BeitModel,
            """image-classification""": BeitForImageClassification,
            """image-segmentation""": BeitForSemanticSegmentation,
        }
        if is_torch_available()
        else {}
    )

    a__			:		Optional[int]      			=   False
    a__			:		List[str]      			=   False
    a__			:		str      			=   False







    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :str    						=	BeitModelTester(self)
        __UpperCamelCase  :Dict    						=	ConfigTester(self				,				config_class=__lowercase				,				has_text_modality=__lowercase				,				hidden_size=37)







    def     UpperCamelCase__     (			self) ->			int:
        self.config_tester.run_common_tests()







    @unittest.skip(reason='''BEiT does not use inputs_embeds''')
    def     UpperCamelCase__     (			self) ->			List[str]:
        pass







    @require_torch_multi_gpu
    @unittest.skip(reason='''BEiT has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`''')
    def     UpperCamelCase__     (			self) ->			Optional[int]:
        pass







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        __UpperCamelCase						, __UpperCamelCase  :Optional[Any]    						=	self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            __UpperCamelCase  :Dict    						=	model_class(__lowercase)
            self.assertIsInstance(model.get_input_embeddings()				,				(nn.Module))
            __UpperCamelCase  :str    						=	model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(__lowercase				,				nn.Linear))







    def     UpperCamelCase__     (			self) ->			Tuple:
        __UpperCamelCase						, __UpperCamelCase  :List[Any]    						=	self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            __UpperCamelCase  :Optional[Any]    						=	model_class(__lowercase)
            __UpperCamelCase  :Optional[Any]    						=	inspect.signature(model.forward)
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            __UpperCamelCase  :Optional[Any]    						=	[*signature.parameters.keys()]

            __UpperCamelCase  :Dict    						=	['''pixel_values''']
            self.assertListEqual(arg_names[:1]				,				__lowercase)







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        __UpperCamelCase  :Optional[Any]    						=	self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*__lowercase)







    def     UpperCamelCase__     (			self) ->			Tuple:
        __UpperCamelCase  :Any    						=	self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*__lowercase)







    def     UpperCamelCase__     (			self) ->			Any:
        __UpperCamelCase  :str    						=	self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*__lowercase)







    def     UpperCamelCase__     (			self) ->			str:
        __UpperCamelCase  :Union[str, Any]    						=	self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_semantic_segmentation(*__lowercase)







    def     UpperCamelCase__     (			self) ->			Dict:
        if not self.model_tester.is_training:
            return

        __UpperCamelCase						, __UpperCamelCase  :Optional[int]    						=	self.model_tester.prepare_config_and_inputs_for_common()
        __UpperCamelCase  :str    						=	True

        for model_class in self.all_model_classes:
            # we don't test BeitForMaskedImageModeling
            if model_class in [*get_values(__lowercase), BeitForMaskedImageModeling]:
                continue

            __UpperCamelCase  :str    						=	model_class(__lowercase)
            model.to(__lowercase)
            model.train()
            __UpperCamelCase  :List[str]    						=	self._prepare_for_class(__lowercase				,				__lowercase				,				return_labels=__lowercase)
            __UpperCamelCase  :List[Any]    						=	model(**__lowercase).loss
            loss.backward()







    def     UpperCamelCase__     (			self) ->			Dict:
        __UpperCamelCase						, __UpperCamelCase  :Optional[int]    						=	self.model_tester.prepare_config_and_inputs_for_common()
        if not self.model_tester.is_training:
            return

        __UpperCamelCase  :int    						=	False
        __UpperCamelCase  :str    						=	True

        for model_class in self.all_model_classes:
            # we don't test BeitForMaskedImageModeling
            if (
                model_class in [*get_values(__lowercase), BeitForMaskedImageModeling]
                or not model_class.supports_gradient_checkpointing
            ):
                continue

            __UpperCamelCase  :Tuple    						=	model_class(__lowercase)
            model.gradient_checkpointing_enable()
            model.to(__lowercase)
            model.train()
            __UpperCamelCase  :List[str]    						=	self._prepare_for_class(__lowercase				,				__lowercase				,				return_labels=__lowercase)
            __UpperCamelCase  :int    						=	model(**__lowercase).loss
            loss.backward()







    def     UpperCamelCase__     (			self) ->			List[str]:
        __UpperCamelCase						, __UpperCamelCase  :List[Any]    						=	self.model_tester.prepare_config_and_inputs_for_common()

        __UpperCamelCase  :List[str]    						=	_config_zero_init(__lowercase)
        for model_class in self.all_model_classes:
            __UpperCamelCase  :Dict    						=	model_class(config=__lowercase)
            for name, param in model.named_parameters():
                # we skip lambda parameters as these require special initial values
                # determined by config.layer_scale_init_value
                if "lambda" in name:
                    continue
                if param.requires_grad:
                    self.assertIn(
                        ((param.data.mean() * 1E9).round() / 1E9).item()				,				[0.0, 1.0]				,				msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""				,				)







    @slow
    def     UpperCamelCase__     (			self) ->			Tuple:
        for model_name in BEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __UpperCamelCase  :Union[str, Any]    						=	BeitModel.from_pretrained(__lowercase)
            self.assertIsNotNone(__lowercase)



def lowerCamelCase						(     ):
    '''simple docstring'''







    __UpperCamelCase  :Optional[int]    						=	Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''			)
    return image


@require_torch
@require_vision
class 			lowerCamelCase_  (     unittest.TestCase				):






    '''simple docstring'''






    @cached_property
    def     UpperCamelCase__     (			self) ->			str:
        return BeitImageProcessor.from_pretrained('''microsoft/beit-base-patch16-224''') if is_vision_available() else None







    @slow
    def     UpperCamelCase__     (			self) ->			Dict:
        __UpperCamelCase  :List[str]    						=	BeitForMaskedImageModeling.from_pretrained('''microsoft/beit-base-patch16-224-pt22k''').to(__lowercase)

        __UpperCamelCase  :Dict    						=	self.default_image_processor
        __UpperCamelCase  :str    						=	prepare_img()
        __UpperCamelCase  :Any    						=	image_processor(images=__lowercase				,				return_tensors='''pt''').pixel_values.to(__lowercase)

        # prepare bool_masked_pos
        __UpperCamelCase  :Dict    						=	torch.ones((1, 196)				,				dtype=torch.bool).to(__lowercase)

        # forward pass
        with torch.no_grad():
            __UpperCamelCase  :Union[str, Any]    						=	model(pixel_values=__lowercase				,				bool_masked_pos=__lowercase)
        __UpperCamelCase  :Optional[Any]    						=	outputs.logits

        # verify the logits
        __UpperCamelCase  :str    						=	torch.Size((1, 196, 8_192))
        self.assertEqual(logits.shape				,				__lowercase)

        __UpperCamelCase  :List[Any]    						=	torch.tensor(
            [[-3.24_37, 0.50_72, -13.91_74], [-3.24_56, 0.49_48, -13.94_01], [-3.20_33, 0.51_21, -13.85_50]]).to(__lowercase)

        self.assertTrue(torch.allclose(logits[bool_masked_pos][:3, :3]				,				__lowercase				,				atol=1E-2))







    @slow
    def     UpperCamelCase__     (			self) ->			Any:
        __UpperCamelCase  :str    						=	BeitForImageClassification.from_pretrained('''microsoft/beit-base-patch16-224''').to(__lowercase)

        __UpperCamelCase  :Tuple    						=	self.default_image_processor
        __UpperCamelCase  :Any    						=	prepare_img()
        __UpperCamelCase  :Optional[int]    						=	image_processor(images=__lowercase				,				return_tensors='''pt''').to(__lowercase)

        # forward pass
        with torch.no_grad():
            __UpperCamelCase  :Optional[Any]    						=	model(**__lowercase)
        __UpperCamelCase  :List[str]    						=	outputs.logits

        # verify the logits
        __UpperCamelCase  :Dict    						=	torch.Size((1, 1_000))
        self.assertEqual(logits.shape				,				__lowercase)

        __UpperCamelCase  :Optional[Any]    						=	torch.tensor([-1.23_85, -1.09_87, -1.01_08]).to(__lowercase)

        self.assertTrue(torch.allclose(logits[0, :3]				,				__lowercase				,				atol=1E-4))

        __UpperCamelCase  :Any    						=	281
        self.assertEqual(logits.argmax(-1).item()				,				__lowercase)







    @slow
    def     UpperCamelCase__     (			self) ->			List[str]:
        __UpperCamelCase  :List[str]    						=	BeitForImageClassification.from_pretrained('''microsoft/beit-large-patch16-224-pt22k-ft22k''').to(
            __lowercase)

        __UpperCamelCase  :List[Any]    						=	self.default_image_processor
        __UpperCamelCase  :Any    						=	prepare_img()
        __UpperCamelCase  :Any    						=	image_processor(images=__lowercase				,				return_tensors='''pt''').to(__lowercase)

        # forward pass
        with torch.no_grad():
            __UpperCamelCase  :str    						=	model(**__lowercase)
        __UpperCamelCase  :List[str]    						=	outputs.logits

        # verify the logits
        __UpperCamelCase  :int    						=	torch.Size((1, 21_841))
        self.assertEqual(logits.shape				,				__lowercase)

        __UpperCamelCase  :Optional[int]    						=	torch.tensor([1.68_81, -0.27_87, 0.59_01]).to(__lowercase)

        self.assertTrue(torch.allclose(logits[0, :3]				,				__lowercase				,				atol=1E-4))

        __UpperCamelCase  :Tuple    						=	2_396
        self.assertEqual(logits.argmax(-1).item()				,				__lowercase)







    @slow
    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :List[str]    						=	BeitForSemanticSegmentation.from_pretrained('''microsoft/beit-base-finetuned-ade-640-640''')
        __UpperCamelCase  :List[str]    						=	model.to(__lowercase)

        __UpperCamelCase  :Tuple    						=	BeitImageProcessor(do_resize=__lowercase				,				size=640				,				do_center_crop=__lowercase)

        __UpperCamelCase  :Union[str, Any]    						=	load_dataset('''hf-internal-testing/fixtures_ade20k'''				,				split='''test''')
        __UpperCamelCase  :Union[str, Any]    						=	Image.open(ds[0]['''file'''])
        __UpperCamelCase  :List[str]    						=	image_processor(images=__lowercase				,				return_tensors='''pt''').to(__lowercase)

        # forward pass
        with torch.no_grad():
            __UpperCamelCase  :List[Any]    						=	model(**__lowercase)
        __UpperCamelCase  :Tuple    						=	outputs.logits

        # verify the logits
        __UpperCamelCase  :Union[str, Any]    						=	torch.Size((1, 150, 160, 160))
        self.assertEqual(logits.shape				,				__lowercase)

        __UpperCamelCase  :Dict    						=	version.parse(PIL.__version__) < version.parse('''9.0.0''')

        if is_pillow_less_than_a:
            __UpperCamelCase  :int    						=	torch.tensor(
                [
                    [[-4.92_25, -2.39_54, -3.05_22], [-2.88_22, -1.00_46, -1.75_61], [-2.95_49, -1.32_28, -2.13_47]],
                    [[-5.81_68, -3.41_29, -4.07_78], [-3.86_51, -2.22_14, -3.02_77], [-3.83_56, -2.46_43, -3.35_35]],
                    [[-0.00_78, 3.99_52, 4.07_54], [2.98_56, 4.69_44, 5.00_35], [3.24_13, 4.78_13, 4.99_69]],
                ]				,				device=__lowercase				,				)
        else:
            __UpperCamelCase  :int    						=	torch.tensor(
                [
                    [[-4.89_60, -2.36_88, -3.03_55], [-2.84_78, -0.98_36, -1.74_18], [-2.94_49, -1.33_32, -2.14_56]],
                    [[-5.80_81, -3.41_24, -4.10_06], [-3.85_61, -2.20_81, -3.03_23], [-3.83_65, -2.46_01, -3.36_69]],
                    [[-0.03_09, 3.98_68, 4.05_40], [2.96_40, 4.68_77, 4.99_76], [3.20_81, 4.76_90, 4.99_42]],
                ]				,				device=__lowercase				,				)

        self.assertTrue(torch.allclose(logits[0, :3, :3, :3]				,				__lowercase				,				atol=1E-4))







    @slow
    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :Optional[int]    						=	BeitForSemanticSegmentation.from_pretrained('''microsoft/beit-base-finetuned-ade-640-640''')
        __UpperCamelCase  :List[str]    						=	model.to(__lowercase)

        __UpperCamelCase  :List[str]    						=	BeitImageProcessor(do_resize=__lowercase				,				size=640				,				do_center_crop=__lowercase)

        __UpperCamelCase  :List[Any]    						=	load_dataset('''hf-internal-testing/fixtures_ade20k'''				,				split='''test''')
        __UpperCamelCase  :List[str]    						=	Image.open(ds[0]['''file'''])
        __UpperCamelCase  :str    						=	image_processor(images=__lowercase				,				return_tensors='''pt''').to(__lowercase)

        # forward pass
        with torch.no_grad():
            __UpperCamelCase  :Union[str, Any]    						=	model(**__lowercase)

        __UpperCamelCase  :Optional[int]    						=	outputs.logits.detach().cpu()

        __UpperCamelCase  :Tuple    						=	image_processor.post_process_semantic_segmentation(outputs=__lowercase				,				target_sizes=[(500, 300)])
        __UpperCamelCase  :Optional[int]    						=	torch.Size((500, 300))
        self.assertEqual(segmentation[0].shape				,				__lowercase)

        __UpperCamelCase  :List[str]    						=	image_processor.post_process_semantic_segmentation(outputs=__lowercase)
        __UpperCamelCase  :Any    						=	torch.Size((160, 160))
        self.assertEqual(segmentation[0].shape				,				__lowercase)






import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
    import optax
    from flax.training.common_utils import onehot

    from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
    from transformers.models.ta.modeling_flax_ta import shift_tokens_right


@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 			lowerCamelCase_  (     unittest.TestCase				):






    '''simple docstring'''






    @slow
    def     UpperCamelCase__     (			self) ->			List[str]:
        __UpperCamelCase  :List[str]    						=	FlaxMTaForConditionalGeneration.from_pretrained('''google/mt5-small''')
        __UpperCamelCase  :Dict    						=	AutoTokenizer.from_pretrained('''google/mt5-small''')

        __UpperCamelCase  :Optional[Any]    						=	tokenizer('''Hello there'''				,				return_tensors='''np''').input_ids
        __UpperCamelCase  :List[str]    						=	tokenizer('''Hi I am'''				,				return_tensors='''np''').input_ids

        __UpperCamelCase  :Optional[int]    						=	shift_tokens_right(__lowercase				,				model.config.pad_token_id				,				model.config.decoder_start_token_id)

        __UpperCamelCase  :Tuple    						=	model(__lowercase				,				decoder_input_ids=__lowercase).logits
        __UpperCamelCase  :Any    						=	optax.softmax_cross_entropy(__lowercase				,				onehot(__lowercase				,				logits.shape[-1])).mean()

        __UpperCamelCase  :str    						=	-(labels.shape[-1] * loss.item())

        __UpperCamelCase  :Optional[Any]    						=	-84.91_27
        self.assertTrue(abs(mtf_score - EXPECTED_SCORE) < 1E-4)





