

import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
           MODEL_FOR_QUESTION_ANSWERING_MAPPING,
           MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
           MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
           LayoutLMvaConfig,
           LayoutLMvaForQuestionAnswering,
           LayoutLMvaForSequenceClassification,
           LayoutLMvaForTokenClassification,
           LayoutLMvaModel,
       )
       from transformers.models.layoutlmva.modeling_layoutlmva import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
       from PIL import Image

       from transformers import LayoutLMvaImageProcessor






class  UpperCAmelCase__	:






  """simple docstring"""


  def __init__(							self		:	Dict			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	Union[str, Any]=2			,     __lowerCamelCase		:	Dict=3			,     __lowerCamelCase		:	Optional[Any]=4			,     __lowerCamelCase		:	Any=2			,     __lowerCamelCase		:	Dict=7			,     __lowerCamelCase		:	List[str]=True			,     __lowerCamelCase		:	Any=True			,     __lowerCamelCase		:	Optional[Any]=True			,     __lowerCamelCase		:	Dict=True			,     __lowerCamelCase		:	Any=99			,     __lowerCamelCase		:	Optional[Any]=36			,     __lowerCamelCase		:	Any=3			,     __lowerCamelCase		:	Dict=4			,     __lowerCamelCase		:	List[Any]=37			,     __lowerCamelCase		:	List[str]="gelu"			,     __lowerCamelCase		:	str=0.1			,     __lowerCamelCase		:	str=0.1			,     __lowerCamelCase		:	str=512			,     __lowerCamelCase		:	str=16			,     __lowerCamelCase		:	Optional[int]=2			,     __lowerCamelCase		:	Union[str, Any]=0.02			,     __lowerCamelCase		:	List[Any]=6			,     __lowerCamelCase		:	Optional[int]=6			,     __lowerCamelCase		:	Tuple=3			,     __lowerCamelCase		:	Any=4			,     __lowerCamelCase		:	Optional[int]=None			,     __lowerCamelCase		:	Dict=1000			,     )	->							List[str]:
       SCREAMING_SNAKE_CASE__					=					parent
       SCREAMING_SNAKE_CASE__					=					batch_size
       SCREAMING_SNAKE_CASE__					=					num_channels
       SCREAMING_SNAKE_CASE__					=					image_size
       SCREAMING_SNAKE_CASE__					=					patch_size
       SCREAMING_SNAKE_CASE__					=					text_seq_length
       SCREAMING_SNAKE_CASE__					=					is_training
       SCREAMING_SNAKE_CASE__					=					use_input_mask
       SCREAMING_SNAKE_CASE__					=					use_token_type_ids
       SCREAMING_SNAKE_CASE__					=					use_labels
       SCREAMING_SNAKE_CASE__					=					vocab_size
       SCREAMING_SNAKE_CASE__					=					hidden_size
       SCREAMING_SNAKE_CASE__					=					num_hidden_layers
       SCREAMING_SNAKE_CASE__					=					num_attention_heads
       SCREAMING_SNAKE_CASE__					=					intermediate_size
       SCREAMING_SNAKE_CASE__					=					hidden_act
       SCREAMING_SNAKE_CASE__					=					hidden_dropout_prob
       SCREAMING_SNAKE_CASE__					=					attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE__					=					max_position_embeddings
       SCREAMING_SNAKE_CASE__					=					type_vocab_size
       SCREAMING_SNAKE_CASE__					=					type_sequence_label_size
       SCREAMING_SNAKE_CASE__					=					initializer_range
       SCREAMING_SNAKE_CASE__					=					coordinate_size
       SCREAMING_SNAKE_CASE__					=					shape_size
       SCREAMING_SNAKE_CASE__					=					num_labels
       SCREAMING_SNAKE_CASE__					=					num_choices
       SCREAMING_SNAKE_CASE__					=					scope
       SCREAMING_SNAKE_CASE__					=					range_bbox

       # LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
       SCREAMING_SNAKE_CASE__					=					text_seq_length
       SCREAMING_SNAKE_CASE__					=					(image_size // patch_size) ** 2 + 1
       SCREAMING_SNAKE_CASE__					=					self.text_seq_length + self.image_seq_length






  def    lowercase_	(							self		:	Union[str, Any]      )	->							Union[str, Any]:
       SCREAMING_SNAKE_CASE__					=					ids_tensor([self.batch_size, self.text_seq_length]			,     self.vocab_size      )

       SCREAMING_SNAKE_CASE__					=					ids_tensor([self.batch_size, self.text_seq_length, 4]			,     self.range_bbox      )
       # Ensure that bbox is legal
       for i in range(bbox.shape[0]      ):
            for j in range(bbox.shape[1]      ):
                 if bbox[i, j, 3] < bbox[i, j, 1]:
                      SCREAMING_SNAKE_CASE__					=					bbox[i, j, 3]
                      SCREAMING_SNAKE_CASE__					=					bbox[i, j, 1]
                      SCREAMING_SNAKE_CASE__					=					t
                 if bbox[i, j, 2] < bbox[i, j, 0]:
                      SCREAMING_SNAKE_CASE__					=					bbox[i, j, 2]
                      SCREAMING_SNAKE_CASE__					=					bbox[i, j, 0]
                      SCREAMING_SNAKE_CASE__					=					t

       SCREAMING_SNAKE_CASE__					=					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]      )

       SCREAMING_SNAKE_CASE__					=					None
       if self.use_input_mask:
            SCREAMING_SNAKE_CASE__					=					random_attention_mask([self.batch_size, self.text_seq_length]      )

       SCREAMING_SNAKE_CASE__					=					None
       if self.use_token_type_ids:
            SCREAMING_SNAKE_CASE__					=					ids_tensor([self.batch_size, self.text_seq_length]			,     self.type_vocab_size      )

       SCREAMING_SNAKE_CASE__					=					None
       SCREAMING_SNAKE_CASE__					=					None
       if self.use_labels:
            SCREAMING_SNAKE_CASE__					=					ids_tensor([self.batch_size]			,     self.type_sequence_label_size      )
            SCREAMING_SNAKE_CASE__					=					ids_tensor([self.batch_size, self.text_seq_length]			,     self.num_labels      )

       SCREAMING_SNAKE_CASE__					=					LayoutLMvaConfig(
           vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     initializer_range=self.initializer_range			,     coordinate_size=self.coordinate_size			,     shape_size=self.shape_size			,     input_size=self.image_size			,     patch_size=self.patch_size			,     )

       return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels






  def    lowercase_	(							self		:	Any			,     __lowerCamelCase		:	List[Any]			,     __lowerCamelCase		:	List[Any]			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	Union[str, Any]			,     __lowerCamelCase		:	List[Any]			,     __lowerCamelCase		:	List[str]      )	->							Union[str, Any]:
       SCREAMING_SNAKE_CASE__					=					LayoutLMvaModel(config=__lowerCamelCase      )
       model.to(__lowerCamelCase      )
       model.eval()

       # text + image
       SCREAMING_SNAKE_CASE__					=					model(__lowerCamelCase			,     pixel_values=__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					model(
           __lowerCamelCase			,     bbox=__lowerCamelCase			,     pixel_values=__lowerCamelCase			,     attention_mask=__lowerCamelCase			,     token_type_ids=__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					model(__lowerCamelCase			,     bbox=__lowerCamelCase			,     pixel_values=__lowerCamelCase			,     token_type_ids=__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					model(__lowerCamelCase			,     bbox=__lowerCamelCase			,     pixel_values=__lowerCamelCase      )

       self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)      )

       # text only
       SCREAMING_SNAKE_CASE__					=					model(__lowerCamelCase      )
       self.parent.assertEqual(
           result.last_hidden_state.shape			,     (self.batch_size, self.text_seq_length, self.hidden_size)      )

       # image only
       SCREAMING_SNAKE_CASE__					=					model(pixel_values=__lowerCamelCase      )
       self.parent.assertEqual(
           result.last_hidden_state.shape			,     (self.batch_size, self.image_seq_length, self.hidden_size)      )






  def    lowercase_	(							self		:	List[Any]			,     __lowerCamelCase		:	List[str]			,     __lowerCamelCase		:	Any			,     __lowerCamelCase		:	Optional[int]			,     __lowerCamelCase		:	Optional[int]			,     __lowerCamelCase		:	Any			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	Tuple			,     __lowerCamelCase		:	Any      )	->							Optional[int]:
       SCREAMING_SNAKE_CASE__					=					self.num_labels
       SCREAMING_SNAKE_CASE__					=					LayoutLMvaForSequenceClassification(__lowerCamelCase      )
       model.to(__lowerCamelCase      )
       model.eval()
       SCREAMING_SNAKE_CASE__					=					model(
           __lowerCamelCase			,     bbox=__lowerCamelCase			,     pixel_values=__lowerCamelCase			,     attention_mask=__lowerCamelCase			,     token_type_ids=__lowerCamelCase			,     labels=__lowerCamelCase			,     )
       self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)      )






  def    lowercase_	(							self		:	List[Any]			,     __lowerCamelCase		:	int			,     __lowerCamelCase		:	Any			,     __lowerCamelCase		:	Optional[Any]			,     __lowerCamelCase		:	List[Any]			,     __lowerCamelCase		:	Optional[Any]			,     __lowerCamelCase		:	Tuple			,     __lowerCamelCase		:	Any			,     __lowerCamelCase		:	int      )	->							Dict:
       SCREAMING_SNAKE_CASE__					=					self.num_labels
       SCREAMING_SNAKE_CASE__					=					LayoutLMvaForTokenClassification(config=__lowerCamelCase      )
       model.to(__lowerCamelCase      )
       model.eval()
       SCREAMING_SNAKE_CASE__					=					model(
           __lowerCamelCase			,     bbox=__lowerCamelCase			,     pixel_values=__lowerCamelCase			,     attention_mask=__lowerCamelCase			,     token_type_ids=__lowerCamelCase			,     labels=__lowerCamelCase			,     )
       self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.text_seq_length, self.num_labels)      )






  def    lowercase_	(							self		:	List[str]			,     __lowerCamelCase		:	List[str]			,     __lowerCamelCase		:	Union[str, Any]			,     __lowerCamelCase		:	Optional[Any]			,     __lowerCamelCase		:	Union[str, Any]			,     __lowerCamelCase		:	Optional[int]			,     __lowerCamelCase		:	Optional[Any]			,     __lowerCamelCase		:	List[str]			,     __lowerCamelCase		:	Optional[int]      )	->							Union[str, Any]:
       SCREAMING_SNAKE_CASE__					=					LayoutLMvaForQuestionAnswering(config=__lowerCamelCase      )
       model.to(__lowerCamelCase      )
       model.eval()
       SCREAMING_SNAKE_CASE__					=					model(
           __lowerCamelCase			,     bbox=__lowerCamelCase			,     pixel_values=__lowerCamelCase			,     attention_mask=__lowerCamelCase			,     token_type_ids=__lowerCamelCase			,     start_positions=__lowerCamelCase			,     end_positions=__lowerCamelCase			,     )
       self.parent.assertEqual(result.start_logits.shape			,     (self.batch_size, self.seq_length)      )
       self.parent.assertEqual(result.end_logits.shape			,     (self.batch_size, self.seq_length)      )







  def    lowercase_	(							self		:	List[str]      )	->							List[str]:
       SCREAMING_SNAKE_CASE__					=					self.prepare_config_and_inputs()
       (
           (
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),(
           SCREAMING_SNAKE_CASE__
       ),
       )					=					config_and_inputs
       SCREAMING_SNAKE_CASE__					=					{
           '''input_ids''': input_ids,
           '''bbox''': bbox,
           '''pixel_values''': pixel_values,
           '''token_type_ids''': token_type_ids,
           '''attention_mask''': input_mask,
       }
       return config, inputs_dict






@require_torch
class  UpperCAmelCase__	( A__		,  A__		,  unittest.TestCase ):






  """simple docstring"""


  a     =      False
  a     =      False
  a     =      False

  a     =      (
      (
          LayoutLMvaModel,
          LayoutLMvaForSequenceClassification,
          LayoutLMvaForTokenClassification,
          LayoutLMvaForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  a     =      (
      {"document-question-answering": LayoutLMvaForQuestionAnswering, "feature-extraction": LayoutLMvaModel}
      if is_torch_available()
      else {}
  )






  def    lowercase_	(							self		:	List[Any]			,     __lowerCamelCase		:	Any			,     __lowerCamelCase		:	List[str]			,     __lowerCamelCase		:	Tuple			,     __lowerCamelCase		:	List[Any]			,     __lowerCamelCase		:	Optional[Any]      )	->							Union[str, Any]:
       # `DocumentQuestionAnsweringPipeline` is expected to work with this model, but it combines the text and visual
       # embedding along the sequence dimension (dim 1), which causes an error during post-processing as `p_mask` has
       # the sequence dimension of the text embedding only.
       # (see the line `embedding_output = torch.cat([embedding_output, visual_embeddings], dim=1)`)
       return True






  def    lowercase_	(							self		:	Tuple      )	->							int:
       SCREAMING_SNAKE_CASE__					=					LayoutLMvaModelTester(self      )
       SCREAMING_SNAKE_CASE__					=					ConfigTester(self			,     config_class=__lowerCamelCase			,     hidden_size=37      )






  def    lowercase_	(							self		:	List[str]			,     __lowerCamelCase		:	Union[str, Any]			,     __lowerCamelCase		:	Optional[Any]			,     __lowerCamelCase		:	int=False      )	->							Dict:
       SCREAMING_SNAKE_CASE__					=					copy.deepcopy(__lowerCamelCase      )
       if model_class in get_values(__lowerCamelCase      ):
            SCREAMING_SNAKE_CASE__					=					{
                k: v.unsqueeze(1      ).expand(-1			,     self.model_tester.num_choices			,     -1      ).contiguous()
                if isinstance(__lowerCamelCase			,     torch.Tensor      ) and v.ndim > 1
                else v
                for k, v in inputs_dict.items()
            }
       if return_labels:
            if model_class in get_values(__lowerCamelCase      ):
                 SCREAMING_SNAKE_CASE__					=					torch.ones(self.model_tester.batch_size			,     dtype=torch.long			,     device=__lowerCamelCase      )
            elif model_class in get_values(__lowerCamelCase      ):
                 SCREAMING_SNAKE_CASE__					=					torch.zeros(
                     self.model_tester.batch_size			,     dtype=torch.long			,     device=__lowerCamelCase      )
                 SCREAMING_SNAKE_CASE__					=					torch.zeros(
                     self.model_tester.batch_size			,     dtype=torch.long			,     device=__lowerCamelCase      )
            elif model_class in [
                *get_values(__lowerCamelCase      ),
            ]:
                 SCREAMING_SNAKE_CASE__					=					torch.zeros(
                     self.model_tester.batch_size			,     dtype=torch.long			,     device=__lowerCamelCase      )
            elif model_class in [
                *get_values(__lowerCamelCase      ),
            ]:
                 SCREAMING_SNAKE_CASE__					=					torch.zeros(
                     (self.model_tester.batch_size, self.model_tester.text_seq_length)			,     dtype=torch.long			,     device=__lowerCamelCase			,     )

       return inputs_dict






  def    lowercase_	(							self		:	Optional[int]      )	->							Any:
       self.config_tester.run_common_tests()






  def    lowercase_	(							self		:	Any      )	->							int:
       SCREAMING_SNAKE_CASE__					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__lowerCamelCase      )






  def    lowercase_	(							self		:	Union[str, Any]      )	->							Optional[Any]:
       SCREAMING_SNAKE_CASE__					=					self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
            SCREAMING_SNAKE_CASE__					=					type
            self.model_tester.create_and_check_model(*__lowerCamelCase      )






  def    lowercase_	(							self		:	Optional[Any]      )	->							Dict:
       SCREAMING_SNAKE_CASE__					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*__lowerCamelCase      )






  def    lowercase_	(							self		:	List[Any]      )	->							Tuple:
       SCREAMING_SNAKE_CASE__					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*__lowerCamelCase      )






  def    lowercase_	(							self		:	Dict      )	->							Optional[Any]:
       SCREAMING_SNAKE_CASE__					=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*__lowerCamelCase      )







  @slow
  def    lowercase_	(							self		:	Optional[int]      )	->							int:
       for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            SCREAMING_SNAKE_CASE__					=					LayoutLMvaModel.from_pretrained(__lowerCamelCase      )
            self.assertIsNotNone(__lowerCamelCase      )



def        UpperCAmelCase_     (		):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       )
     return image







@require_torch
class  UpperCAmelCase__	( unittest.TestCase ):






  """simple docstring"""


  @cached_property
  def    lowercase_	(							self		:	Dict      )	->							Any:
       return LayoutLMvaImageProcessor(apply_ocr=__lowerCamelCase      ) if is_vision_available() else None







  @slow
  def    lowercase_	(							self		:	Any      )	->							Optional[int]:
       SCREAMING_SNAKE_CASE__					=					LayoutLMvaModel.from_pretrained('''microsoft/layoutlmv3-base'''      ).to(__lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					self.default_image_processor
       SCREAMING_SNAKE_CASE__					=					prepare_img()
       SCREAMING_SNAKE_CASE__					=					image_processor(images=__lowerCamelCase			,     return_tensors='''pt'''      ).pixel_values.to(__lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					torch.tensor([[1, 2]]      )
       SCREAMING_SNAKE_CASE__					=					torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]      ).unsqueeze(0      )

       # forward pass
       SCREAMING_SNAKE_CASE__					=					model(
           input_ids=input_ids.to(__lowerCamelCase      )			,     bbox=bbox.to(__lowerCamelCase      )			,     pixel_values=pixel_values.to(__lowerCamelCase      )			,     )

       # verify the logits
       SCREAMING_SNAKE_CASE__					=					torch.Size((1, 199, 768)      )
       self.assertEqual(outputs.last_hidden_state.shape			,     __lowerCamelCase      )

       SCREAMING_SNAKE_CASE__					=					torch.tensor(
           [[-0.0529, 0.3618, 0.1632], [-0.1587, -0.1667, -0.0400], [-0.1557, -0.1671, -0.0505]]      ).to(__lowerCamelCase      )

       self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,     __lowerCamelCase			,     atol=1e-4      )      )





import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
       import torch_xla.core.xla_model as xm
       import torch_xla.debug.metrics as met






class  UpperCAmelCase__	( A__ ):






  """simple docstring"""


  def __init__(							self		:	Tuple			,     *__lowerCamelCase		:	List[str]			,     __lowerCamelCase		:	Any=None			,     __lowerCamelCase		:	Optional[int]=None			,     **__lowerCamelCase		:	Optional[int]      )	->							List[str]:
       super().__init__(*__lowerCamelCase			,     **__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					eval_examples
       SCREAMING_SNAKE_CASE__					=					post_process_function






  def    lowercase_	(							self		:	Optional[Any]			,     __lowerCamelCase		:	Dict=None			,     __lowerCamelCase		:	int=None			,     __lowerCamelCase		:	str=None			,     __lowerCamelCase		:	str = "eval"      )	->							Optional[Any]:
       SCREAMING_SNAKE_CASE__					=					self.eval_dataset if eval_dataset is None else eval_dataset
       SCREAMING_SNAKE_CASE__					=					self.get_eval_dataloader(__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					self.eval_examples if eval_examples is None else eval_examples

       # Temporarily disable metric computation, we will do it in the loop here.
       SCREAMING_SNAKE_CASE__					=					self.compute_metrics
       SCREAMING_SNAKE_CASE__					=					None
       SCREAMING_SNAKE_CASE__					=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
       SCREAMING_SNAKE_CASE__					=					time.time()
       try:
            SCREAMING_SNAKE_CASE__					=					eval_loop(
                __lowerCamelCase			,     description='''Evaluation'''			,     prediction_loss_only=True if compute_metrics is None else None			,     ignore_keys=__lowerCamelCase			,     metric_key_prefix=__lowerCamelCase			,     )
       finally:
            SCREAMING_SNAKE_CASE__					=					compute_metrics
       SCREAMING_SNAKE_CASE__					=					self.args.eval_batch_size * self.args.world_size
       if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
            start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
       output.metrics.update(
           speed_metrics(
               __lowerCamelCase			,     __lowerCamelCase			,     num_samples=output.num_samples			,     num_steps=math.ceil(output.num_samples / total_batch_size      )			,     )      )
       if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
            # Only the main node write the results by default
            SCREAMING_SNAKE_CASE__					=					self.post_process_function(__lowerCamelCase			,     __lowerCamelCase			,     output.predictions      )
            SCREAMING_SNAKE_CASE__					=					self.compute_metrics(__lowerCamelCase      )

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()      ):
                 if not key.startswith(f'''{metric_key_prefix}_'''      ):
                      SCREAMING_SNAKE_CASE__					=					metrics.pop(__lowerCamelCase      )
            metrics.update(output.metrics      )
       else:
            SCREAMING_SNAKE_CASE__					=					output.metrics

       if self.args.should_log:
            # Only the main node log the results by default
            self.log(__lowerCamelCase      )

       if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()      )

       SCREAMING_SNAKE_CASE__					=					self.callback_handler.on_evaluate(self.args			,     self.state			,     self.control			,     __lowerCamelCase      )
       return metrics







  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	Dict			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	str=None			,     __lowerCamelCase		:	str = "test"      )	->							Any:
       SCREAMING_SNAKE_CASE__					=					self.get_test_dataloader(__lowerCamelCase      )

       # Temporarily disable metric computation, we will do it in the loop here.
       SCREAMING_SNAKE_CASE__					=					self.compute_metrics
       SCREAMING_SNAKE_CASE__					=					None
       SCREAMING_SNAKE_CASE__					=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
       SCREAMING_SNAKE_CASE__					=					time.time()
       try:
            SCREAMING_SNAKE_CASE__					=					eval_loop(
                __lowerCamelCase			,     description='''Prediction'''			,     prediction_loss_only=True if compute_metrics is None else None			,     ignore_keys=__lowerCamelCase			,     metric_key_prefix=__lowerCamelCase			,     )
       finally:
            SCREAMING_SNAKE_CASE__					=					compute_metrics
       SCREAMING_SNAKE_CASE__					=					self.args.eval_batch_size * self.args.world_size
       if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
            start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
       output.metrics.update(
           speed_metrics(
               __lowerCamelCase			,     __lowerCamelCase			,     num_samples=output.num_samples			,     num_steps=math.ceil(output.num_samples / total_batch_size      )			,     )      )

       if self.post_process_function is None or self.compute_metrics is None:
            return output

       SCREAMING_SNAKE_CASE__					=					self.post_process_function(__lowerCamelCase			,     __lowerCamelCase			,     output.predictions			,     '''predict'''      )
       SCREAMING_SNAKE_CASE__					=					self.compute_metrics(__lowerCamelCase      )

       # Prefix all keys with metric_key_prefix + '_'
       for key in list(metrics.keys()      ):
            if not key.startswith(f'''{metric_key_prefix}_'''      ):
                 SCREAMING_SNAKE_CASE__					=					metrics.pop(__lowerCamelCase      )
       metrics.update(output.metrics      )
       return PredictionOutput(predictions=predictions.predictions			,     label_ids=predictions.label_ids			,     metrics=__lowerCamelCase      )


