

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
       import torch

       from transformers import XLMRobertaModel






@require_sentencepiece
@require_tokenizers
@require_torch
class  UpperCAmelCase__	( unittest.TestCase ):






  """simple docstring"""


  @slow
  def    lowercase_	(							self		:	Any      )	->							Dict:
       SCREAMING_SNAKE_CASE__     :   List[Any]					=					XLMRobertaModel.from_pretrained('''xlm-roberta-base'''      )
       SCREAMING_SNAKE_CASE__     :   Dict					=					torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]]      )
       # The dog is cute and lives in the garden house

       SCREAMING_SNAKE_CASE__     :   str					=					torch.Size((1, 12, 768)      )  # batch_size, sequence_length, embedding_vector_dim
       SCREAMING_SNAKE_CASE__     :   Any					=					torch.tensor(
           [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]]      )
       #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
       #  xlmr.eval()
       #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
       with torch.no_grad():
            SCREAMING_SNAKE_CASE__     :   Union[str, Any]					=					model(lowercase__      )["""last_hidden_state"""].detach()
       self.assertEqual(output.shape			,     lowercase__      )
       # compare the actual values for a slice of last dim
       self.assertTrue(torch.allclose(output[:, :, -1]			,     lowercase__			,     atol=1e-3      )      )







  @slow
  def    lowercase_	(							self		:	List[Any]      )	->							List[Any]:
       SCREAMING_SNAKE_CASE__     :   List[str]					=					XLMRobertaModel.from_pretrained('''xlm-roberta-large'''      )
       SCREAMING_SNAKE_CASE__     :   Tuple					=					torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]]      )
       # The dog is cute and lives in the garden house

       SCREAMING_SNAKE_CASE__     :   List[Any]					=					torch.Size((1, 12, 1024)      )  # batch_size, sequence_length, embedding_vector_dim
       SCREAMING_SNAKE_CASE__     :   List[str]					=					torch.tensor(
           [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]]      )
       #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
       #  xlmr.eval()
       #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
       with torch.no_grad():
            SCREAMING_SNAKE_CASE__     :   Optional[int]					=					model(lowercase__      )["""last_hidden_state"""].detach()
       self.assertEqual(output.shape			,     lowercase__      )
       # compare the actual values for a slice of last dim
       self.assertTrue(torch.allclose(output[:, :, -1]			,     lowercase__			,     atol=1e-3      )      )





import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
       import torch_xla.core.xla_model as xm
       import torch_xla.debug.metrics as met






class  UpperCAmelCase__	( A__ ):






  """simple docstring"""


  def __init__(							self		:	Tuple			,     *__lowerCamelCase		:	List[str]			,     __lowerCamelCase		:	Any=None			,     __lowerCamelCase		:	Optional[int]=None			,     **__lowerCamelCase		:	Optional[int]      )	->							List[str]:
       super().__init__(*__lowerCamelCase			,     **__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					eval_examples
       SCREAMING_SNAKE_CASE__					=					post_process_function






  def    lowercase_	(							self		:	Optional[Any]			,     __lowerCamelCase		:	Dict=None			,     __lowerCamelCase		:	int=None			,     __lowerCamelCase		:	str=None			,     __lowerCamelCase		:	str = "eval"      )	->							Optional[Any]:
       SCREAMING_SNAKE_CASE__					=					self.eval_dataset if eval_dataset is None else eval_dataset
       SCREAMING_SNAKE_CASE__					=					self.get_eval_dataloader(__lowerCamelCase      )
       SCREAMING_SNAKE_CASE__					=					self.eval_examples if eval_examples is None else eval_examples

       # Temporarily disable metric computation, we will do it in the loop here.
       SCREAMING_SNAKE_CASE__					=					self.compute_metrics
       SCREAMING_SNAKE_CASE__					=					None
       SCREAMING_SNAKE_CASE__					=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
       SCREAMING_SNAKE_CASE__					=					time.time()
       try:
            SCREAMING_SNAKE_CASE__					=					eval_loop(
                __lowerCamelCase			,     description='''Evaluation'''			,     prediction_loss_only=True if compute_metrics is None else None			,     ignore_keys=__lowerCamelCase			,     metric_key_prefix=__lowerCamelCase			,     )
       finally:
            SCREAMING_SNAKE_CASE__					=					compute_metrics
       SCREAMING_SNAKE_CASE__					=					self.args.eval_batch_size * self.args.world_size
       if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
            start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
       output.metrics.update(
           speed_metrics(
               __lowerCamelCase			,     __lowerCamelCase			,     num_samples=output.num_samples			,     num_steps=math.ceil(output.num_samples / total_batch_size      )			,     )      )
       if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
            # Only the main node write the results by default
            SCREAMING_SNAKE_CASE__					=					self.post_process_function(__lowerCamelCase			,     __lowerCamelCase			,     output.predictions      )
            SCREAMING_SNAKE_CASE__					=					self.compute_metrics(__lowerCamelCase      )

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()      ):
                 if not key.startswith(f'''{metric_key_prefix}_'''      ):
                      SCREAMING_SNAKE_CASE__					=					metrics.pop(__lowerCamelCase      )
            metrics.update(output.metrics      )
       else:
            SCREAMING_SNAKE_CASE__					=					output.metrics

       if self.args.should_log:
            # Only the main node log the results by default
            self.log(__lowerCamelCase      )

       if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()      )

       SCREAMING_SNAKE_CASE__					=					self.callback_handler.on_evaluate(self.args			,     self.state			,     self.control			,     __lowerCamelCase      )
       return metrics







  def    lowercase_	(							self		:	Dict			,     __lowerCamelCase		:	Dict			,     __lowerCamelCase		:	str			,     __lowerCamelCase		:	str=None			,     __lowerCamelCase		:	str = "test"      )	->							Any:
       SCREAMING_SNAKE_CASE__					=					self.get_test_dataloader(__lowerCamelCase      )

       # Temporarily disable metric computation, we will do it in the loop here.
       SCREAMING_SNAKE_CASE__					=					self.compute_metrics
       SCREAMING_SNAKE_CASE__					=					None
       SCREAMING_SNAKE_CASE__					=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
       SCREAMING_SNAKE_CASE__					=					time.time()
       try:
            SCREAMING_SNAKE_CASE__					=					eval_loop(
                __lowerCamelCase			,     description='''Prediction'''			,     prediction_loss_only=True if compute_metrics is None else None			,     ignore_keys=__lowerCamelCase			,     metric_key_prefix=__lowerCamelCase			,     )
       finally:
            SCREAMING_SNAKE_CASE__					=					compute_metrics
       SCREAMING_SNAKE_CASE__					=					self.args.eval_batch_size * self.args.world_size
       if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
            start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
       output.metrics.update(
           speed_metrics(
               __lowerCamelCase			,     __lowerCamelCase			,     num_samples=output.num_samples			,     num_steps=math.ceil(output.num_samples / total_batch_size      )			,     )      )

       if self.post_process_function is None or self.compute_metrics is None:
            return output

       SCREAMING_SNAKE_CASE__					=					self.post_process_function(__lowerCamelCase			,     __lowerCamelCase			,     output.predictions			,     '''predict'''      )
       SCREAMING_SNAKE_CASE__					=					self.compute_metrics(__lowerCamelCase      )

       # Prefix all keys with metric_key_prefix + '_'
       for key in list(metrics.keys()      ):
            if not key.startswith(f'''{metric_key_prefix}_'''      ):
                 SCREAMING_SNAKE_CASE__					=					metrics.pop(__lowerCamelCase      )
       metrics.update(output.metrics      )
       return PredictionOutput(predictions=predictions.predictions			,     label_ids=predictions.label_ids			,     metrics=__lowerCamelCase      )


