
'''simple docstring'''



import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
     import torch_xla.core.xla_model as xm
     import torch_xla.debug.metrics as met

class    _a				(_lowerCamelCase):





       """simple docstring"""




       def __init__(							self       ,							*A__       ,							A__=None       ,							A__=None       ,							**A__      )    ->			Union[str, Any]:
          super().__init__(*A__       ,							**A__      )
          _SCREAMING_SNAKE_CASE      =				eval_examples
          _SCREAMING_SNAKE_CASE      =				post_process_function






       def        UpperCamelCase				(							self       ,							A__=None       ,							A__=None       ,							A__=None       ,							A__ = "eval"      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				self.eval_dataset if eval_dataset is None else eval_dataset
          _SCREAMING_SNAKE_CASE      =				self.get_eval_dataloader(A__      )
          _SCREAMING_SNAKE_CASE      =				self.eval_examples if eval_examples is None else eval_examples

          # Temporarily disable metric computation, we will do it in the loop here.
          _SCREAMING_SNAKE_CASE      =				self.compute_metrics
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          _SCREAMING_SNAKE_CASE      =				time.time()
          try:
             _SCREAMING_SNAKE_CASE      =				eval_loop(
                 A__       ,							description="""Evaluation"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=A__       ,							metric_key_prefix=A__       ,							)
          finally:
             _SCREAMING_SNAKE_CASE      =				compute_metrics
          _SCREAMING_SNAKE_CASE      =				self.args.eval_batch_size * self.args.world_size
          if F"{metric_key_prefix}_jit_compilation_time" in output.metrics:
             start_time += output.metrics[F"{metric_key_prefix}_jit_compilation_time"]
          output.metrics.update(
              speed_metrics(
                  A__       ,							A__       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size      )       ,							)      )
          if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
             # Only the main node write the results by default
             _SCREAMING_SNAKE_CASE      =				self.post_process_function(A__       ,							A__       ,							output.predictions      )
             _SCREAMING_SNAKE_CASE      =				self.compute_metrics(A__      )

             # Prefix all keys with metric_key_prefix + '_'
             for key in list(metrics.keys()      ):
                if not key.startswith(F"{metric_key_prefix}_"      ):
                   _SCREAMING_SNAKE_CASE      =				metrics.pop(A__      )
             metrics.update(output.metrics      )
          else:
             _SCREAMING_SNAKE_CASE      =				output.metrics

          if self.args.should_log:
             # Only the main node log the results by default
             self.log(A__      )

          if self.args.tpu_metrics_debug or self.args.debug:
             # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
             xm.master_print(met.metrics_report()      )

          _SCREAMING_SNAKE_CASE      =				self.callback_handler.on_evaluate(self.args       ,							self.state       ,							self.control       ,							A__      )
          return metrics






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__=None       ,							A__ = "test"      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				self.get_test_dataloader(A__      )

          # Temporarily disable metric computation, we will do it in the loop here.
          _SCREAMING_SNAKE_CASE      =				self.compute_metrics
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          _SCREAMING_SNAKE_CASE      =				time.time()
          try:
             _SCREAMING_SNAKE_CASE      =				eval_loop(
                 A__       ,							description="""Prediction"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=A__       ,							metric_key_prefix=A__       ,							)
          finally:
             _SCREAMING_SNAKE_CASE      =				compute_metrics
          _SCREAMING_SNAKE_CASE      =				self.args.eval_batch_size * self.args.world_size
          if F"{metric_key_prefix}_jit_compilation_time" in output.metrics:
             start_time += output.metrics[F"{metric_key_prefix}_jit_compilation_time"]
          output.metrics.update(
              speed_metrics(
                  A__       ,							A__       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size      )       ,							)      )

          if self.post_process_function is None or self.compute_metrics is None:
             return output

          _SCREAMING_SNAKE_CASE      =				self.post_process_function(A__       ,							A__       ,							output.predictions       ,							"""predict"""      )
          _SCREAMING_SNAKE_CASE      =				self.compute_metrics(A__      )

          # Prefix all keys with metric_key_prefix + '_'
          for key in list(metrics.keys()      ):
             if not key.startswith(F"{metric_key_prefix}_"      ):
                _SCREAMING_SNAKE_CASE      =				metrics.pop(A__      )
          metrics.update(output.metrics      )
          return PredictionOutput(predictions=predictions.predictions       ,							label_ids=predictions.label_ids       ,							metrics=A__      )








'''simple docstring'''



from sklearn.metrics import matthews_corrcoef

import datasets


UpperCamelCase__      :		List[str]       							=  "\nCompute the Matthews correlation coefficient (MCC)\n\nThe Matthews correlation coefficient is used in machine learning as a\nmeasure of the quality of binary and multiclass classifications. It takes\ninto account true and false positives and negatives and is generally\nregarded as a balanced measure which can be used even if the classes are of\nvery different sizes. The MCC is in essence a correlation coefficient value\nbetween -1 and +1. A coefficient of +1 represents a perfect prediction, 0\nan average random prediction and -1 an inverse prediction.  The statistic\nis also known as the phi coefficient. [source: Wikipedia]\n"

UpperCamelCase__      :		List[Any]       							=  "\nArgs:\n    predictions (list of int): Predicted labels, as returned by a model.\n    references (list of int): Ground truth labels.\n    sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`.\nReturns:\n    matthews_correlation (dict containing float): Matthews correlation.\nExamples:\n    Example 1, a basic example with only predictions and references as inputs:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.54\n\n    Example 2, the same example as above, but also including sample weights:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 3, 1, 1, 1, 2])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.1\n\n    Example 3, the same example as above, but with sample weights that cause a negative correlation:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 1, 0, 0, 0, 1])\n        >>> print(round(results['matthews_correlation'], 2))\n        -0.25\n"

UpperCamelCase__      :		Any       							=  "\\n@article{scikit-learn,\n  title={Scikit-learn: Machine Learning in {P}ython},\n  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n  journal={Journal of Machine Learning Research},\n  volume={12},\n  pages={2825--2830},\n  year={2011}\n}\n"

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,					_KWARGS_DESCRIPTION)
class    _a				(datasets.Metric):





       """simple docstring"""




       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          return datasets.MetricInfo(
              description=_DESCRIPTION       ,							citation=_CITATION       ,							inputs_description=_KWARGS_DESCRIPTION       ,							features=datasets.Features(
                  {
                      """predictions""": datasets.Value("""int32"""      ),
                      """references""": datasets.Value("""int32"""      ),
                  }      )       ,							reference_urls=[
                  """https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html"""
              ]       ,							)






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__=None      )    ->			List[str]:
          return {
              "matthews_correlation": float(matthews_corrcoef(A__       ,							A__       ,							sample_weight=A__      )      ),
          }






