





"""simple docstring"""






import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


lowerCAmelCase_			=							'''\
@article{hendrycksmath2021,
  title={Measuring Mathematical Problem Solving With the MATH Dataset},
  author={Dan Hendrycks
    and Collin Burns
    and Saurav Kadavath
    and Akul Arora
    and Steven Basart
    and Eric Tang
    and Dawn Song
    and Jacob Steinhardt},
  journal={arXiv preprint arXiv:2103.03874},
  year={2021}
}
'''


lowerCAmelCase_			=							'''\
This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.
It first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.
'''


lowerCAmelCase_			=							r'''
Calculates accuracy after canonicalizing inputs.

Args:
    predictions: list of predictions to score. Each prediction
        is a string that contains natural language and LaTex.
    references: list of reference for each prediction. Each
        reference is a string that contains natural language
        and LaTex.
Returns:
    accuracy: accuracy after canonicalizing inputs
        (e.g., converting "1/2" to "\\frac{1}{2}")

Examples:
    >>> metric = datasets.load_metric("competition_math")
    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])
    >>> print(results)
    {\'accuracy\': 1.0}
'''



@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION			,	_KWARGS_DESCRIPTION							)
class 	lowerCAmelCase					(      datasets.Metric							):






       def 							__A (   self     ):
              return datasets.MetricInfo(
                  description=_DESCRIPTION    ,  citation=_CITATION    ,  inputs_description=_KWARGS_DESCRIPTION    ,  features=datasets.Features(
                      {
                          'predictions': datasets.Value('string'     ),
                          'references': datasets.Value('string'     ),
                      }     )    ,  homepage='https://github.com/hendrycks/math'    ,  codebase_urls=['https://github.com/hendrycks/math']    ,  )






       def 							__A (   self    ,  a__    ,  a__     ):
              _UpperCAmelCase     =     0.0
              for i, j in zip(__UpperCamelCase    ,  __UpperCamelCase     ):
                     n_correct += 1.0 if math_equivalence.is_equiv(__UpperCamelCase    ,  __UpperCamelCase     ) else 0.0
              _UpperCAmelCase     =     n_correct / len(__UpperCamelCase     )
              return {
                  "accuracy": accuracy,
              }













"""simple docstring"""






import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
    import optax
    from flax.training.common_utils import onehot

    from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
    from transformers.models.ta.modeling_flax_ta import shift_tokens_right



@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 	lowerCAmelCase					(      unittest.TestCase							):






       @slow
       def 							__A (   self     ):
              _UpperCAmelCase     =     FlaxMTaForConditionalGeneration.from_pretrained('google/mt5-small'     )
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained('google/mt5-small'     )

              _UpperCAmelCase     =     tokenizer('Hello there'    ,  return_tensors='np'     ).input_ids
              _UpperCAmelCase     =     tokenizer('Hi I am'    ,  return_tensors='np'     ).input_ids

              _UpperCAmelCase     =     shift_tokens_right(a__    ,  model.config.pad_token_id    ,  model.config.decoder_start_token_id     )

              _UpperCAmelCase     =     model(a__    ,  decoder_input_ids=a__     ).logits
              _UpperCAmelCase     =     optax.softmax_cross_entropy(a__    ,  onehot(a__    ,  logits.shape[-1]     )     ).mean()

              _UpperCAmelCase     =     -(labels.shape[-1] * loss.item())

              _UpperCAmelCase     =     -84.9_127
              self.assertTrue(abs(mtf_score - EXPECTED_SCORE     ) < 1E-4     )






