Upload from GitHub Actions: Add math benchmarks

datasets.json

@@ -249,6 +249,50 @@
         "implemented": false,
         "group": "Multitask Language Understanding"
     },
+    {
+        "name": "MGSM",
+        "author": "Google",
+        "author_url": "https://google.com",
+        "url": "https://huggingface.co/datasets/juletxara/mgsm",
+        "n_languages": 10,
+        "tasks": [
+            "math"
+        ],
+        "parallel": true,
+        "base": "MGSM",
+        "implemented": true,
+        "group": "Grade School Math"
+    },
+    {
+        "name": "AfriMGSM",
+        "author": "Masakhane",
+        "author_url": "https://www.masakhane.io",
+        "url": "https://huggingface.co/datasets/masakhane/afrimgsm",
+        "n_languages": 18,
+        "tasks": [
+            "math"
+        ],
+        "parallel": true,
+        "translation": "human",
+        "base": "MGSM",
+        "implemented": true,
+        "group": "Grade School Math"
+    },
+    {
+        "name": "GSM8K-X",
+        "author": "OpenGPT-X",
+        "author_url": "https://opengpt-x.de",
+        "url": "https://huggingface.co/datasets/openGPT-X/gsm8kx",
+        "n_languages": 20,
+        "tasks": [
+            "math"
+        ],
+        "parallel": true,
+        "translation": "machine",
+        "base": "MGSM",
+        "implemented": true,
+        "group": "Grade School Math"
+    },
     {
         "name": "FLEURS",
         "author": "Meta",
@@ -477,49 +521,6 @@
         "implemented": false,
         "group": "Adversarial Language Modelling"
     },
-    {
-        "name": "MGSM",
-        "author": "Google",
-        "author_url": "https://google.com",
-        "url": "https://huggingface.co/datasets/juletxara/mgsm",
-        "n_languages": 10,
-        "tasks": [
-            "math"
-        ],
-        "parallel": true,
-        "base": "MGSM",
-        "group": "Grade School Math"
-    },
-    {
-        "name": "AfriMGSM",
-        "author": "Masakhane",
-        "author_url": "https://www.masakhane.io",
-        "url": "https://huggingface.co/datasets/masakhane/afrimgsm",
-        "n_languages": 18,
-        "tasks": [
-            "math"
-        ],
-        "parallel": true,
-        "translation": "human",
-        "base": "MGSM",
-        "implemented": false,
-        "group": "Grade School Math"
-    },
-    {
-        "name": "GSM8K-X",
-        "author": "OpenGPT-X",
-        "author_url": "https://opengpt-x.de",
-        "url": "https://huggingface.co/datasets/openGPT-X/gsm8kx",
-        "n_languages": 20,
-        "tasks": [
-            "math"
-        ],
-        "parallel": true,
-        "translation": "machine",
-        "base": "MGSM",
-        "implemented": false,
-        "group": "Grade School Math"
-    },
     {
         "name": "WikiANN / PAN-X",
         "author": "Academic",

evals/backend.py

@@ -25,6 +25,7 @@ task_metrics = [
     "translation_to_bleu",
     "classification_accuracy",
     "mmlu_accuracy",
+    "mgsm_accuracy",
 ]
 
 

evals/datasets_/mgsm.py

@@ -0,0 +1,45 @@
+from datasets_.util import _get_dataset_config_names, _load_dataset
+from langcodes import Language, standardize_tag
+
+slug_mgsm = "juletxara/mgsm"
+tags_mgsm = {
+    standardize_tag(a, macro=True): a for a in _get_dataset_config_names(slug_mgsm)
+}
+slug_afrimgsm = "masakhane/afrimgsm"
+tags_afrimgsm = {
+    standardize_tag(a, macro=True): a for a in _get_dataset_config_names(slug_afrimgsm)
+}
+slug_gsm8kx = "Eurolingua/gsm8kx"
+tags_gsm8kx = {
+    standardize_tag(a, macro=True): a
+    for a in _get_dataset_config_names(slug_gsm8kx, trust_remote_code=True)
+}
+
+def parse_number(i):
+    if isinstance(i, int):
+        return i
+    try:
+        return int(i.replace(",", "").replace(".", ""))
+    except ValueError:
+        return None
+
+def load_mgsm(language_bcp_47, nr):
+    if language_bcp_47 in tags_mgsm.keys():
+        ds = _load_dataset(slug_mgsm, subset=tags_mgsm[language_bcp_47], split="test")
+        return slug_mgsm, ds[nr]
+    elif language_bcp_47 in tags_afrimgsm.keys():
+        ds = _load_dataset(
+            slug_afrimgsm, subset=tags_afrimgsm[language_bcp_47], split="test"
+        )
+        return slug_afrimgsm, ds[nr]
+    elif language_bcp_47 in tags_gsm8kx.keys():
+        row = _load_dataset(
+            slug_gsm8kx,
+            subset=tags_gsm8kx[language_bcp_47],
+            split="test",
+            trust_remote_code=True,
+        )[nr]
+        row["answer_number"] = row["answer"].split("####")[1].strip()
+        return slug_gsm8kx, row
+    else:
+        return None, None

evals/main.py

@@ -16,12 +16,9 @@ n_models = 35
 
 
 async def evaluate():
-    # save up-to-date info on models and languages
-    args = dict(orient="records", indent=2, force_ascii=False)
-    pd.DataFrame(models).to_json("models.json", **args)
-    pd.DataFrame(languages).to_json("languages.json", **args)
     print("running evaluations")
     old_results = pd.read_json("results.json")
+    old_models = pd.read_json("models.json")
     # get all combinations of model, language and task
     combis = [
         (model, lang.bcp_47, task_name)
@@ -41,6 +38,7 @@ async def evaluate():
     ]
     results = await tqdm_asyncio.gather(*results, miniters=1)
     results = [r for group in results for r in group]
+    args = dict(orient="records", indent=2, force_ascii=False)
     if results:
         # aggregate results
         results = pd.DataFrame(results)
@@ -53,6 +51,11 @@ async def evaluate():
         results = pd.concat([old_results, results])
         results = results.sort_values(by=["model", "bcp_47", "task", "metric"])
         results.to_json("results.json", **args)
+    # save up-to-date info on models and languages
+    all_models = pd.concat([old_models, pd.DataFrame(models)])
+    all_models = all_models.drop_duplicates(subset=["id"]).sort_values(by=["id"])
+    all_models.to_json("models.json", **args)
+    pd.DataFrame(languages).to_json("languages.json", **args)
 
 
 if __name__ == "__main__":

evals/tasks.py

@@ -1,10 +1,12 @@
 import random
 from functools import partial
+from textwrap import dedent
 
 import evaluate
 import pandas as pd
 import sentencepiece as spm
 from datasets_.flores import flores_sentences
+from datasets_.mgsm import load_mgsm, parse_number
 from datasets_.mmlu import load_mmlu
 from languages import languages, script_name
 from models import complete, transcribe
@@ -247,6 +249,44 @@ async def mmlu_and_evaluate(model, language_bcp_47, nr):
     ]
 
 
+async def mgsm_and_evaluate(model, language_bcp_47, nr):
+    system_prompt = """
+    Solve the math problem. Use reasoning, and finally give the answer as a number.
+    Response format: <reasoning> #### <number>
+    """
+    system_prompt = dedent(system_prompt).strip()
+    ds_slug, question = load_mgsm(language_bcp_47, nr)
+    if not question:
+        return []
+    response = await complete(
+        model=model,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": question["question"]},
+        ],
+        temperature=0,
+        max_tokens=1024,
+    )
+    number = response.split("####")
+    if len(number) == 2:
+        accuracy = int(
+            parse_number(number[1].strip()) == parse_number(question["answer_number"])
+        )
+    else:
+        accuracy = 0
+
+    return [
+        {
+            "model": model,
+            "bcp_47": language_bcp_47,
+            "task": "mgsm",
+            "metric": "accuracy",
+            "score": accuracy,
+            "sentence_nr": nr,
+        }
+    ]
+
+
 async def transcribe_and_evaluate(model, language_bcp_47, nr):
     language = languages[languages["bcp_47"] == language_bcp_47].iloc[0]
     fleurs = pd.read_csv(
@@ -284,5 +324,6 @@ tasks = {
     "classification": classify_and_evaluate,
     # "mlm": mlm_and_evaluate,
     "mmlu": mmlu_and_evaluate,
+    "mgsm": mgsm_and_evaluate,
     # "asr": transcribe_and_evaluate,
 }

frontend/src/components/ModelTable.js

@@ -5,7 +5,6 @@ import { MultiSelect } from 'primereact/multiselect'
 import { useState, useEffect } from 'react'
 import Medal from './Medal'
 import { Slider } from 'primereact/slider'
-import ScoreField from './ScoreField'
 import ScoreColumns from './ScoreColumns'
 const ModelTable = ({ data }) => {
   const [filters, setFilters] = useState({

frontend/src/components/ScoreColumns.js

@@ -64,7 +64,7 @@ const ScoreColumns = [
   //   />,
   <Column
     field='mmlu_accuracy'
-    header='MMLU'
+    header='Q&A'
     headerTooltip='Question Answering performance (accuracy on a sample of multilingual versions of the MMLU benchmark)'
     sortable
     body={scoreBodyTemplate('mmlu_accuracy', {
@@ -72,7 +72,18 @@ const ScoreColumns = [
       maxScore: 1
     })}
     style={{ minWidth: '5rem', maxWidth: '10rem' }}
-  />
+  />,
+  <Column
+    field='mgsm_accuracy'
+    header='Math'
+    headerTooltip='Math Problem Solving performance (accuracy on a sample of the MGMS benchmark)'
+    sortable
+    body={scoreBodyTemplate('mgsm_accuracy', {
+      minScore: 0,
+      maxScore: 1
+    })}
+    style={{ minWidth: '5rem', maxWidth: '10rem' }}
+  />,
 ]
 
 export default ScoreColumns