Test

.gitignore

@@ -5,7 +5,6 @@ __pycache__/
 .ipynb_checkpoints
 *ipynb
 .vscode/
-.idea/
 
 eval-queue/
 eval-results/

README.md

@@ -1,5 +1,5 @@
 ---
-title: Eval Leaderboard
+title: Leaderboard Test
 emoji: 🥇
 colorFrom: green
 colorTo: indigo

app.py

@@ -1,121 +1,103 @@
 import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
-import json
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
 
 from src.about import (
-    REPRODUCIBILITY_TEXT,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
-    ABOUT_TEXT,
+    LLM_BENCHMARKS_TEXT,
     TITLE,
 )
-from src.display.css_html_js import custom_css, custom_js
-from src.display.formatting import make_clickable_field
-
-def build_leaderboard(type):
-    with open('data/results.json', 'r') as f:
-        results = json.load(f)
-
-    with open('data/tasks.json', 'r') as f:
-        tasks = json.load(f)
-
-    # Filter tasks based on type
-    filtered_tasks = {k: v for k, v in tasks.items() if v['type'] == type}
-
-    data = []
-    for model_name, model_data in results.items():
-        # For agentic type, skip models that have all null values for agentic tasks
-        if type == "agentic":
-            has_agentic_results = any(
-                model_data['results'].get(task, {}).get(tasks[task]['metric']) is not None 
-                for task in filtered_tasks
-            )
-            if not has_agentic_results:
-                continue
-
-        model_sha = model_data["config"]["model_sha"]
-        model_name = model_data["config"]["model_name"]
-        row = {
-            'Model': make_clickable_field(model_name, model_sha)
-        }
-        
-        for dataset, metrics in model_data['results'].items():
-            # Only include metrics for tasks of the specified type
-            if dataset in filtered_tasks:
-                value = next(iter(metrics.values()))
-                log_url = metrics.get('log_url')
-                # Use display name from tasks.json instead of raw dataset name
-                display_name = filtered_tasks[dataset]['display_name']
-                # Round non-null values to 2 decimal places and make clickable if log_url exists
-                if value is not None:
-                    value = round(value*100, 2)
-                    if log_url:
-                        value = make_clickable_field(value, log_url)
-                row[display_name] = value
-        data.append(row)
-
-    results_df = pd.DataFrame(data)
-    
-    # Round all numeric columns to 2 decimal places
-    numeric_cols = results_df.select_dtypes(include=['float64', 'float32']).columns
-    results_df[numeric_cols] = results_df[numeric_cols].round(2)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    COLS,
+    ST_BENCHMARK_COLS,
+    AGENTIC_BENCHMARK_COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    ModelType,
+    fields,
+    WeightType,
+    Precision
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, TASK_NAME_INVERSE_MAP
+from src.submission.submit import add_new_eval
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+### Space initialisation
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
 
-    # Fill null values with "-"
-    results_df = results_df.fillna("--")
 
-    if type == "agentic":
-        # Include agent column as second column after Model
-        results_df.insert(1, 'Agent', make_clickable_field('Basic Agent', 'https://inspect.ai-safety-institute.org.uk/agents.html#sec-basic-agent'))
+ST_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, ST_BENCHMARK_COLS)
+AGENTIC_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, AGENTIC_BENCHMARK_COLS)
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+def init_leaderboard(dataframe, benchmark_type):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
     
+    AutoEvalColumnSubset = [c for c in fields(AutoEvalColumn) if ((c.name=="Model") or (TASK_NAME_INVERSE_MAP.get(c.name, dict()).get("type", "")==benchmark_type))]
+
     return gr.components.Dataframe(
-        value=results_df,
-        datatype=["html" for _ in results_df.columns],
-        column_widths=["250px" if c == "Model" else "150px" for c in results_df.columns],
+        value=dataframe,
+        datatype=[c.type for c in AutoEvalColumnSubset],
+        column_widths=["150px" if c.name != "Model" else "250px" for c in AutoEvalColumnSubset],
         wrap=False,
     )
 
 
-black_logo_path = "src/assets/logo-icon-black.png"
-white_logo_path = "src/assets/logo-icon-white.png"
-
-demo = gr.Blocks(
-    css=custom_css,
-    js=custom_js,
-    theme=gr.themes.Default(primary_hue=gr.themes.colors.pink),
-    fill_height=True,
-    fill_width=True,
-)
+demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(f"""
-    <div id="page-header">
-        <div id="header-container">
-            <div id="left-container">
-                <img id="black-logo" src="/gradio_api/file={black_logo_path}">
-                <img id="white-logo" src="/gradio_api/file={white_logo_path}">
-            </div>
-            <div id="centre-container">
-                <h1 style="margin-bottom: 0.25rem;">{TITLE}</h1>
-                <p style="color:#eb088a; margin:0; font-size:1.2rem;">Explore Interactive Results &amp; Traces</p>
-            </div>
-            <div id="right-container">
-            </div>
-        </div>
+    <div style="text-align:center; margin-bottom:1rem;">
+        <h1 style="margin-bottom: 0.25rem;">{TITLE}</h1>
+        <p style="color:#eb088a; margin:0; font-size:1.2rem;">Performance Insights &amp; Comparison</p>
     </div>
     """)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="intro-text", sanitize_html=False)
-
-    with gr.Tabs(elem_classes=["leaderboard-table", "tab-buttons"]) as tabs:
-        with gr.TabItem("Base Benchmarks", elem_classes="llm-benchmark-tab-table", id=0):
-            build_leaderboard("base")
+    # gr.HTML(TITLE)
+    with gr.Group(elem_classes="intro-block"):
+        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("Agentic Benchmarks", elem_classes="llm-benchmark-tab-table", id=1):
-            build_leaderboard("agentic")
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("Base Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard = init_leaderboard(ST_LEADERBOARD_DF, "base")
 
-        with gr.TabItem("About", elem_classes="llm-benchmark-tab-table", id=2):
-            gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text", sanitize_html=False)
+        with gr.TabItem("Agentic Benchmark", elem_id="llm-benchmark-tab-table", id=1):
+            leaderboard = init_leaderboard(AGENTIC_LEADERBOARD_DF, "agentic")
 
-        # with gr.TabItem("Reproducibility", elem_classes="llm-benchmark-tab-table", id=3):
-        #     gr.Markdown(REPRODUCIBILITY_TEXT, elem_classes="markdown-text", sanitize_html=False)
+        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-assets = [black_logo_path, white_logo_path]
-demo.launch(allowed_paths=assets)
 
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

create_log_file_map.py

@@ -0,0 +1,36 @@
+import json
+import os
+
+from collections import defaultdict
+
+
+def main():
+
+    base_bm_input_path = "./base_benchmarking_logs"
+    agentic_bm_input_path = "./agentic_benchmarking_logs"
+
+    log_file_map = defaultdict()
+
+    for model_name in os.listdir(base_bm_input_path):
+        log_file_map[model_name] = defaultdict(str)
+        if os.path.isdir(os.path.join(base_bm_input_path, model_name)):
+            for task_log_file in os.listdir(os.path.join(base_bm_input_path, model_name)):
+                with open(os.path.join(base_bm_input_path, model_name, task_log_file), "r") as f:
+                    result = json.load(f)
+                task_name = result["eval"]["task"].split("/")[-1]
+                log_file_map[model_name][task_name] = task_log_file
+
+    for model_name in os.listdir(agentic_bm_input_path):
+        if os.path.isdir(os.path.join(agentic_bm_input_path, model_name)):
+            for task_log_file in os.listdir(os.path.join(agentic_bm_input_path, model_name)):
+                with open(os.path.join(agentic_bm_input_path, model_name, task_log_file), "r") as f:
+                    result = json.load(f)
+                task_name = result["eval"]["task"].split("/")[-1]
+                log_file_map[model_name][task_name] = task_log_file
+                
+    with open("./inspect_log_file_names.json", "w") as f:
+        json.dump(log_file_map, f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

data/results.json

@@ -1,948 +0,0 @@
-{
-    "DeepSeek-R1": {
-        "config": {
-            "model_name": "DeepSeek-R1",
-            "model_sha": "https://api-docs.deepseek.com/news/news250120",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "mmlu_pro": {
-                "accuracy": 0.8382646276595744,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-12T11-02-35-05-00_mmlu-pro_BhD89DYN9KM3k4weSDfaQK.eval"
-            },
-            "humaneval": {
-                "mean": 0.9567901234567902,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-03T11-45-22-05-00_humaneval_hnkHWYqrb5HxiBt2CWzCnq.eval"
-            },
-            "math": {
-                "accuracy": 0.9272,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-11T11-38-10-05-00_math_ZYFSqsWsmP5kLRLHEMWULU.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.954510993176649,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-02T16-28-05-05-00_gsm8k_YMw6WiZkgTBQ54z5UHtDDX.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9667235494880546,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-01-30T15-42-39-05-00_arc-challenge_CviW9ro6rKBbctkwJzQstp.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.9179163378058406,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-04T00-25-12-05-00_winogrande_NPgTbtqom2QSPKxeThWrdZ.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9873737373737373,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.7045454545454546,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/DeepSeek-R1/index.html?log_file=logs/logs/2025-02-11T11-37-45-05-00_gpqa-diamond_MwnVeLwyuiEAALr3M5q3dn.eval"
-            },
-            "drop": {
-                "mean": null,
-                "log_url": null
-            },
-            "hellaswag": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "ifeval": {
-                "final_acc": null,
-                "log_url": null
-            },
-            "mmlu": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "mmmu_open": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gaia": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "agentharm": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "agentharm_benign": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "swe_bench": {
-                "mean": null,
-                "log_url": null
-            }
-        }
-    },
-    "Meta-Llama-3.1-70B-Instruct": {
-        "config": {
-            "model_name": "Meta-Llama-3.1-70B-Instruct",
-            "model_sha": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "hellaswag": {
-                "accuracy": 0.869946225851424,
-                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T00-45-54-04-00_hellaswag_BKfQG9yGAr383MGnooMLBH.eval"
-            },
-            "drop": {
-                "mean": 0.8811263765076035,
-                "log_url": "https://storage.googleapis.com/inspect-evals/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T21-01-02-04-00_drop_LzAWvLWkNrNKu5qf56wXRo.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.4318181818181818,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T23-41-39-04-00_gpqa-diamond_TdLdYmVM6GCVMAECcXkuhj.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.8666140489344909,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T09-20-56-04-00_winogrande_WnUgkSRhSMvh3zUjnuJWQZ.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9469294920394238,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T00-03-31-04-00_gsm8k_bKsUfCAfcmBCeryboNaLoX.eval"
-            },
-            "math": {
-                "accuracy": 0.6004,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T02-34-50-04-00_math_2xiNcrGih26uzJdG4q88bM.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.8604907201780166,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T02-29-32-04-00_ifeval_Dwh3CF2ZYFrvw7UcTwrsvK.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9445392491467577,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T20-58-56-04-00_arc-challenge_oFL5wFjT7KwNFhMFfe72JN.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9823232323232324,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-29T20-53-12-04-00_arc-easy_UXzR7cDeNteP39NoXUYnhm.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.6688829787234043,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T06-11-16-04-00_mmlu-pro_oQiEBJdeKtEEt4cm9KL7uy.eval"
-            },
-            "humaneval": {
-                "mean": 0.7865853658536586,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T02-28-25-04-00_humaneval_KcJV2rHuHJ2JLxijihEkcW.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.8033755875231449,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Meta-Llama-3.1-70B-Instruct/index.html?log_file=logs/logs/2024-10-30T03-51-50-04-00_mmlu_6SNjs2QmPRvqGnvbnNtaqb.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "mmmu_open": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gaia": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "agentharm": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "agentharm_benign": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "swe_bench": {
-                "mean": null,
-                "log_url": null
-            }
-        }
-    },
-    "Mistral-Large-Instruct-2407": {
-        "config": {
-            "model_name": "Mistral-Large-Instruct-2407",
-            "model_sha": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "drop": {
-                "mean": 0.7424257996853698,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T01-56-12-04-00_drop_NtvuCoU2LoMbH8DztcCTen.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.8285172231900246,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T06-30-16-04-00_ifeval_TLkvCSFEWo4PLv6hAha7YB.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.8035892323030908,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T07-21-48-04-00_mmlu_YnUhmHoStr3WuJdchWmNPt.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.4734848484848485,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T04-22-52-04-00_gpqa-diamond_SuZUZxGdqS2ZecbLRNkKd4.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9378316906747536,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T04-28-49-04-00_gsm8k_5tQp9tbwUMj6NpjNKCAfVm.eval"
-            },
-            "math": {
-                "accuracy": 0.6574,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T06-33-09-04-00_math_2CmjBedAfUxqvmcHRdBgyB.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9852693602693603,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T01-48-39-04-00_arc-easy_YbfuBT3usZXt2xgZkkR5dq.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.6942320478723404,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T09-41-25-04-00_mmlu-pro_fyYT4aabPesfY5TpzFMPnd.eval"
-            },
-            "humaneval": {
-                "mean": 0.8658536585365854,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T06-29-24-04-00_humaneval_nu8SUSGekKJWB8HLKDigYK.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.9047998406691894,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T04-50-00-04-00_hellaswag_ZzQoZ6gkRQsTzMhQr7GYNn.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9436860068259386,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T01-54-13-04-00_arc-challenge_WfQRhMkFcywefpU46isBVP.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.8547750591949487,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/Mistral-Large-Instruct-2407/index.html?log_file=logs/logs/2024-10-31T11-57-58-04-00_winogrande_TP3UGwpp37Dyv6ks9Ty5Hk.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "mmmu_open": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gaia": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "agentharm": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "agentharm_benign": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "swe_bench": {
-                "mean": null,
-                "log_url": null
-            }
-        }
-    },
-    "c4ai-command-r-plus": {
-        "config": {
-            "model_name": "Command R+",
-            "model_sha": "https://huggingface.co/CohereForAI/c4ai-command-r-plus"
-        },
-        "results": {
-            "ifeval": {
-                "final_acc": 0.7779591483929307,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T17-23-04-04-00_ifeval_RGucUMwdGmUnRpqyMTZTzW.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.7490134175217048,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T14-42-18-04-00_winogrande_bY8yg7aRR5dCCK7NDCZEcc.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.8506825938566553,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-29T17-30-03-04-00_arc-challenge_XB7LURXEGaxskWuLtYwdnW.eval"
-            },
-            "drop": {
-                "mean": 0.743557420031463,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T12-06-30-04-00_drop_itY9cLiYAW2BF7NTeDceNd.eval"
-            },
-            "math": {
-                "accuracy": 0.2626,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T17-26-34-04-00_math_kohBUMpMFuMsR4jz4vUNWM.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.3194444444444444,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-29T22-47-45-04-00_gpqa-diamond_JKpb6ya4pec9hh7uovPPCZ.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.441156914893617,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-31T01-11-38-04-00_mmlu-pro_gZVAuy3zMKR23BieM5PqAX.eval"
-            },
-            "humaneval": {
-                "mean": 0.6219512195121951,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T17-22-23-04-00_humaneval_5ByPqUhoofSbKgvsUQNFCX.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.7816527672479151,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T15-03-35-04-00_gsm8k_QxbfbriJsKGQAg96JyjkoT.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.7954590718980283,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T15-18-17-04-00_hellaswag_UYyBTR6N8VJnKRmnbCrB8N.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.695128899017234,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-30T21-55-26-04-00_mmlu_JUPPLTzfe3Kme6UuorPTqg.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9377104377104377,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/c4ai-command-r-plus/index.html?log_file=logs/logs/2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.eval"
-            }
-        }
-    },
-    "claude-3-5-sonnet-20241022": {
-        "config": {
-            "model_name": "Claude-3.5-Sonnet",
-            "model_sha": "https://www.anthropic.com/claude/sonnet",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "mmmu_multiple_choice": {
-                "accuracy": 0.6481700118063755,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-21T11-20-03-05-00_mmmu-multiple-choice_CWhKvGdoFo6pdHhDyi9GNm.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.7762632978723404,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T19-01-05-05-00_mmlu-pro_3vi84or97gQupuj5sT6vgZ.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.9228241386178052,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T15-09-33-05-00_hellaswag_QXqFxojvSToMu8ckHEMLkB.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.6098484848484849,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T13-56-36-05-00_gpqa-diamond_eg4gFaMRENjnnYvQNtSB59.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9620924943138741,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T14-23-25-05-00_gsm8k_nHB8Z4uZAwRAZFYpKmTptA.eval"
-            },
-            "mmmu_open": {
-                "accuracy": 0.41509433962264153,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-21T11-24-21-05-00_mmmu-open_SSjv3Dq9gZkEEUnvJUd5xf.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9915824915824916,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T10-06-24-05-00_arc-easy_oBReQZQM5SAwMMD2jFshPb.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9692832764505119,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T10-12-11-05-00_arc-challenge_X8i6caCzkcQo5AT5zXkXso.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.8665432274604757,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T15-16-51-05-00_mmlu_NFDs2kxmh3kQEbpbd8sz3w.eval"
-            },
-            "math": {
-                "accuracy": 0.7942,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T12-29-54-05-00_math_NvNQU58M8r3fpiwPGnvq8h.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.8958114469607309,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T11-28-44-05-00_ifeval_fmWxch4ZjbmYCST6yUZsdV.eval"
-            },
-            "humaneval": {
-                "mean": 0.9451219512195121,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T11-26-12-05-00_humaneval_kUASiaNd9uZfWvCwYHhdF5.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.9021310181531176,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T22-09-41-05-00_winogrande_mSWGAKg75E5RP79KWizvb9.eval"
-            },
-            "drop": {
-                "mean": 0.8977608809648663,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T10-15-15-05-00_drop_Z9A2Y84HYponNxnzNT9TNq.eval"
-            },
-            "gaia": {
-                "accuracy": 0.3381818181818182,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-12T23-57-37-05-00_claude-3-5-sonnet_gaia_merged.eval"
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": 0.8556962025316455,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-11T02-47-45-05-00_claude-3-5-sonnet_gdm-intercode-ctf_merged.eval"
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": 0.6153846153846154,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-11T07-41-14+00-00_claude-3-5-sonnet_gdm-in-house-ctf.eval"
-            },
-            "agentharm": {
-                "avg_score": 0.14767992424242424,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-15T08-05-14-08-00_agentharm_VJGhWKLrVLdQczBZVgCXHc.eval"
-            },
-            "agentharm_benign": {
-                "avg_score": 0.800704570051161,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-21T15-09-48-08-00_agentharm-benign_A3uBBWNvv88P5BsgqwFCfg.eval"
-            },
-            "swe_bench": {
-                "mean": 0.0672,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/claude-3-5-sonnet-20241022/index.html?log_file=logs/logs/2025-01-16T18-56-55+00-00_anthropic-claude-3-5-sonnet.eval"
-            }
-        }
-    },
-    "gemini-1.5-flash": {
-        "config": {
-            "model_name": "Gemini-1.5-Flash",
-            "model_sha": "https://deepmind.google/technologies/gemini/flash",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "gpqa_diamond": {
-                "accuracy": 0.40404040404040403,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9308873720136519,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T17-34-51-04-00_arc-challenge_FbGgLswBZbRE4EhWiMyRt6.eval"
-            },
-            "math": {
-                "accuracy": 0.452,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.eval"
-            },
-            "mmmu_open": {
-                "accuracy": 0.16981132075471697,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2025-01-20T23-13-27-05-00_mmmu-open_GWi6XNYUSLq99BdabtScGm.eval"
-            },
-            "drop": {
-                "mean": 0.751044572627163,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T17-36-25-04-00_drop_6TzJGqqEkpFUCxGD4QejV6.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.5993184840425532,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.7681296737102001,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T21-15-06-04-00_ifeval_nYs9KujQMQjcpbpbLtVx8G.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.8557060346544513,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T19-35-34-04-00_hellaswag_2SAz3cvMpDxFaApdHDR3s4.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.7884767166535123,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-31T00-59-07-04-00_winogrande_Ci55vHvbGGW38zVpMCwtWa.eval"
-            },
-            "humaneval": {
-                "mean": 0.7439024390243902,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.984006734006734,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T17-29-56-04-00_arc-easy_XcEzqqPqJsRV29NqYDfnNo.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.8582259287338894,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-30T19-32-39-04-00_gsm8k_nLSssETKDDWNktAFWnVwfv.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.7714713003845606,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2024-10-31T10-49-43-04-00_mmlu_oGb9mspeGbYS2gfbkknskN.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": 0.5702479338842975,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-flash/index.html?log_file=logs/logs/2025-01-20T23-10-01-05-00_mmmu-multiple-choice_c5rLkrXkV83udX6DVJui5F.eval"
-            },
-            "gaia": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "agentharm": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "agentharm_benign": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "swe_bench": {
-                "mean": null,
-                "log_url": null
-            }
-        }
-    },
-    "gemini-1.5-pro": {
-        "config": {
-            "model_name": "Gemini-1.5-Pro",
-            "model_sha": "https://deepmind.google/technologies/gemini/pro",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "mmlu": {
-                "accuracy": 0.8467454778521578,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T16-56-26-05-00_mmlu_Z9KrcK7x4ZLAR5nJ9JaVUe.eval"
-            },
-            "humaneval": {
-                "mean": 0.8719512195121951,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-43-07-05-00_humaneval_5JBjtymGtK23qwVKxqidhV.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": 0.6304604486422668,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2025-01-20T23-16-04-05-00_mmmu-multiple-choice_NLmxmHYt6CJymRVVa5UsbD.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.7563996010638298,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T20-13-09-05-00_mmlu-pro_Hv2ujvKLV6H7ZwQu2q8LNw.eval"
-            },
-            "math": {
-                "accuracy": 0.852,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T15-48-46-05-00_math_9DAZmGEfhpa3nUcmMAwqZe.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9877946127946128,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-31-43-05-00_arc-easy_eGxYWywpLuREcaCKvHa8Uk.eval"
-            },
-            "mmmu_open": {
-                "accuracy": 0.3584905660377358,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2025-01-20T23-19-25-05-00_mmmu-open_CDbtEQ7tjs5zkj4ScBbzod.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9613343442001516,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T15-15-26-05-00_gsm8k_cTebw3ugfrVz3dyPwxtdUZ.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.5782828282828283,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-05T09-56-31-05-00_gpqa-diamond_FBq2bnoyGYQ3NF96xQw8iy.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.8982344623377084,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-43-32-05-00_ifeval_mSwZ7AwA7akj5PjZbQMjgC.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.8768745067087609,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-40-46-05-00_winogrande_5SmD6rx47zmZvHHkQSSfHK.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9633105802047781,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-37-36-05-00_arc-challenge_5VVApyQD22QpJoMm53EMdU.eval"
-            },
-            "drop": {
-                "mean": 0.8800912427897221,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-04T12-44-32-05-00_drop_9dzPKVJojSVsxmiBFnej2m.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.9123680541724756,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gemini-1.5-pro/index.html?log_file=logs/logs/2024-11-05T13-14-31-05-00_hellaswag_N98eeftuY2pucRtgpUYk5m.eval"
-            },
-            "gaia": {
-                "accuracy": 0.13818181818181818,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T15-33-29-05-00_gemini-1.5-pro_gaia_merged.eval"
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": 0.5291139240506328,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T23-59-58+00-00_gemini-1.5-pro_gdm-intercode-ctf_merged.eval"
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": 0.23076923076923078,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-22T03-42-16+00-00_gemini-1.5-pro_gdm-in-house-ctf.eval"
-            },
-            "agentharm": {
-                "avg_score": 0.2898649645808737,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T12-45-43-08-00_agentharm_VmD26soLwmRgWPo3hpRHBr.eval"
-            },
-            "agentharm_benign": {
-                "avg_score": 0.5961489079102715,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-21T13-18-51-08-00_agentharm-benign_gP3pQPxAuCtFLiHzt2Egt7.eval"
-            },
-            "swe_bench": {
-                "mean": 0.004,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gemini-1.5-pro-002/index.html?log_file=logs/logs/2025-01-22T03-00-08+00-00_google-gemini-1.5-pro_swe.eval"
-            }
-        }
-    },
-    "gpt-4o": {
-        "config": {
-            "model_name": "GPT-4o",
-            "model_sha": "https://openai.com/index/hello-gpt-4o",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "gpqa_diamond": {
-                "accuracy": 0.51010101010101,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T03-29-33-04-00_gpqa-diamond_nFmRv5MJiYjHjezmq4V6Va.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9633105802047781,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T01-45-55-04-00_arc-challenge_nrsPPxh4DpzgLPQDFdcfVp.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9446550416982562,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T03-31-16-04-00_gsm8k_jVXeSvHowbietZCFsFYCwB.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.8435408061529697,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T10-49-43-04-00_mmlu_GarLpfQFSpM3C22nbbGp54.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.8780386042367585,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T05-00-11-04-00_ifeval_jxreUu8JqRdkrcHP4E3hLR.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.7450964095744681,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T06-59-42-04-00_mmlu-pro_EuAKDwAWSfNVpqyyqrf2Ba.eval"
-            },
-            "mmmu_open": {
-                "accuracy": 0.3584905660377358,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2025-01-20T23-07-46-05-00_mmmu-open_d3Q2HvuPZzEX6FAM4NBhnp.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.9013417521704814,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T09-02-03-04-00_winogrande_44kKF7M9mKoqVC7ixZVXuq.eval"
-            },
-            "drop": {
-                "mean": 0.7511693759832198,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T01-47-20-04-00_drop_3gxDcn6vUoR3nvHX9BcSq4.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9915824915824916,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T01-41-34-04-00_arc-easy_nUavRHdiRVfrxo6dmCPadh.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": 0.5903187721369539,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2025-01-20T23-03-21-05-00_mmmu-multiple-choice_eoycAFLMirSqiURdXmBP2e.eval"
-            },
-            "humaneval": {
-                "mean": 0.9085365853658537,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T04-59-42-04-00_humaneval_nmJcd84CcNKjWS8fBfMbZM.eval"
-            },
-            "math": {
-                "accuracy": 0.7054,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T05-01-22-04-00_math_cDSpKPp3nLrFy8uYfYKEbM.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.924317864967138,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o/index.html?log_file=logs/logs/2024-10-31T03-33-47-04-00_hellaswag_JNnnPuz3dhZRpyXzizMUBF.eval"
-            },
-            "gaia": {
-                "accuracy": 0.16606060606060608,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-13T15-53-22+00-00_gpt-4o_gaia_merged.eval"
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": 0.6379746835443038,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-08T10-06-29-05-00_gpt-4o_gdm-intercode-ctf_merged.eval"
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": 0.23076923076923078,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-11T07-02-14+00-00_gpt-4o_gdm-in-house-ctf.eval"
-            },
-            "agentharm": {
-                "avg_score": 0.49953844451003543,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-07T16-34-15-08-00_agentharm_UfSoyHEAH2E5RVdrPVUemy.eval"
-            },
-            "agentharm_benign": {
-                "avg_score": 0.8249433048012594,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-21T13-45-18-08-00_agentharm-benign_8DhGJqEAvw6o8uCv4a4dVz.eval"
-            },
-            "swe_bench": {
-                "mean": 0.012,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/gpt-4o-2024-08-06/index.html?log_file=logs/logs/2025-01-14T23-09-10+00-00_openai-gpt-4o_swe.eval"
-            }
-        }
-    },
-    "gpt-4o-mini": {
-        "config": {
-            "model_name": "GPT-4o-mini",
-            "model_sha": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "drop": {
-                "mean": 0.8065915049816466,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T17-36-25-04-00_drop_6TzJGqqEkpFUCxGD4QejV6.eval"
-            },
-            "humaneval": {
-                "mean": 0.8597560975609756,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.3838383838383838,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.eval"
-            },
-            "mmmu_open": {
-                "accuracy": 0.18867924528301888,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2025-01-20T23-13-27-05-00_mmmu-open_GWi6XNYUSLq99BdabtScGm.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9249146757679181,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T17-34-51-04-00_arc-challenge_FbGgLswBZbRE4EhWiMyRt6.eval"
-            },
-            "mmlu": {
-                "accuracy": 0.7698333570716422,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-31T10-49-43-04-00_mmlu_oGb9mspeGbYS2gfbkknskN.eval"
-            },
-            "hellaswag": {
-                "accuracy": 0.8750248954391555,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T19-35-34-04-00_hellaswag_2SAz3cvMpDxFaApdHDR3s4.eval"
-            },
-            "ifeval": {
-                "final_acc": 0.8419061423689144,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T21-15-06-04-00_ifeval_nYs9KujQMQjcpbpbLtVx8G.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": 0.5395513577331759,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2025-01-20T23-10-01-05-00_mmmu-multiple-choice_c5rLkrXkV83udX6DVJui5F.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9793771043771043,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T17-29-56-04-00_arc-easy_XcEzqqPqJsRV29NqYDfnNo.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.7529597474348856,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-31T00-59-07-04-00_winogrande_Ci55vHvbGGW38zVpMCwtWa.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.6396276595744681,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.eval"
-            },
-            "math": {
-                "accuracy": 0.633,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9181197877179682,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/gpt-4o-mini/index.html?log_file=logs/logs/2024-10-30T19-32-39-04-00_gsm8k_nLSssETKDDWNktAFWnVwfv.eval"
-            },
-            "gaia": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "agentharm": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "agentharm_benign": {
-                "avg_score": null,
-                "log_url": null
-            },
-            "swe_bench": {
-                "mean": null,
-                "log_url": null
-            }
-        }
-    },
-    "o1": {
-        "config": {
-            "model_name": "o1",
-            "model_sha": "https://openai.com/o1",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "winogrande": {
-                "accuracy": 0.9392265193370166,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T16-46-06-05-00_winogrande_YUtAdEsForRffqe4Sm3wtR.eval"
-            },
-            "humaneval": {
-                "mean": 0.9695121951219512,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T14-59-12-05-00_humaneval_RRL8GMy9NakTxUHsDVWNng.eval"
-            },
-            "mmmu_open": {
-                "accuracy": 0.6981132075471698,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T22-48-09-05-00_mmmu-open_oBzxJBYbvnktbbAwhoCrYK.eval"
-            },
-            "math": {
-                "accuracy": 0.959,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T15-03-22-05-00_math_6BbvHFF8hLMsVYozyNLbyQ.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9911616161616161,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T11-29-26-05-00_arc-easy_DFbir4BdgQDbKd52r7tRKR.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9786689419795221,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T11-44-42-05-00_arc-challenge_PsWXaBqrgv3EcTZC55gRzJ.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9416224412433661,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T12-56-38-05-00_gsm8k_iD8275qeyNTgX523pn45bF.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.7550505050505051,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-17T11-53-53-05-00_gpqa-diamond_EJV7ULFSQLRoFTEqsv3t6q.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.8447473404255319,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T14-02-37-05-00_mmlu-pro_EvDzvqaahQwhv6fJovN4BT.eval"
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": 0.8063754427390791,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o1/index.html?log_file=logs/logs/2025-01-20T21-04-57-05-00_mmmu-multiple-choice_MctxjookaeTLCL8KpUeazT.eval"
-            },
-            "drop": {
-                "mean": null,
-                "log_url": null
-            },
-            "hellaswag": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "ifeval": {
-                "final_acc": null,
-                "log_url": null
-            },
-            "mmlu": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gaia": {
-                "accuracy": 0.41090909090909084,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-22T13-42-00-05-00_o1_gaia_merged.eval"
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": 0.8481012658227849,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-22T20-46-35+00-00_o1_gdm-intercode-ctf_merged.eval"
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": 0.46153846153846156,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-22T05-52-25+00-00_o1_gdm-in-house-ctf.eval"
-            },
-            "agentharm": {
-                "avg_score": 0.08782061688311688,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-21T09-05-42-08-00_agentharm_UGDq2yJeLAnPH6p7FgDgD8.eval"
-            },
-            "agentharm_benign": {
-                "avg_score": 0.7235176849665487,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-21T18-20-15-08-00_agentharm-benign_bkW2Bf5xLyDQdNtfLdjCpJ.eval"
-            },
-            "swe_bench": {
-                "mean": 0.0036,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-01-21T17-42-11+00-00_openai-o1_swe.eval "
-            }
-        }
-    },
-    "o3-mini": {
-        "config": {
-            "model_name": "o3-mini",
-            "model_sha": "https://openai.com/index/openai-o3-mini",
-            "model_dtype": "torch.float16"
-        },
-        "results": {
-            "math": {
-                "accuracy": 0.9691320905993185,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T18-33-30-05-00_math_86Gx8n4BxhpyfaSHmRcCUm.eval"
-            },
-            "humaneval": {
-                "mean": 0.9817073170731707,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T20-58-48-05-00_humaneval_Dkod7CS9RmbbogYx9aEXtx.eval"
-            },
-            "mmlu_pro": {
-                "accuracy": 0.7924606807023383,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T19-49-27-05-00_mmlu-pro_jz9woKfdKt8VMzqNFsy7kY.eval"
-            },
-            "gpqa_diamond": {
-                "accuracy": 0.7365319865319865,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T17-57-54-05-00_gpqa-diamond_2znyMtdc7X4LJufxXeXA8Z.eval"
-            },
-            "winogrande": {
-                "accuracy": 0.8492501973164956,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T22-50-40-05-00_winogrande_VsTW2uU2Kj66YoNoFfRfUj.eval"
-            },
-            "gsm8k": {
-                "accuracy": 0.9454131918119788,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T18-23-05-05-00_gsm8k_d523pJzkcvobxamhhobCRb.eval"
-            },
-            "arc_challenge": {
-                "accuracy": 0.9641638225255973,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T17-53-30-05-00_arc-challenge_AYFHec7wmd4jELF2Rgzfya.eval"
-            },
-            "arc_easy": {
-                "accuracy": 0.9755892255892256,
-                "log_url": "https://storage.googleapis.com/inspect-evals/base/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-06T17-45-57-05-00_arc-easy_Nd8NP3K48tvwLVZb8kXDwg.eval"
-            },
-            "drop": {
-                "mean": null,
-                "log_url": null
-            },
-            "hellaswag": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "ifeval": {
-                "final_acc": null,
-                "log_url": null
-            },
-            "mmlu": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "mmmu_multiple_choice": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "mmmu_open": {
-                "accuracy": null,
-                "log_url": null
-            },
-            "gaia": {
-                "accuracy": 0.27030303030303043,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o1-2024-12-17/index.html?log_file=logs/logs/2025-02-05T23-21-20+00-00_gaia_hyMq8MzMm6NgAeq3dNqZSU.eval"
-            },
-            "gdm_intercode_ctf": {
-                "accuracy": 0.8278481012658225,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-05T21-43-18+00-00_gdm-intercode-ctf_gdm29C6DuTEsX9qm9ymmrC.eval"
-            },
-            "gdm_in_house_ctf": {
-                "accuracy": 0.38461538461538464,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-05T23-59-08+00-00_gdm-in-house-ctf_2zkAX5nkJoxDnVKpJL9VgW.eval"
-            },
-            "agentharm": {
-                "avg_score": 0.1241931080283353,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-03T18-17-03-08-00_agentharm_DmN6i5HrgXHNARjsuSewjg.eval"
-            },
-            "agentharm_benign": {
-                "avg_score": 0.5429306867375049,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-03T18-49-08-08-00_agentharm-benign_Gv94YFpAXaaCJqe3Fc6yr3.eval"
-            },
-            "swe_bench": {
-                "mean": 0.0024,
-                "log_url": "https://storage.googleapis.com/inspect-evals/agentic/eval/o3-mini-2025-01-31/index.html?log_file=logs/logs/2025-02-03T06-49-09+00-00_openai-o3-mini_swe.eval"
-            }
-        }
-    }
-}

data/tasks.json

@@ -1,142 +0,0 @@
-{
-    "arc_easy": {
-        "benchmark": "arc_easy",
-        "metric": "accuracy",
-        "display_name": "ARC-E",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc"
-    },
-    "arc_challenge": {
-        "benchmark": "arc_challenge",
-        "metric": "accuracy",
-        "display_name": "ARC-C",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc"
-    },
-    "drop": {
-        "benchmark": "drop",
-        "metric": "mean",
-        "display_name": "DROP",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/drop"
-    },
-    "winogrande": {
-        "benchmark": "winogrande",
-        "metric": "accuracy",
-        "display_name": "WinoGrande",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/winogrande"
-    },
-    "gsm8k": {
-        "benchmark": "gsm8k",
-        "metric": "accuracy",
-        "display_name": "GSM8K",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gsm8k"
-    },
-    "hellaswag": {
-        "benchmark": "hellaswag",
-        "metric": "accuracy",
-        "display_name": "HellaSwag",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/hellaswag"
-    },
-    "humaneval": {
-        "benchmark": "humaneval",
-        "metric": "mean",
-        "display_name": "HumanEval",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/humaneval"
-    },
-    "ifeval": {
-        "benchmark": "ifeval",
-        "metric": "final_acc",
-        "display_name": "IFEval",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/ifeval"
-    },
-    "math": {
-        "benchmark": "math",
-        "metric": "accuracy",
-        "display_name": "MATH",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mathematics"
-    },
-    "mmlu": {
-        "benchmark": "mmlu",
-        "metric": "accuracy",
-        "display_name": "MMLU",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu"
-    },
-    "mmlu_pro": {
-        "benchmark": "mmlu_pro",
-        "metric": "accuracy",
-        "display_name": "MMLU-Pro",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu_pro"
-    },
-    "gpqa_diamond": {
-        "benchmark": "gpqa_diamond",
-        "metric": "accuracy",
-        "display_name": "GPQA-D",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gpqa"
-    },
-    "mmmu_multiple_choice": {
-        "benchmark": "mmmu_multiple_choice",
-        "metric": "accuracy",
-        "display_name": "MMMU-MC",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu"
-    },
-    "mmmu_open": {
-        "benchmark": "mmmu_open",
-        "metric": "accuracy",
-        "display_name": "MMMU-OE",
-        "type": "base",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu"
-    },
-    "gaia": {
-        "benchmark": "gaia",
-        "metric": "accuracy",
-        "display_name": "GAIA",
-        "type": "agentic",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gaia"
-    },
-    "gdm_intercode_ctf": {
-        "benchmark": "gdm_intercode_ctf",
-        "metric": "accuracy",
-        "display_name": "InterCode-CTF",
-        "type": "agentic",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf"
-    },
-    "gdm_in_house_ctf": {
-        "benchmark": "gdm_in_house_ctf",
-        "metric": "accuracy",
-        "display_name": "In-House-CTF",
-        "type": "agentic",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/in_house_ctf"
-    },
-    "agentharm": {
-        "benchmark": "agentharm",
-        "metric": "avg_score",
-        "display_name": "AgentHarm",
-        "type": "agentic",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm"
-    },
-    "agentharm_benign": {
-        "benchmark": "agentharm_benign",
-        "metric": "avg_score",
-        "display_name": "AgentHarm-Benign",
-        "type": "agentic",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm"
-    },
-    "swe_bench": {
-        "benchmark": "swe_bench",
-        "metric": "mean",
-        "display_name": "SWE-Bench",
-        "type": "agentic",
-        "source": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/swe_bench"
-    }
-}

inspect_log_file_names.json

@@ -0,0 +1,150 @@
+{
+    "gemini-1.5-pro": {
+        "mmlu": "2024-11-04T16-56-26-05-00_mmlu_Z9KrcK7x4ZLAR5nJ9JaVUe.json",
+        "humaneval": "2024-11-04T12-43-07-05-00_humaneval_5JBjtymGtK23qwVKxqidhV.json",
+        "mmlu_pro": "2024-11-04T20-13-09-05-00_mmlu-pro_Hv2ujvKLV6H7ZwQu2q8LNw.json",
+        "math": "2024-11-04T15-48-46-05-00_math_9DAZmGEfhpa3nUcmMAwqZe.json",
+        "arc_easy": "2024-11-04T12-31-43-05-00_arc-easy_eGxYWywpLuREcaCKvHa8Uk.json",
+        "gsm8k": "2024-11-04T15-15-26-05-00_gsm8k_cTebw3ugfrVz3dyPwxtdUZ.json",
+        "gpqa_diamond": "2024-11-05T09-56-31-05-00_gpqa-diamond_FBq2bnoyGYQ3NF96xQw8iy.json",
+        "ifeval": "2024-11-04T12-43-32-05-00_ifeval_mSwZ7AwA7akj5PjZbQMjgC.json",
+        "winogrande": "2024-11-04T12-40-46-05-00_winogrande_5SmD6rx47zmZvHHkQSSfHK.json",
+        "arc_challenge": "2024-11-04T12-37-36-05-00_arc-challenge_5VVApyQD22QpJoMm53EMdU.json",
+        "drop": "2024-11-04T12-44-32-05-00_drop_9dzPKVJojSVsxmiBFnej2m.json",
+        "hellaswag": "2024-11-05T13-14-31-05-00_hellaswag_N98eeftuY2pucRtgpUYk5m.json",
+        "gaia": "2024-11-15T12-53-32-05-00_gaia_NvyGRTXFrFskJfUvuLwvVr.json",
+        "gdm_intercode_ctf": "2024-11-15T16-23-23-05-00_gdm-intercode-ctf_3JrgtTMcijTUxHVaagPRYh.json"
+    },
+    "gemini-1.5-flash": {
+        "gpqa_diamond": "2024-11-04T12-47-34-05-00_gpqa-diamond_cL5kQj8DWbRfxz79piTSdy.json",
+        "arc_challenge": "2024-11-04T12-45-59-05-00_arc-challenge_YQLMHfEXqeYgGJY86EB9bp.json",
+        "math": "2024-11-04T15-25-38-05-00_math_eaYBRMFgo8p6VUUCYxnCWj.json",
+        "drop": "2024-11-04T12-52-08-05-00_drop_5i253AQzbENgHTYN4ATemV.json",
+        "mmlu_pro": "2024-11-04T19-44-13-05-00_mmlu-pro_8GrR6wUsYNkthiZNMmLa8y.json",
+        "ifeval": "2024-11-04T12-51-30-05-00_ifeval_ZATErMbLHoyxh4kDaSqy8j.json",
+        "hellaswag": "2024-11-05T23-19-25-05-00_hellaswag_MRffohuzgVjighGb8FoqSJ.json",
+        "winogrande": "2024-11-04T12-48-29-05-00_winogrande_Hmqo6Ydz3nfCnQAdUwgrbD.json",
+        "humaneval": "2024-11-04T12-50-47-05-00_humaneval_9j4rYguKeKmxEoD9VuddwX.json",
+        "arc_easy": "2024-11-04T12-39-50-05-00_arc-easy_NwmTEw6C8VSCXzzwZCFy48.json",
+        "gsm8k": "2024-11-04T15-22-21-05-00_gsm8k_hdJs3Z6XzpR5netTcWLXJT.json",
+        "mmlu": "2024-11-04T16-26-13-05-00_mmlu_QvfQ46qJen2bvxiktHu86H.json",
+        "gdm_intercode_ctf": "2024-11-15T20-52-53-05-00_gdm-intercode-ctf_oLYr3H6bFtrcmgM6EABmNt.json"
+    },
+    "o1": {
+        "winogrande": "2025-01-20T16-46-06-05-00_winogrande_YUtAdEsForRffqe4Sm3wtR.json",
+        "humaneval": "2025-01-17T14-59-12-05-00_humaneval_RRL8GMy9NakTxUHsDVWNng.json",
+        "mmmu_open": "2025-01-20T22-48-09-05-00_mmmu-open_oBzxJBYbvnktbbAwhoCrYK.json",
+        "mmlu_pro": "2025-01-20T14-02-37-05-00_mmlu-pro_EvDzvqaahQwhv6fJovN4BT.json",
+        "math": "2025-01-17T15-03-22-05-00_math_6BbvHFF8hLMsVYozyNLbyQ.json",
+        "arc_easy": "2025-01-17T11-29-26-05-00_arc-easy_DFbir4BdgQDbKd52r7tRKR.json",
+        "arc_challenge": "2025-01-17T11-44-42-05-00_arc-challenge_PsWXaBqrgv3EcTZC55gRzJ.json",
+        "gsm8k": "2025-01-17T12-56-38-05-00_gsm8k_iD8275qeyNTgX523pn45bF.json",
+        "gpqa_diamond": "2025-01-17T11-53-53-05-00_gpqa-diamond_EJV7ULFSQLRoFTEqsv3t6q.json",
+        "hellaswag": "2025-01-17T13-14-39-05-00_hellaswag_73sQJFnwpzWjTvEqKjUk4M.json",
+        "mmmu_multiple_choice": "2025-01-20T21-04-57-05-00_mmmu-multiple-choice_MctxjookaeTLCL8KpUeazT.json"
+    },
+    "claude-3-5-sonnet-20241022": {
+        "mmmu_multiple_choice": "2025-01-21T11-20-03-05-00_mmmu-multiple-choice_CWhKvGdoFo6pdHhDyi9GNm.json",
+        "mmlu_pro": "2025-01-16T19-01-05-05-00_mmlu-pro_3vi84or97gQupuj5sT6vgZ.json",
+        "hellaswag": "2025-01-15T15-09-33-05-00_hellaswag_QXqFxojvSToMu8ckHEMLkB.json",
+        "gpqa_diamond": "2025-01-15T13-56-36-05-00_gpqa-diamond_eg4gFaMRENjnnYvQNtSB59.json",
+        "gsm8k": "2025-01-15T14-23-25-05-00_gsm8k_nHB8Z4uZAwRAZFYpKmTptA.json",
+        "mmmu_open": "2025-01-21T11-24-21-05-00_mmmu-open_SSjv3Dq9gZkEEUnvJUd5xf.json",
+        "arc_easy": "2025-01-15T10-06-24-05-00_arc-easy_oBReQZQM5SAwMMD2jFshPb.json",
+        "arc_challenge": "2025-01-15T10-12-11-05-00_arc-challenge_X8i6caCzkcQo5AT5zXkXso.json",
+        "mmlu": "2025-01-16T15-16-51-05-00_mmlu_NFDs2kxmh3kQEbpbd8sz3w.json",
+        "math": "2025-01-16T12-29-54-05-00_math_NvNQU58M8r3fpiwPGnvq8h.json",
+        "ifeval": "2025-01-16T11-28-44-05-00_ifeval_fmWxch4ZjbmYCST6yUZsdV.json",
+        "humaneval": "2025-01-16T11-26-12-05-00_humaneval_kUASiaNd9uZfWvCwYHhdF5.json",
+        "winogrande": "2025-01-16T22-09-41-05-00_winogrande_mSWGAKg75E5RP79KWizvb9.json",
+        "drop": "2025-01-15T10-15-15-05-00_drop_Z9A2Y84HYponNxnzNT9TNq.json"
+    },
+    "c4ai-command-r-plus": {
+        "ifeval": "2024-10-30T17-23-04-04-00_ifeval_RGucUMwdGmUnRpqyMTZTzW.json",
+        "winogrande": "2024-10-30T14-42-18-04-00_winogrande_bY8yg7aRR5dCCK7NDCZEcc.json",
+        "arc_challenge": "2024-10-29T17-30-03-04-00_arc-challenge_XB7LURXEGaxskWuLtYwdnW.json",
+        "drop": "2024-10-30T12-06-30-04-00_drop_itY9cLiYAW2BF7NTeDceNd.json",
+        "math": "2024-10-30T17-26-34-04-00_math_kohBUMpMFuMsR4jz4vUNWM.json",
+        "gpqa_diamond": "2024-10-29T22-47-45-04-00_gpqa-diamond_JKpb6ya4pec9hh7uovPPCZ.json",
+        "mmlu_pro": "2024-10-31T01-11-38-04-00_mmlu-pro_gZVAuy3zMKR23BieM5PqAX.json",
+        "humaneval": "2024-10-30T17-22-23-04-00_humaneval_5ByPqUhoofSbKgvsUQNFCX.json",
+        "gsm8k": "2024-10-30T15-03-35-04-00_gsm8k_QxbfbriJsKGQAg96JyjkoT.json",
+        "hellaswag": "2024-10-30T15-18-17-04-00_hellaswag_UYyBTR6N8VJnKRmnbCrB8N.json",
+        "mmlu": "2024-10-30T21-55-26-04-00_mmlu_JUPPLTzfe3Kme6UuorPTqg.json",
+        "arc_easy": "2024-10-29T17-10-40-04-00_arc-easy_UvprihBMLXPF8JENVLRkdx.json"
+    },
+    "gpt-4o-mini": {
+        "drop": "2024-10-30T17-36-25-04-00_drop_6TzJGqqEkpFUCxGD4QejV6.json",
+        "humaneval": "2024-10-30T21-14-41-04-00_humaneval_Z9aXdUERuwYxoTheZ5GANC.json",
+        "gpqa_diamond": "2024-10-30T19-31-26-04-00_gpqa-diamond_7aNe9wQiQKpNN96mfaWBPg.json",
+        "mmmu_open": "2025-01-20T23-13-27-05-00_mmmu-open_GWi6XNYUSLq99BdabtScGm.json",
+        "arc_challenge": "2024-10-30T17-34-51-04-00_arc-challenge_FbGgLswBZbRE4EhWiMyRt6.json",
+        "mmlu": "2024-10-31T10-49-43-04-00_mmlu_oGb9mspeGbYS2gfbkknskN.json",
+        "hellaswag": "2024-10-30T19-35-34-04-00_hellaswag_2SAz3cvMpDxFaApdHDR3s4.json",
+        "ifeval": "2024-10-30T21-15-06-04-00_ifeval_nYs9KujQMQjcpbpbLtVx8G.json",
+        "mmmu_multiple_choice": "2025-01-20T23-10-01-05-00_mmmu-multiple-choice_c5rLkrXkV83udX6DVJui5F.json",
+        "arc_easy": "2024-10-30T17-29-56-04-00_arc-easy_XcEzqqPqJsRV29NqYDfnNo.json",
+        "winogrande": "2024-10-31T00-59-07-04-00_winogrande_Ci55vHvbGGW38zVpMCwtWa.json",
+        "mmlu_pro": "2024-10-30T22-43-30-04-00_mmlu-pro_Dc2uu3EV7MJtjg6gg5Y9qH.json",
+        "math": "2024-10-30T21-15-49-04-00_math_YsWdRzpqMq2dqQ9SPKfack.json",
+        "gsm8k": "2024-10-30T19-32-39-04-00_gsm8k_nLSssETKDDWNktAFWnVwfv.json"
+    },
+    "Meta-Llama-3.1-70B-Instruct": {
+        "hellaswag": "2024-10-30T00-45-54-04-00_hellaswag_BKfQG9yGAr383MGnooMLBH.json",
+        "drop": "2024-10-29T21-01-02-04-00_drop_LzAWvLWkNrNKu5qf56wXRo.json",
+        "gpqa_diamond": "2024-10-29T23-41-39-04-00_gpqa-diamond_TdLdYmVM6GCVMAECcXkuhj.json",
+        "winogrande": "2024-10-30T09-20-56-04-00_winogrande_WnUgkSRhSMvh3zUjnuJWQZ.json",
+        "gsm8k": "2024-10-30T00-03-31-04-00_gsm8k_bKsUfCAfcmBCeryboNaLoX.json",
+        "math": "2024-10-30T02-34-50-04-00_math_2xiNcrGih26uzJdG4q88bM.json",
+        "ifeval": "2024-10-30T02-29-32-04-00_ifeval_Dwh3CF2ZYFrvw7UcTwrsvK.json",
+        "arc_challenge": "2024-10-29T20-58-56-04-00_arc-challenge_oFL5wFjT7KwNFhMFfe72JN.json",
+        "arc_easy": "2024-10-29T20-53-12-04-00_arc-easy_UXzR7cDeNteP39NoXUYnhm.json",
+        "mmlu_pro": "2024-10-30T06-11-16-04-00_mmlu-pro_oQiEBJdeKtEEt4cm9KL7uy.json",
+        "humaneval": "2024-10-30T02-28-25-04-00_humaneval_KcJV2rHuHJ2JLxijihEkcW.json",
+        "mmlu": "2024-10-30T03-51-50-04-00_mmlu_6SNjs2QmPRvqGnvbnNtaqb.json"
+    },
+    "gpt-4o": {
+        "gpqa_diamond": "2024-10-31T03-29-33-04-00_gpqa-diamond_nFmRv5MJiYjHjezmq4V6Va.json",
+        "arc_challenge": "2024-10-31T01-45-55-04-00_arc-challenge_nrsPPxh4DpzgLPQDFdcfVp.json",
+        "gsm8k": "2024-10-31T03-31-16-04-00_gsm8k_jVXeSvHowbietZCFsFYCwB.json",
+        "mmlu": "2024-10-31T10-49-43-04-00_mmlu_GarLpfQFSpM3C22nbbGp54.json",
+        "ifeval": "2024-10-31T05-00-11-04-00_ifeval_jxreUu8JqRdkrcHP4E3hLR.json",
+        "mmlu_pro": "2024-10-31T06-59-42-04-00_mmlu-pro_EuAKDwAWSfNVpqyyqrf2Ba.json",
+        "mmmu_open": "2025-01-20T23-07-46-05-00_mmmu-open_d3Q2HvuPZzEX6FAM4NBhnp.json",
+        "winogrande": "2024-10-31T09-02-03-04-00_winogrande_44kKF7M9mKoqVC7ixZVXuq.json",
+        "drop": "2024-10-31T01-47-20-04-00_drop_3gxDcn6vUoR3nvHX9BcSq4.json",
+        "arc_easy": "2024-10-31T01-41-34-04-00_arc-easy_nUavRHdiRVfrxo6dmCPadh.json",
+        "mmmu_multiple_choice": "2025-01-20T23-03-21-05-00_mmmu-multiple-choice_eoycAFLMirSqiURdXmBP2e.json",
+        "humaneval": "2024-10-31T04-59-42-04-00_humaneval_nmJcd84CcNKjWS8fBfMbZM.json",
+        "math": "2024-10-31T05-01-22-04-00_math_cDSpKPp3nLrFy8uYfYKEbM.json",
+        "hellaswag": "2024-10-31T03-33-47-04-00_hellaswag_JNnnPuz3dhZRpyXzizMUBF.json"
+    },
+    "Mistral-Large-Instruct-2407": {
+        "drop": "2024-10-31T01-56-12-04-00_drop_NtvuCoU2LoMbH8DztcCTen.json",
+        "ifeval": "2024-10-31T06-30-16-04-00_ifeval_TLkvCSFEWo4PLv6hAha7YB.json",
+        "mmlu": "2024-10-31T07-21-48-04-00_mmlu_YnUhmHoStr3WuJdchWmNPt.json",
+        "gpqa_diamond": "2024-10-31T04-22-52-04-00_gpqa-diamond_SuZUZxGdqS2ZecbLRNkKd4.json",
+        "gsm8k": "2024-10-31T04-28-49-04-00_gsm8k_5tQp9tbwUMj6NpjNKCAfVm.json",
+        "math": "2024-10-31T06-33-09-04-00_math_2CmjBedAfUxqvmcHRdBgyB.json",
+        "arc_easy": "2024-10-31T01-48-39-04-00_arc-easy_YbfuBT3usZXt2xgZkkR5dq.json",
+        "mmlu_pro": "2024-10-31T09-41-25-04-00_mmlu-pro_fyYT4aabPesfY5TpzFMPnd.json",
+        "humaneval": "2024-10-31T06-29-24-04-00_humaneval_nu8SUSGekKJWB8HLKDigYK.json",
+        "hellaswag": "2024-10-31T04-50-00-04-00_hellaswag_ZzQoZ6gkRQsTzMhQr7GYNn.json",
+        "arc_challenge": "2024-10-31T01-54-13-04-00_arc-challenge_WfQRhMkFcywefpU46isBVP.json",
+        "winogrande": "2024-10-31T11-57-58-04-00_winogrande_TP3UGwpp37Dyv6ks9Ty5Hk.json"
+    },
+    "Qwen2.5-72B-Instruct": {
+        "arc_challenge": "2024-10-31T13-46-34-04-00_arc-challenge_FSybKYYwpXVLQag8VwpjKe.json",
+        "mmlu_pro": "2024-11-01T20-31-04-04-00_mmlu-pro_2TfSPmsVmKatntHy2CnR7A.json",
+        "gpqa_diamond": "2024-10-31T13-48-32-04-00_gpqa-diamond_8qSySicySUyNvRRYVFBKLU.json",
+        "winogrande": "2024-10-31T14-46-29-04-00_winogrande_CX692dYh53gJ6JigT9GMpa.json",
+        "mmlu": "2024-11-01T10-08-50-04-00_mmlu_AgK27yYvmAo2LxotBH7ZL9.json",
+        "hellaswag": "2024-11-01T02-55-55-04-00_hellaswag_RSk8rGcQWg3HRrLffTNoiM.json",
+        "gsm8k": "2024-11-01T01-15-16-04-00_gsm8k_3h4W6xZjXpz9oCwtgKNYzo.json",
+        "arc_easy": "2024-10-31T13-40-08-04-00_arc-easy_3JUyzfoEHxhSBUdCU2AaVC.json",
+        "math": "2024-11-01T10-06-46-04-00_math_UUpS2R9eQc9KxBxkanT2gE.json",
+        "ifeval": "2024-10-31T14-51-45-04-00_ifeval_VGxA7gTZLZSruceM9Ci37C.json",
+        "humaneval": "2024-10-31T14-49-39-04-00_humaneval_9u7khnxivCDroJoPNRFpjs.json",
+        "drop": "2024-10-31T15-03-20-04-00_drop_DDLi98VhiV2bLzuw7fx6H4.json"
+    }
+}

refactor_eval_results.py

@@ -0,0 +1,127 @@
+import json
+import os
+
+
+METRIC_NAME = {
+    # single-turn
+    "arc_easy": "accuracy",
+    "arc_challenge": "accuracy",
+    "gpqa_diamond": "accuracy",
+    "drop": "mean",
+    "winogrande": "accuracy",
+    "gsm8k": "accuracy",
+    "hellaswag": "accuracy",
+    "humaneval": "mean",
+    "ifeval": "final_acc",
+    "math": "accuracy",
+    "mmlu": "accuracy",
+    "mmlu_pro": "accuracy",
+    "mmmu_multiple_choice": "accuracy",
+    "mmmu_open": "accuracy",
+
+    # agentic
+    "gaia": "mean",
+    "gdm_intercode_ctf": "accuracy",
+}
+
+MODEL_SHA_MAP = {
+    # open source models
+    "c4ai-command-r-plus": "https://huggingface.co/CohereForAI/c4ai-command-r-plus", # TODO: verify for the 08-2024 version
+    "Meta-Llama-3.1-70B-Instruct": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
+    "Mistral-Large-Instruct-2407": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
+    "Qwen2.5-72B-Instruct": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct",
+
+    # closed source models
+    "claude-3-5-sonnet-20241022": "https://www.anthropic.com/claude/sonnet",
+    "gemini-1.5-flash": "https://deepmind.google/technologies/gemini/flash", # TODO: points to 2.0, can't find page for 1.5
+    "gemini-1.5-pro": "https://deepmind.google/technologies/gemini/pro",
+    "gpt-4o": "https://openai.com/index/hello-gpt-4o",
+    "gpt-4o-mini": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
+    "o1": "https://openai.com/o1",
+}
+
+
+def combine_eval_results(results_path: str, model_name: str) -> dict:
+    results = dict(
+        {
+            "config": {
+                "model_name": model_name,
+                # dummy keys
+                "model_sha": MODEL_SHA_MAP[model_name],
+                "model_dtype": "torch.float16",
+            },
+            "results": {},
+        }
+    )
+    for file in os.listdir(os.path.join(results_path, model_name)):
+        if file.endswith(".json"):
+            with open(os.path.join(results_path, model_name, file), "r") as f:
+                try:
+                    result = json.load(f)
+                    task_name = result["eval"]["task"].split("/")[-1]
+                    if task_name == "math":
+                        metrics = [elm for elm in result["results"]["scores"] if elm["name"] == "expression_equivalance"][0]["metrics"] # TODO: change scorer if required
+                    else:
+                        metrics = result["results"]["scores"][0]["metrics"]
+                    metric_name = metrics[METRIC_NAME[task_name]]["name"]
+                    metric_value = metrics[METRIC_NAME[task_name]]["value"]
+                    results["results"].update(
+                        {
+                            task_name: {
+                                metric_name: metric_value
+                            }
+                        }
+                    )
+                except KeyError as e:
+                    print(f"KeyError: {e}")
+                    print(model_name)
+                    print(file)
+    return results
+
+
+def main():
+
+    CACHE_PATH=os.getenv("HF_HOME", ".")
+    EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+    EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+
+    base_bm_input_path = "./base_benchmarking_logs"
+    agentic_bm_input_path = "./agentic_benchmarking_logs"
+    os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
+    os.makedirs(EVAL_REQUESTS_PATH, exist_ok=True)
+
+    for model_name in os.listdir(base_bm_input_path):
+        if os.path.isdir(os.path.join(base_bm_input_path, model_name)):
+            results = combine_eval_results(base_bm_input_path, model_name)
+        # TMP: Add dummy agentic benchmarks to the results
+        for metric in METRIC_NAME.items():
+            if metric[0] not in results["results"]:
+                results["results"].update({metric[0]: {metric[1]: None}})
+        if os.path.isdir(os.path.join(agentic_bm_input_path, model_name)):
+            agentic_bm_results = combine_eval_results(agentic_bm_input_path, model_name)
+            results["results"].update(agentic_bm_results["results"])
+        with open(os.path.join(EVAL_RESULTS_PATH, f"{model_name}.json"), "w") as f:
+            json.dump(results, f, indent=4)        
+
+        # Create dummy requests file
+        requests = {
+            "model": model_name,
+            "model_sha": MODEL_SHA_MAP[model_name],
+            "base_model": "",
+            "revision": "main",
+            "private": False,
+            "precision": "float16",
+            "weight_type": "Original",
+            "status": "FINISHED",
+            "submitted_time": "",
+            "model_type": "pretrained",
+            "likes": 0,
+            "params": 0,
+            "license": "custom",
+        }
+        with open(os.path.join(EVAL_REQUESTS_PATH, f"{model_name}.json"), "w") as f:
+            json.dump(requests, f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

src/about.py

@@ -1,96 +1,132 @@
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Evaluation Leaderboard</h1>"""
+from dataclasses import dataclass
+from enum import Enum
 
-# SINGLE_TURN_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "base"])
-# AGENTIC_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "agentic"])
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+    type: str
+    source: str
 
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = f"""
-Powered by **Inspect** and **Inspect Evals**, the **Vector Evaluation Leaderboard** presents an evaluation of leading frontier models across a comprehensive suite of benchmarks. Go beyond the summary metrics: click through to interactive reporting for each model and benchmark to explore sample-level performance and detailed traces."""
 
-# Which evaluations are you running? how can people reproduce what you have?
-ABOUT_TEXT = f"""
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
 
-## Vector Institute
-The **Vector Institute** is dedicated to advancing the field of artificial intelligence through cutting-edge research and application. Our mission is to drive excellence and innovation in AI, fostering a community of researchers, developers, and industry partners.
+    # base
+    task0 = Task("arc_easy", "accuracy", "ARC-Easy", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc")
+    task1 = Task("arc_challenge", "accuracy", "ARC-Challenge", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/arc")
+    task2 = Task("drop", "mean", "DROP", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/drop")
+    task3 = Task("winogrande", "accuracy", "WinoGrande", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/winogrande")
+    task4 = Task("gsm8k", "accuracy", "GSM8K", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gsm8k")
+    task5 = Task("hellaswag", "accuracy", "HellaSwag", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/hellaswag")
+    task6 = Task("humaneval", "mean", "HumanEval", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/humaneval")
+    task7 = Task("ifeval", "final_acc", "IFEval", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/ifeval")
+    task8 = Task("math", "accuracy", "MATH", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mathematics")
+    task9 = Task("mmlu", "accuracy", "MMLU", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu")
+    task10 = Task("mmlu_pro", "accuracy", "MMLU-Pro", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmlu_pro")
+    task11 = Task("gpqa_diamond", "accuracy", "GPQA-Diamond", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gpqa")
+    task12 = Task("mmmu_multiple_choice", "accuracy", "MMMU-Multiple-Choice", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu")
+    task13 = Task("mmmu_open", "accuracy", "MMMU-Open-Ended", "base", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/mmmu")
 
-## 🎯 Benchmarks
+    # agentic
+    task14 = Task("gaia", "mean", "GAIA", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gaia")
+    task15 = Task("gdm_intercode_ctf", "accuracy", "GDM-InterCode-CTF", "agentic", "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/gdm_capabilities/intercode_ctf")
 
-This leaderboard showcases performance across a comprehensive suite of benchmarks, designed to rigorously evaluate different aspects of AI model capabilities. Let's explore the benchmarks we use:
 
-### Inspect Evals
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
 
-This leaderboard leverages [Inspect Evals](https://ukgovernmentbeis.github.io/inspect_evals/) to power evaluation. Inspect Evals is an open-source repository built upon the Inspect AI framework. Developed in collaboration between the Vector Institute, Arcadia Impact and the UK AI Security Institute, Inspect Evals provides a comprehensive suite of high-quality benchmarks spanning diverse domains like coding, mathematics, cybersecurity, reasoning, and general knowledge.
 
-#### Transparent and Detailed Insights
 
-All evaluations presented on this leaderboard are run using Inspect Evals. To facilitate in-depth analysis and promote transparency, we provide [Inspect Logs](https://inspect.ai-safety-institute.org.uk/log-viewer.html) for every benchmark run. These logs offer sample and trace level reporting, allowing the community to explore the granular details of model performance. 
-
-### ⚙️ Base Benchmarks
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">Vector State of Evaluation Leaderboard</h1>"""
 
-These benchmarks assess fundamental reasoning and knowledge capabilities of models.
+SINGLE_TURN_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "base"])
+AGENTIC_TASK_NAMES = ", ".join([f"[{task.value.col_name}]({task.value.source})" for task in Tasks if task.value.type == "agentic"])
 
-<div class="benchmark-table-container">
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = f"""
+This leaderboard presents the performance of selected LLM models on a set of tasks. The tasks are divided into two categories: base and agentic. The base tasks are: {SINGLE_TURN_TASK_NAMES}. The agentic tasks are: {AGENTIC_TASK_NAMES}."""
 
-| Benchmark           | Description                                                                      |
-|--------------------|----------------------------------------------------------------------------------|
-| **ARC-Easy** / **ARC-Challenge** | Multiple-choice science questions. |
-| **DROP**             | Comprehension benchmark evaluating advanced reasoning capability.             |
-| **WinoGrande**        | Commonsense reasoning challenge.             |
-| **GSM8K**             | Grade-school math word problems testing math capability & multi-step reasoning.         |
-| **HellaSwag**        | Commonsense reasoning task.                        |
-| **HumanEval**         | Evaluates code generation and reasoning in a programming context.                |
-| **IFEval**            | Specialized benchmark for instruction following.                          |
-| **MATH**              | Challenging questions sourced from  math competitions.                |
-| **MMLU** / **MMLU-Pro**| Multi-subject multiple-choice tests of advanced knowledge.                     |
-| **GPQA-Diamond**      | Question-answering benchmark assessing deeper reasoning.      |
-| **MMMU** (Multi-Choice / Open-Ended) | Multi-modal tasks testing structured & open responses.   |
-</div>
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = f"""
+# Vector State of Evaluation Leaderboard
 
-### 🚀 Agentic Benchmarks
+## Overview
+The **Vector State of Evaluation Leaderboard** presents the performance of selected LLM models on a variety of tasks. These tasks are divided into two categories:
 
-These benchmarks go beyond basic reasoning and evaluate more advanced, autonomous, or "agentic" capabilities of models, such as planning and interaction.
+- **Base Tasks**: ARC-Easy, ARC-Challenge, DROP, WinoGrande, GSM8K, HellaSwag, HumanEval, IFEval, MATH, MMLU, MMLU-Pro, GPQA-Diamond, MMMU-Multiple-Choice, MMMU-Open-Ended  
+- **Agentic Tasks**: GAIA, GDM-InterCode-CTF  
 
-<div class="benchmark-table-container">
+Users can compare models side by side to see how they perform on both base-level understanding tasks and more advanced, “agentic” tasks.
 
-| Benchmark              | Description                                                                 |
-|-----------------------|----------------------------------------------------------------------------|
-| **GAIA**                | Evaluates autonomous reasoning, planning, problem-solving for question answering. |
-| **InterCode-CTF**  | Capture-the-flag challenge testing cyber-security skills.        |
-| **In-House-CTF**    | Capture-the-flag challenge testing cyber-security skills.         |
-| **AgentHarm** / **AgentHarm-Benign** | Measures harmfulness of LLM agents (and benign behavior baseline).   |
-| **SWE-Bench-Verified**           | Tests AI agent ability to solve software engineering tasks.                 |
-</div>
+## Vector Institute
+The **Vector Institute** is dedicated to advancing the fields of artificial intelligence and machine learning through cutting-edge research, collaborative projects, and open-source contributions. This leaderboard is part of Vector’s broader effort to promote transparency and progress in AI research.
+
+## Model
+We evaluate a variety of **Large Language Models (LLMs)** across the included benchmarks. Each model:
+- Is tested on the same set of tasks.
+- Has standardized prompts or evaluation methodologies.
+- Generates performance metrics (accuracy, F1, etc.) for comparison.
+
+Our goal is to provide clear, reproducible metrics that shed light on how each model handles different task complexities and reasoning requirements.
+
+## Benchmarks
+Here is a closer look at each benchmark included in the leaderboard:
+
+### Base Benchmarks
+- **ARC-Easy / ARC-Challenge**: A set of multiple-choice science questions designed to measure a model’s scientific and commonsense reasoning.
+- **DROP**: A reading comprehension benchmark emphasizing discrete reasoning steps.
+- **WinoGrande**: A commonsense reasoning challenge focused on co-reference resolution.
+- **GSM8K**: Grade-school math word problems testing arithmetic and multi-step reasoning.
+- **HellaSwag**: A commonsense inference task centered on action completion.
+- **HumanEval**: Evaluates code generation and reasoning in a programming context.
+- **IFEval**: A specialized benchmark for incremental formal reasoning.
+- **MATH**: High school-level math questions requiring detailed solutions.
+- **MMLU / MMLU-Pro**: Multi-subject multiple-choice tests covering advanced high school and collegiate-level knowledge.
+- **GPQA-Diamond**: A question-answering benchmark that assesses deeper reasoning and knowledge linking.
+- **MMMU (Multiple-Choice / Open-Ended)**: A suite of multilingual and multi-domain tasks testing both structured and open-form responses.
+
+### Agentic Benchmarks
+- **GAIA**: Evaluates more autonomous or “agentic” reasoning, including planning and problem-solving.
+- **GDM-InterCode-CTF**: A capture-the-flag style challenge focusing on code interpretation and generative debugging strategies.
+
+---
 """
 
-REPRODUCIBILITY_TEXT = """
-## 🛠️ Reproducibility
-The [Vector State of Evaluation Leaderboard Repository](https://github.com/VectorInstitute/evaluation) repository contains the evaluation script to reproduce results presented on the leaderboard.
-
-### Install dependencies
+EVALUATION_QUEUE_TEXT = """
+## Some good practices before submitting a model
 
-1. Create a python virtual env. with ```python>=3.10``` and activate it
-```bash
-python -m venv env
-source env/bin/activate
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 
-2. Install ```inspect_ai```, ```inspect_evals``` and other dependencies based on ```requirements.txt```
-```bash
-python -m pip install -r requirements.txt
-```
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
 
-3. Install any packages required for models you'd like to evaluate and use as grader models
-```bash
-python -m pip install <model_package>
-```
-Note: ```openai``` package is already included in ```requirements.txt```
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
 
-### Run Inspect evaluation
-1. Update the ```src/evals_cfg/run_cfg.yaml``` file to select the evals (base/agentic) and include all models to be evaluated
-2. Now run evaluation as follows:
-```bash
-python src/run_evals.py
-```
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 """
 
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+"""

src/display/css_html_js.py

@@ -1,192 +1,236 @@
-custom_js = """
-function tableLinkHack() {
-    // This is a hack to make the table links work
-    var allTableLinks = document.querySelectorAll(".llm-benchmark-tab-table .table-wrap table.table a")
-    for (var link of allTableLinks) {
-        link.addEventListener("click", e => { 
-            window.open(e.target.href, e.target.target);
-        });
-    }
-}
-"""
+# custom_css = """
+
+# .markdown-text {
+#     font-size: 16px !important;
+# }
+
+# #models-to-add-text {
+#     font-size: 18px !important;
+# }
+
+# #citation-button span {
+#     font-size: 16px !important;
+# }
+
+# #citation-button textarea {
+#     font-size: 16px !important;
+# }
+
+# #citation-button > label > button {
+#     margin: 6px;
+#     transform: scale(1.3);
+# }
+
+# #leaderboard-table {
+#     margin-top: 15px
+# }
+
+# #leaderboard-table-lite {
+#     margin-top: 15px
+# }
+
+# #search-bar-table-box > div:first-child {
+#     background: none;
+#     border: none;
+# }
+ 
+# #search-bar {
+#     padding: 0px;
+# }
+
+# /* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+# #leaderboard-table td:nth-child(2),
+# #leaderboard-table th:nth-child(2) {
+#     max-width: 400px;
+#     overflow: auto;
+#     white-space: nowrap;
+# }
+
+# .tab-buttons button {
+#     font-size: 20px;
+# }
+
+# #scale-logo {
+#     border-style: none !important;
+#     box-shadow: none;
+#     display: block;
+#     margin-left: auto;
+#     margin-right: auto;
+#     max-width: 600px;
+# }
+
+# #scale-logo .download {
+#     display: none;
+# }
+# #filter_type{
+#     border: 0;
+#     padding-left: 0;
+#     padding-top: 0;
+# }
+# #filter_type label {
+#     display: flex;
+# }
+# #filter_type label > span{
+#     margin-top: var(--spacing-lg);
+#     margin-right: 0.5em;
+# }
+# #filter_type label > .wrap{
+#     width: 103px;
+# }
+# #filter_type label > .wrap .wrap-inner{  
+#     padding: 2px;
+# }
+# #filter_type label > .wrap .wrap-inner input{
+#     width: 1px
+# }
+# #filter-columns-type{
+#     border:0;
+#     padding:0.5;
+# }
+# #filter-columns-size{
+#     border:0;
+#     padding:0.5;
+# }
+# #box-filter > .form{
+#     border: 0
+# }
+
+# body, .gradio-container {
+#     font-family: Roboto, sans-serif;
+#     background-color: #ffffff;
+#     color: #000000; /* main text color */
+#     margin: 0;
+#     padding: 0;
+# }
+
+# h1, h2, h3, h4, h5, h6 {
+#     color: #eb088a; /* your brand color for headings */
+#     font-weight: 600;
+#     margin-bottom: 1rem;
+# }
+
+# /* Example ‘intro-block’ styling if you want extra flair */
+# .intro-block {
+#     background-color: #eb088a10; /* light tinted background */
+#     padding: 1.5rem;
+#     border-radius: 10px;
+#     margin-bottom: 2rem;
+# }
+
+# """
+
+# custom_css = """
+# /* 1) Load Karbon Font: Make sure this points to your actual font files */
+# @font-face {
+#     font-family: 'Karbon';
+#     src: url('path/to/Karbon.woff2') format('woff2'),
+#          url('path/to/Karbon.woff') format('woff');
+#     font-weight: normal;
+#     font-style: normal;
+# }
+
+# /* 2) Global Container */
+# body, .gradio-container {
+#     font-family: 'Karbon', sans-serif;
+#     margin: 0;
+#     padding: 0;
+#     background-color: #fafafa; /* Light background */
+#     color: #000000;
+# }
+
+# .gradio-container {
+#     max-width: 1200px; 
+#     margin: 0 auto;
+#     padding: 2rem 1rem;
+# }
+
+# /* 3) Headings, with brand color #eb088a */
+# h1, h2, h3, h4, h5, h6 {
+#     color: #000000;
+#     margin-bottom: 1rem;
+#     font-weight: 600;
+# }
+
+# /* 4) Intro Block for a slight highlight */
+# .intro-block {
+#     background-color: #ffe2f1; /* lighter tint of #eb088a */
+#     padding: 1.5rem;
+#     border-radius: 8px;
+#     border: 1px solid #f8badb;
+#     margin-bottom: 2rem;
+# }
+
+# /* 5) Tab styling - remove default orange styling */
+# .tab-buttons {
+#     margin-top: 1rem;
+#     margin-bottom: 1rem;
+#     display: flex;
+# }
+# .tab-buttons > .tabitem {
+#     padding: 0.6rem 1.2rem;
+#     background-color: #ffffff;
+#     border: 1px solid #eb088a;
+#     border-radius: 6px;
+#     color: #eb088a;
+#     margin-right: 5px;
+#     cursor: pointer;
+#     transition: background-color 0.2s ease, color 0.2s ease;
+#     font-weight: 500;
+# }
+# .tab-buttons > .tabitem.selected {
+#     background-color: #eb088a;
+#     color: #ffffff;
+# }
+# .tab-buttons > .tabitem:hover {
+#     background-color: #eb088a;
+#     color: #ffffff;
+# }
+
+# /* 6) Dataframe Styling */
+# .gr-dataframe table {
+#     width: 100%;
+#     border-collapse: collapse;
+#     border: 1px solid #cccccc;
+#     margin-bottom: 2rem;
+# }
+# .gr-dataframe th {
+#     background-color: #eb088a;
+#     color: #ffffff;
+#     padding: 0.6rem;
+#     text-align: left;
+#     font-weight: 600;
+# }
+# .gr-dataframe td {
+#     padding: 0.6rem;
+#     border-bottom: 1px solid #e0e0e0;
+# }
+# .gr-dataframe tr:nth-child(even) {
+#     background-color: #fdfdfd;
+# }
+
+# /* 7) Make default markdown text nice */
+# .markdown-text p {
+#     margin-bottom: 1rem;
+#     line-height: 1.6;
+# }
+# """
 
 custom_css = """
-footer {visibility: hidden}
-
-#main {
-    max-width: 1200px;
-    margin: 0 auto;
-    padding: 0 20px;
-}
-
-.gradio-container {
-    max-width: 1200px !important;
-    margin: 0 auto !important;
-}
-
-.intro-text {
-    text-align: center; /* Center the text */
-    font-size: 3rem; /* Slightly larger font size if desired */
-    color: #555;      /* A slightly softer color than black */
-    margin-bottom: 5px; /* Add some space below the text before the tabs */
-    padding: 0 10px;    /* Add some horizontal padding for wider screens */
-    line-height: 1.2;   /* Improve readability with line height */
-    max-width: 1200px;   /* Limit width for better readability on large screens */
-    margin-left: auto;  /* Center the block horizontally */
-    margin-right: auto; /* Center the block horizontally */
-}
 
 .tab-buttons button {
     font-size: 20px;
 }
 
-.intro-block {
-    padding: 20px;
-}
-
-.header-row {
-    height: 0;
-    min-height: 0;
-}
-
-.tabitem {
-    padding-top: 0;
-}
-
-.html-container {
-    padding: 0;
-}
-
-#page-header {
-    display: flex;
-    justify-content: center;
-    text-align: center;
-    margin-bottom: 1rem;
-}
-
-#header-container{
-    display: flex;
-    width: 800px;
-}
-
-#left-container {
-    flex: 1;
-}
-
-#left-container #black-logo, #left-container #white-logo {
-    height: 150px;
-    width: 150px;
-}
-
-#left-container #black-logo {
-    display: block;
-}
-
-#left-container #white-logo {
-    display: none;
-}
-
-#centre-container {
-    align-self: center;
-}
-
-#right-container {
-    flex: 1;
-    align-self: center;
-}
-
-#right-container #black-logo, #right-container #white-logo {
-    height: 150px;
-    width: 150px;
-}
-
-#right-container #black-logo {
-    display: block;
-}
-
-#right-container #white-logo {
-    display: none;
-}
-
-.llm-benchmark-tab-table .table-wrap table.table {
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
-    color: rgb(97, 97, 97);
-    overflow-y: auto;
-    overflow-x: auto;
-}
-
-.llm-benchmark-tab-table .table-wrap table.table a {
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
-    color: #ec4899
-}
-
-.llm-benchmark-tab-table .table-wrap table.table tr td,
-.llm-benchmark-tab-table .table-wrap table.table tr th {
-    border-bottom: 1px solid var(--border-color-primary);
-}
-
-.llm-benchmark-tab-table .table-wrap table.table a {
-    color: rgb(2, 136, 209) !important;
-}
-
-.llm-benchmark-tab-table .table-wrap table.table th {
-    font-size: 14px; /* Smaller font size for table headers */
-    font-weight: 600; /* Semi-bold instead of full bold */
-}
-
-@media (prefers-color-scheme: dark) {
-    #left-container #black-logo {
-        display: none;
-    }    
-    #left-container #white-logo {
-        display: block;
-    }
-    #right-container #black-logo {
-        display: none;
-    }    
-    #right-container #white-logo {
-        display: block;
-    }
-}
-
-.benchmark-table-container table {
-    width: 100%; /* Make table take full width of its container */
-    border-collapse: collapse; /* Remove spacing between table cells */
-    margin-bottom: 20px; /* Add some space below the table */
-}
-
-.benchmark-table-container th, .benchmark-table-container td {
-    border: 1px solid #ddd; /* Light gray border for cells */
-    padding: 8px; /* Padding within cells for better spacing */
-    text-align: left; /* Align text to the left within cells */
-    vertical-align: top; /* Align content to the top of cells */
-}
 
-.benchmark-table-container th {
-    background-color: #f2f2f2; /* Light gray background for header row */
-    font-weight: bold; /* Make header text bold */
-}
 
-.benchmark-table-container tbody tr:nth-child(even) {
-    background-color: #f9f9f9; /* Very light gray background for even rows for zebra striping */
-}
+"""
 
-@media (prefers-color-scheme: dark) {
-    #left-container #black-logo {
-        display: none;
-    }    
-    #left-container #white-logo {
-        display: block;
-    }
-    .benchmark-table-container th {
-        background-color: #3b3b3b;
-    }
-    .benchmark-table-container tbody tr:nth-child(even) {
-        background-color: #2b2b2b;
-    }
-}
+# .selected.svelte-1tcem6n.svelte-1tcem6n {
+#     background-color: #000000 !important; /* Desired background color */
+#     color: #eb088a !important; /* Desired text color */
+#     border-color: #eb088a !important; /* Desired border color */
+# }
 
-"""
 
 get_window_url_params = """
     function(url_params) {

src/display/formatting.py

@@ -5,8 +5,6 @@ def model_hyperlink(link, model_name):
 def make_clickable_model(model_name, model_sha):
     return model_hyperlink(model_sha, model_name)
 
-def make_clickable_field(name, url):
-    return model_hyperlink(url, name)
 
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"

src/display/utils.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from src.about import Tasks
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# Scores
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "markdown", True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+ST_BENCHMARK_COLS = [t.value.col_name for t in Tasks if t.value.type=="base"]
+AGENTIC_BENCHMARK_COLS = [t.value.col_name for t in Tasks if t.value.type=="agentic"]

src/envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "vector-institute" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/llm-eval-leaderboard"
+QUEUE_REPO = f"{OWNER}/llm-eval-requests"
+RESULTS_REPO = f"{OWNER}/llm-eval-results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,188 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            # if accs.size == 0 or any([acc is None for acc in accs]):
+            #     continue
+            if accs.size == 0:
+                continue
+            elif any([acc is None for acc in accs]):
+                mean_acc = None
+            else:
+                mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.revision),
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update(eval_result.results)
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -0,0 +1,91 @@
+import json
+import os
+
+import numpy as np
+import pandas as pd
+
+from src.about import Tasks
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+TASK_NAME_INVERSE_MAP = dict()
+for task in Tasks:
+    TASK_NAME_INVERSE_MAP[task.value.col_name] = {
+        "name": task.value.benchmark,
+        "type": task.value.type,
+        "source": task.value.source,
+    }
+
+EMPTY_SYMBOL = "--"
+
+
+def get_inspect_log_url(model_name: str, benchmark_name: str) -> str:
+    """Returns the URL to the log file for a given model and benchmark"""
+    with open("./inspect_log_file_names.json", "r") as f:
+        inspect_log_files = json.load(f)
+    log_file_name = inspect_log_files[model_name].get(benchmark_name, None)
+    if log_file_name is None:
+        return ""
+    else:
+        # replace .json with .eval
+        log_file_name = log_file_name.replace(".json", ".eval")
+        return f"https://storage.googleapis.com/inspect-evals/eval/{model_name}/index.html?log_file=logs/logs/{log_file_name}"
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+
+    df = df[cols].round(decimals=2)
+
+    # subset for model and benchmark cols
+    df = df[[AutoEvalColumn.model.name] + benchmark_cols]
+
+    df = df.fillna(EMPTY_SYMBOL)
+
+    # make values clickable and link to log files
+    for col in benchmark_cols:
+        df[col] = df[[AutoEvalColumn.model.name, col]].apply(lambda x: f"[{x[col]}]({get_inspect_log_url(model_name=x[AutoEvalColumn.model.name].split('>')[1].split('<')[0], benchmark_name=TASK_NAME_INVERSE_MAP[col]['name'])})" if x[col] != EMPTY_SYMBOL else x[col], axis=1)
+
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"], data["model_sha"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"], data["model_sha"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,119 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )