Clean sync from main branch - 2025-10-11 18:53:25

.devcontainer/devcontainer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a525cdb835f1b6c36c5d09b1663e2dc0b2e5a40b97214fc9ee2fc0366b9df622
+size 986

.gitattributes

@@ -0,0 +1,12 @@
+*.json filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.db filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/SevenNet.pkl filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/input.pkl filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/M3GNet.pkl filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/MACE-MPA.pkl filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/MACE-MP(M).pkl filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/MatterSim.pkl filter=lfs diff=lfs merge=lfs -text
+examples/mof/classification/ORBv2.pkl filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.github/README.md

@@ -0,0 +1,206 @@
+<div align="center">
+    <h1>⚔️ MLIP Arena ⚔️</h1>
+    <a href="https://huggingface.co/spaces/atomind/mlip-arena"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Space-blue" alt="Hugging Face"></a>
+    <a href="https://neurips.cc/virtual/2025/poster/121648"><img alt="Static Badge" src="https://img.shields.io/badge/NeurIPS-Spotlight-blue"></a>
+    <a href="https://arxiv.org/abs/2509.20630"><img src="https://img.shields.io/badge/arXiv-2509.20630-b31b1b"></a>
+    <a href="https://openreview.net/forum?id=ysKfIavYQE#discussion"><img alt="Static Badge" src="https://img.shields.io/badge/ICLR-AI4Mat-blue"></a>
+    <br>
+    <a href="https://github.com/atomind-ai/mlip-arena/actions"><img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/atomind-ai/mlip-arena/test.yaml"></a>
+    <a href="https://pypi.org/project/mlip-arena/"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/mlip-arena"></a>
+    <a href="https://pypi.org/project/mlip-arena/"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/mlip-arena"></a>
+    <a href="https://zenodo.org/doi/10.5281/zenodo.13704399"><img src="https://zenodo.org/badge/776930320.svg" alt="DOI"></a>
+    <!-- <a href="https://discord.gg/W8WvdQtT8T"><img alt="Discord" src="https://img.shields.io/discord/1299613474820984832?logo=discord"> -->
+</a>
+</div>
+
+Foundation machine learning interatomic potentials (MLIPs), trained on extensive databases containing millions of density functional theory (DFT) calculations, have revolutionized molecular and materials modeling, but existing benchmarks suffer from data leakage, limited transferability, and an over-reliance on error-based metrics tied to specific density functional theory (DFT) references.
+
+We introduce MLIP Arena, a unified benchmark platform for evaluating foundation MLIP performance beyond conventional error metrics. It focuses on revealing the physical soundness learned by MLIPs and assessing their utilitarian performance agnostic to underlying model architecture and training dataset. 
+
+***By moving beyond static DFT references and revealing the important failure modes*** of current foundation MLIPs in real-world settings, MLIP Arena provides a reproducible framework to guide the next-generation MLIP development toward improved predictive accuracy and runtime efficiency while maintaining physical consistency.
+
+MLIP Arena leverages modern pythonic workflow orchestrator 💙
+ [Prefect](https://www.prefect.io/) 💙
+ to enable advanced task/flow chaining and caching.
+
+![Thumnail](../serve/assets/workflow.png)
+
+> [!NOTE]
+> Contributions of new tasks through PRs are very welcome! See [project page](https://github.com/orgs/atomind-ai/projects/1) for some outstanding tasks, or propose new feature requests in [Discussion](https://github.com/atomind-ai/mlip-arena/discussions/new?category=ideas).
+
+## Announcement
+
+- **[Sep 18, 2025]** [🎊 **MLIP Arena is accepted as NeurIPS Spotlight!** 🎊](https://neurips.cc/virtual/2025/poster/121648)
+- **[Apr 8, 2025]** [🎉 **MLIP Arena is accepted as an ICLR AI4Mat Spotlight!** 🎉](https://openreview.net/forum?id=ysKfIavYQE#discussion) Huge thanks to all co-authors for their contributions!
+
+
+## Installation
+
+### From PyPI (prefect workflow only, without pretrained models)
+
+```bash
+pip install mlip-arena
+```
+
+### From source (with integrated pretrained models, advanced)
+
+> [!CAUTION] 
+> We strongly recommend clean build in a new virtual environment due to the compatibility issues between multiple popular MLIPs. We provide a single installation script using `uv` for minimal package conflicts and fast installation!
+
+> [!CAUTION]
+> To automatically download farichem OMat24 checkpoint, please make sure you have gained downloading access to their HuggingFace [***model repo***](https://huggingface.co/facebook/OMAT24) (not dataset repo), and login locally on your machine through `huggginface-cli login` (see [HF hub authentication](https://huggingface.co/docs/huggingface_hub/en/quick-start#authentication))
+
+**Linux**
+
+```bash
+# (Optional) Install uv, way faster than pip, why not? :)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+source $HOME/.local/bin/env
+
+git clone https://github.com/atomind-ai/mlip-arena.git
+cd mlip-arena
+
+# One script uv pip installation
+bash scripts/install.sh
+```
+
+> [!TIP]
+> Sometimes installing all compiled models takes all the available local storage. Optional pip flag `--no-cache` could be uesed. `uv cache clean` will be helpful too.
+
+**Mac**
+
+```bash
+# (Optional) Install uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+source $HOME/.local/bin/env
+# One script uv pip installation
+bash scripts/install-macosx.sh
+```
+
+## Workflow Overview
+
+### ✅ The first Prefect workflow: molecular dynamics
+
+Arena provides a unified interface to run all the compiled MLIPs. This can be achieved simply by looping through `MLIPEnum`:
+
+```python
+from mlip_arena.models import MLIPEnum
+from mlip_arena.tasks import MD 
+from mlip_arena.tasks.utils import get_calculator
+
+from ase import units
+from ase.build import bulk
+
+atoms = bulk("Cu", "fcc", a=3.6) * (5, 5, 5)
+
+results = []
+
+for model in MLIPEnum:
+    result = MD(
+        atoms=atoms,
+        calculator=get_calculator(
+            model,
+            calculator_kwargs=dict(), # passing into calculator
+            dispersion=True,
+            dispersion_kwargs=dict(
+                damping='bj', xc='pbe', cutoff=40.0 * units.Bohr
+            ), # passing into TorchDFTD3Calculator
+        ), # compatible with custom ASE Calculator
+        ensemble="nve", # nvt, nvt available
+        dynamics="velocityverlet", # compatible with any ASE Dynamics objects and their class names
+        total_time=1e3, # 1 ps = 1e3 fs
+        time_step=2, # fs
+    )
+    results.append(result)
+```
+
+### 🚀 Parallelize benchmarks at scale
+
+To run multiple benchmarks in parallel, add `.submit` before the task function and wrap all the tasks into a flow to dispatch the tasks to worker for concurrent execution. See Prefect Doc on [tasks](https://docs.prefect.io/v3/develop/write-tasks) and [flow](https://docs.prefect.io/v3/develop/write-flows) for more details.
+
+```python
+...
+from prefect import flow
+
+@flow
+def run_all_tasks:
+
+    futures = []
+    for model in MLIPEnum:
+        future = MD.submit(
+            atoms=atoms,
+            ...
+        )
+        future.append(future)
+
+    return [f.result(raise_on_failure=False) for f in futures]
+```
+
+For a more practical example using HPC resources, please now refer to [MD stability benchmark](../benchmarks/stability/temperature.ipynb).
+
+### List of modular tasks
+
+The implemented tasks are available under `mlip_arena.tasks.<module>.run` or `from mlip_arena.tasks import *` for convenient imports (currently doesn't work if [phonopy](https://phonopy.github.io/phonopy/install.html) is not installed).
+
+- [OPT](../mlip_arena/tasks/optimize.py#L56): Structure optimization
+- [EOS](../mlip_arena/tasks/eos.py#L42): Equation of state (energy-volume scan)
+- [MD](../mlip_arena/tasks/md.py#L200): Molecular dynamics with flexible dynamics (NVE, NVT, NPT) and temperature/pressure scheduling (annealing, shearing, *etc*)
+- [PHONON](../mlip_arena/tasks/phonon.py#L110): Phonon calculation driven by [phonopy](https://phonopy.github.io/phonopy/install.html)
+- [NEB](../mlip_arena/tasks/neb.py#L96): Nudged elastic band
+- [NEB_FROM_ENDPOINTS](../mlip_arena/tasks/neb.py#L164): Nudge elastic band with convenient image interpolation (linear or IDPP)
+- [ELASTICITY](../mlip_arena/tasks/elasticity.py#L78): Elastic tensor calculation
+
+## Workflow Quickstart
+
+Instruction for individual benchmark is provided in the README in each corresponding folder under [/benchmark](../benchmarks/).
+
+## Contribute and Development
+
+PRs are welcome. Please clone the repo and submit PRs with changes.
+
+To make change to huggingface space, fetch large files from git lfs first and run streamlit:
+
+```
+git lfs fetch --all
+git lfs pull
+streamlit run serve/app.py
+```
+
+### Add new benchmark
+
+> [!NOTE]
+> Please reuse, extend, or chain the general tasks defined [above](#list-of-implemented-tasks) and add new folder and script under [/benchmark](../benchmarks/)
+
+### Add new MLIP models 
+
+If you have pretrained MLIP models that you would like to contribute to the MLIP Arena and show benchmark in real-time, there are two ways:
+
+#### External ASE Calculator (easy)
+
+1. Implement new ASE Calculator class in [mlip_arena/models/externals](../mlip_arena/models/externals). 
+2. Name your class with awesome model name and add the same name to [registry](../mlip_arena/models/registry.yaml) with metadata.
+
+> [!CAUTION] 
+> Remove unneccessary outputs under `results` class attributes to avoid error for MD simulations. Please refer to [CHGNet](../mlip_arena/models/externals/chgnet.py) as an example.
+
+#### Hugging Face Model (recommended, difficult)
+
+0. Inherit Hugging Face [ModelHubMixin](https://huggingface.co/docs/huggingface_hub/en/package_reference/mixins) class to your awesome model class definition. We recommend [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/en/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin).
+1. Create a new [Hugging Face Model](https://huggingface.co/new) repository and upload the model file using [push_to_hub function](https://huggingface.co/docs/huggingface_hub/en/package_reference/mixins#huggingface_hub.ModelHubMixin.push_to_hub).
+2. Follow the template to code the I/O interface for your model [here](../mlip_arena/models/README.md). 
+3. Update model [registry](../mlip_arena/models/registry.yaml) with metadata
+
+## Citation
+
+If you find the work useful, please consider citing the following:
+
+```bibtex
+@inproceedings{
+    chiang2025mlip,
+    title={{MLIP} Arena: Advancing Fairness and Transparency in Machine Learning Interatomic Potentials through an Open and Accessible Benchmark Platform},
+    author={Yuan Chiang and Tobias Kreiman and Elizabeth Weaver and Ishan Amin and Matthew Kuner and Christine Zhang and Aaron Kaplan and Daryl Chrzan and Samuel M Blau and Aditi S. Krishnapriyan and Mark Asta},
+    booktitle={AI for Accelerated Materials Design - ICLR 2025},
+    year={2025},
+    url={https://openreview.net/forum?id=ysKfIavYQE}
+}
+```

.github/workflows/release.yaml

@@ -0,0 +1,96 @@
+name: Publish Release
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write  # Ensure write access to push tags
+
+jobs:
+  pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+
+    steps:
+    # Step 1: Checkout the code
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    # Step 2: Set up Python
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+
+    # Step 3: Install dependencies
+    - name: Install dependencies
+      run: pip install toml requests
+
+    # Step 4: Extract current version from pyproject.toml
+    - name: Extract current version
+      id: get_version
+      run: |
+        VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])")
+        echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+    # Step 5: Get latest version from PyPI
+    - name: Get latest version from PyPI
+      id: get_pypi_version
+      run: |
+        LATEST_PYPI_VERSION=$(python -c "import toml; import requests; PACKAGE_NAME = toml.load('pyproject.toml')['project']['name']; response = requests.get(f'https://pypi.org/pypi/{PACKAGE_NAME}/json'); print(response.json()['info']['version'])")
+        echo "LATEST_PYPI_VERSION=$LATEST_PYPI_VERSION" >> $GITHUB_ENV
+
+    # Step 6: Compare current version with the latest tag
+    - name: Check if version is bumped
+      id: check_version
+      run: |
+        if [ "${{ env.VERSION }}" = "${{ env.LATEST_PYPI_VERSION }}" ]; then
+          echo "Version not bumped. Exiting."
+          echo "version_bumped=false" >> $GITHUB_ENV
+        else
+          echo "Version bumped. Proceeding."
+          echo "version_bumped=true" >> $GITHUB_ENV
+        fi
+    
+    # Step 5: Remove problematic optional dependencies
+    - name: Strip problematic optional dependencies
+      run: |
+        python - <<EOF
+        import toml
+        from pathlib import Path
+
+        pyproject_path = Path("pyproject.toml")
+        data = toml.loads(pyproject_path.read_text())
+        
+        # Process optional dependencies
+        optional_deps = data.get("project", {}).get("optional-dependencies", {})
+        for key, deps in optional_deps.items():
+            new_deps = []
+            for dep in deps:
+                if "@git" in dep:
+                    dep = dep.split("@git")[0].strip()  # Remove everything after "@git"
+                new_deps.append(dep)
+            optional_deps[key] = new_deps
+            
+        pyproject_path.write_text(toml.dumps(data))
+        EOF
+
+    # Step 7: Install Flit (only if version bumped)
+    - name: Install Flit
+      if: env.version_bumped == 'true'
+      run: pip install flit
+
+    # Step 8: Create .pypirc file (only if version bumped)
+    - name: Create .pypirc file
+      if: env.version_bumped == 'true'
+      run: |
+        echo "[pypi]" > ~/.pypirc
+        echo "username = __token__" >> ~/.pypirc
+        echo "password = ${{ secrets.PYPI_API_TOKEN }}" >> ~/.pypirc
+
+    # Step 9: Build and publish package (only if version bumped)
+    - name: Build and Publish Package
+      if: env.version_bumped == 'true'
+      run: |
+        flit build
+        flit publish

.github/workflows/sync-hf.yaml

@@ -0,0 +1,39 @@
+name: Sync to Hugging Face hub
+
+on:
+  workflow_run:
+    workflows: [Python Test]
+    branches: [main]
+    types: [completed]
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+          
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          # Configure Git user identity
+          git config user.name "github-actions[ci]"
+          git config user.email "github-actions[ci]@users.noreply.github.com"
+
+          # Configure LFS tracking
+          git lfs track "*.pdf"
+          git lfs track "*.png"
+
+          # Create a new orphan branch (no history)
+          git checkout --orphan hf-clean
+
+          git add .
+          git commit -m "Clean sync from main branch - $(date '+%Y-%m-%d %H:%M:%S')"
+
+          # Force push to Hugging Face main branch
+          git push -f https://HF_USERNAME:[email protected]/spaces/atomind/mlip-arena hf-clean:main

.github/workflows/test.yaml

@@ -0,0 +1,103 @@
+name: Python Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  UV_SYSTEM_PYTHON: 1
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+    - name: Checkout PR with full history
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: "pyproject.toml"
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: bash scripts/install-linux.sh
+
+    - name: List dependencies
+      run: pip list
+
+    - name: Login to Hugging Face
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN_READ_ONLY }}
+      run: huggingface-cli login --token $HF_TOKEN
+
+    - name: Run tests
+      env:
+        PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
+        PREFECT_API_URL: ${{ secrets.PREFECT_API_URL }}
+      run: pytest -vra -n 5 --dist=loadscope tests
+
+    - name: Squash commits and trial push to Hugging Face
+      if: github.event_name == 'pull_request'
+      id: trial_push
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        TRIAL_BRANCH: trial-sync-${{ github.sha }}-${{ matrix.python-version }}
+      run: |
+        # Configure Git user identity
+        git config user.name "github-actions[ci]"
+        git config user.email "github-actions[ci]@users.noreply.github.com"
+
+        # Install Git LFS
+        sudo apt-get update
+        sudo apt-get install -y git-lfs
+        git lfs install
+
+        # Configure LFS tracking for binary files (only for HF push)
+        git lfs track "*.pdf"
+        git lfs track "*.png"
+
+        git add .gitattributes
+
+        # Setup LFS for the remote
+        git lfs fetch
+        git lfs checkout
+
+        # Rebase and squash all PR commits into one
+        BASE=$(git merge-base origin/main HEAD)
+        git reset --soft $BASE
+        
+        # Re-add all files (binary files will now be tracked by LFS)
+        git add .
+        git commit -m "Squashed commit from PR #${{ github.event.pull_request.number }}"
+
+        # Create a new orphan branch (no history)
+        git checkout --orphan hf-clean
+
+        git add .
+        git commit -m "Clean sync from main branch - $(date '+%Y-%m-%d %H:%M:%S')"
+
+        # Push to temporary branch on Hugging Face
+        git push -f https://HF_USERNAME:[email protected]/spaces/atomind/mlip-arena HEAD:refs/heads/$TRIAL_BRANCH
+
+    - name: Delete trial branch from Hugging Face
+      if: steps.trial_push.outcome == 'success'
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        TRIAL_BRANCH: trial-sync-${{ github.sha }}-${{ matrix.python-version }}
+      run: |
+        git push https://HF_USERNAME:[email protected]/spaces/atomind/mlip-arena --delete $TRIAL_BRANCH || true

.gitignore

@@ -0,0 +1,169 @@
+*.out
+*.extxyz
+*.traj
+mlip_arena/tasks/*/
+benchmarks/
+lab/
+manuscripts/
+datasets/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[server]
+fileWatcherType = "poll"

CITATION.cff

@@ -0,0 +1,23 @@
+# This CITATION.cff file was generated with cffinit.
+# Visit https://bit.ly/cffinit to generate yours today!
+
+cff-version: 1.2.0
+title: MLIP Arena
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Yuan
+    family-names: Chiang
+    email: [email protected]
+    affiliation: Lawrence Berkeley National Laboratory
+    orcid: 'https://orcid.org/0000-0002-4017-7084'
+repository-code: 'https://github.com/atomind-ai/mlip-arena'
+keywords:
+  - Quantum Chemistry
+  - Foundation Model
+  - Interatomic Potentials
+  - Machine Learning
+  - Force Fields
+license: Apache-2.0

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,14 @@
+---
+title: MLIP Arena
+emoji: ⚛
+sdk: streamlit
+sdk_version: 1.43.2 # The latest supported version
+python_version: 3.11
+app_file: serve/app.py
+colorFrom: indigo
+colorTo: yellow
+pinned: true
+short_description: Benchmark machine learning interatomic potential at scale
+---
+
+

benchmarks/bzo/BZO_cubic_prim.xyz

@@ -0,0 +1,7 @@
+5
+Lattice="4.2 0.0 0.0 0.0 4.2 0.0 0.0 0.0 4.2" Properties=species:S:1:pos:R:3 pbc="T T T"
+Ba       0.00000000       0.00000000       0.00000000
+Zr       2.10000000       2.10000000       2.10000000
+O        2.10000000       2.10000000       0.00000000
+O        2.10000000       0.00000000       2.10000000
+O        0.00000000       2.10000000       2.10000000

benchmarks/bzo/README.md

@@ -0,0 +1,6 @@
+# Second-order phase transition in BZO perovskite (A.10.3)
+
+To benchmark new MLIP, add model in model list or simply swap ASE calculator in [run.ipynb](run.ipynb).
+
+To reproduce the DFT PBE phonon modes, run [dft.ipynb](dft.ipynb) file. The example phonon mode used to generate plot in A.10.3 has been provided in [dft/mode-1.npy](dft/mode-1.npy).
+

benchmarks/bzo/run.ipynb

@@ -0,0 +1,130 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "154d45d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "eigvec = np.load(\"pbe/mode-1.npy\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aeb7637b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "from mlip_arena.models import MLIPEnum\n",
+    "\n",
+    "npoints = 101\n",
+    "max_disp = 0.5\n",
+    "disps = np.linspace(-max_disp, max_disp, npoints)\n",
+    "\n",
+    "replicas = (2, 2, 2)\n",
+    "\n",
+    "models = [\"MACE-MP(M)\", \"MatterSim\", \"ORBv2\", \"SevenNet\", \"CHGNet\", \"M3GNet\"]\n",
+    "\n",
+    "fig, axes = plt.subplots(\n",
+    "    figsize=(6, 4),\n",
+    "    nrows=2,\n",
+    "    ncols=len(models) // 2,\n",
+    "    constrained_layout=True,\n",
+    ")\n",
+    "\n",
+    "axes = axes.flatten()\n",
+    "\n",
+    "i = 0\n",
+    "\n",
+    "for model in MLIPEnum:\n",
+    "    if model.name not in models:\n",
+    "        continue\n",
+    "\n",
+    "    calc = get_calculator(model, device=\"cuda\")\n",
+    "\n",
+    "    emin = float(\"inf\")\n",
+    "    for a in [3.7, 3.85, 4.0, 4.15]:  # [3.8, 3.9, 4.0, 4.1]\n",
+    "        atoms = read(\"BZO_cubic_prim.xyz\")\n",
+    "        atoms.set_cell(cell=[a, a, a], scale_atoms=True)\n",
+    "        # atoms = atoms * replicas\n",
+    "\n",
+    "        energies = []\n",
+    "\n",
+    "        for disp in disps:\n",
+    "            atoms_disp = atoms.copy()\n",
+    "            atoms_disp.calc = calc\n",
+    "\n",
+    "            atoms_disp.positions += eigvec * disp\n",
+    "\n",
+    "            energy = atoms_disp.get_potential_energy() / len(atoms_disp)\n",
+    "            energies.append(energy)\n",
+    "\n",
+    "        energies = np.array(energies)\n",
+    "\n",
+    "        # shift the middle to 0\n",
+    "        energies -= energies[npoints // 2]\n",
+    "        emin = min(emin, energies.min())\n",
+    "\n",
+    "        axes[i].plot(disps, energies * 1000, label=f\"{a:.2f} Å\")\n",
+    "\n",
+    "    axes[i].axhline(0, color=\"black\", alpha=0.1)\n",
+    "    emin -= 1e-4\n",
+    "    axes[i].set(\n",
+    "        title=model.name,\n",
+    "        xlabel=\"Displacement (Å)\",\n",
+    "        ylabel=\"Energy (meV/atom)\",\n",
+    "        # xlim=(-0.1, 0.1),\n",
+    "        # ylim=(-0.1, 0.5),\n",
+    "    )\n",
+    "\n",
+    "    i += 1\n",
+    "    # break\n",
+    "\n",
+    "fig.legend(\n",
+    "    axes[0].get_legend_handles_labels()[0],\n",
+    "    axes[0].get_legend_handles_labels()[1],\n",
+    "    loc=\"lower center\",\n",
+    "    bbox_to_anchor=(0.5, 1),\n",
+    "    ncol=len(axes[0].get_legend_handles_labels()[0]),\n",
+    "    # fontsize=6\n",
+    ")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b5eeb801",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

benchmarks/c2db/ALIGNN.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce4d250afce0a7ef62dd27c5531b1e3a91f761035cc595e64ff6aae225e4ad73
+size 272171

benchmarks/c2db/CHGNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6063fa72efb16a5255b79f5e1a03bd13409ed129016496ff1f494c6f83b98be
+size 292909

benchmarks/c2db/M3GNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32e1517a85a1b64f12fb262a0948a95be58c69edde133ce7ddf683154b8f2a95
+size 290358

benchmarks/c2db/MACE-MP(M).parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f722eac6799bfecaa02188d59475862895a639cc596fa8b7d1e9d2b96cfb415b
+size 293633

benchmarks/c2db/MACE-MPA.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3ea679b5f6c9940358a2121a496544be91ba01ed8383509c65773f9fc69b9ec
+size 293820

benchmarks/c2db/MatterSim.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d150c1b31b99ddcbbf21401189289aead13791c683aa379d75163b8bc4dbc6b4
+size 293177

benchmarks/c2db/ORBv2.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2496f96d4aff1536936e58e65c1d608cc1953d41006221ba62ea2daab23f30b
+size 293012

benchmarks/c2db/README.md

@@ -0,0 +1,40 @@
+# Dynamical stability of 2D materials (A.10.2)
+
+## Run
+
+This benchmark requires parellel orchestration using Prefect utility. To run the benchmark, please modify the SLURM cluster setting (or change to desired job queing systems following [dask-jobqueuue documentation](https://jobqueue.dask.org/en/latest/)) in [run.py](run.py).
+
+
+```
+    nodes_per_alloc = 1
+    gpus_per_alloc = 1
+    ntasks = 1
+
+    cluster_kwargs = dict(
+        cores=1,
+        memory="64 GB",
+        shebang="#!/bin/bash",
+        account="matgen",
+        walltime="00:30:00",
+        job_mem="0",
+        job_script_prologue=[
+            "source ~/.bashrc",
+            "module load python",
+            "module load cudatoolkit/12.4",
+            "source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena",
+        ],
+        job_directives_skip=["-n", "--cpus-per-task", "-J"],
+        job_extra_directives=[
+            "-J eos_bulk",
+            "-q regular",
+            f"-N {nodes_per_alloc}",
+            "-C gpu",
+            f"-G {gpus_per_alloc}",
+            # "--exclusive",
+        ],
+    )
+```
+
+## Analysis
+
+The example analysis code is provided in [analysis.ipynb](analysis.ipynb)

benchmarks/c2db/SevenNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c2ee18ce70f24f70e65d70c2e54151e86dd0ccb3e412b8fbbc572e44e8bf5e8
+size 293973

benchmarks/c2db/analysis.ipynb

@@ -0,0 +1,402 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0625f0a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import random\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import numpy as np\n",
+    "from ase.db import connect\n",
+    "\n",
+    "random.seed(0)\n",
+    "\n",
+    "DATA_DIR = Path(\".\")\n",
+    "\n",
+    "db = connect(DATA_DIR / \"c2db.db\")\n",
+    "random_indices = random.sample(range(1, len(db) + 1), 1000)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "005708b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import itertools\n",
+    "\n",
+    "import pandas as pd\n",
+    "import phonopy\n",
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "from mlip_arena.models import MLIPEnum\n",
+    "\n",
+    "for row, model in tqdm(\n",
+    "    itertools.product(db.select(filter=lambda r: r[\"id\"] in random_indices), MLIPEnum)\n",
+    "):\n",
+    "    uid = row[\"uid\"]\n",
+    "\n",
+    "    if Path(f\"{model.name}.parquet\").exists():\n",
+    "        df = pd.read_parquet(f\"{model.name}.parquet\")\n",
+    "        if uid in df[\"uid\"].unique():\n",
+    "            continue\n",
+    "    else:\n",
+    "        df = pd.DataFrame(columns=[\"model\", \"uid\", \"eigenvalues\", \"frequencies\"])\n",
+    "\n",
+    "    try:\n",
+    "        path = Path(model.name) / uid\n",
+    "        phonon = phonopy.load(path / \"phonopy.yaml\")\n",
+    "        frequencies = phonon.get_frequencies(q=(0, 0, 0))\n",
+    "\n",
+    "        data = np.load(path / \"elastic.npz\")\n",
+    "\n",
+    "        eigenvalues = data[\"eigenvalues\"]\n",
+    "\n",
+    "        new_row = pd.DataFrame(\n",
+    "            [\n",
+    "                {\n",
+    "                    \"model\": model.name,\n",
+    "                    \"uid\": uid,\n",
+    "                    \"eigenvalues\": eigenvalues,\n",
+    "                    \"frequencies\": frequencies,\n",
+    "                }\n",
+    "            ]\n",
+    "        )\n",
+    "\n",
+    "        df = pd.concat([df, new_row], ignore_index=True)\n",
+    "        df.drop_duplicates(subset=[\"model\", \"uid\"], keep=\"last\", inplace=True)\n",
+    "\n",
+    "        df.to_parquet(f\"{model.name}.parquet\", index=False)\n",
+    "    except Exception:\n",
+    "        pass\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "b8d87638",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "uids = []\n",
+    "stabilities = []\n",
+    "for row in db.select(filter=lambda r: r[\"id\"] in random_indices):\n",
+    "    stable = row.key_value_pairs[\"dyn_stab\"]\n",
+    "    if stable.lower() == \"unknown\":\n",
+    "        stable = None\n",
+    "    else:\n",
+    "        stable = True if stable.lower() == \"yes\" else False\n",
+    "    uids.append(row.key_value_pairs[\"uid\"])\n",
+    "    stabilities.append(stable)\n",
+    "\n",
+    "\n",
+    "stabilities = np.array(stabilities)\n",
+    "\n",
+    "(stabilities == True).sum(), (stabilities == False).sum(), (stabilities == None).sum()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a3c516a7",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 104,
+   "id": "0052d0ff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%matplotlib inline\n",
+    "\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from matplotlib import pyplot as plt\n",
+    "from sklearn.metrics import (\n",
+    "    ConfusionMatrixDisplay,\n",
+    "    classification_report,\n",
+    "    confusion_matrix,\n",
+    ")\n",
+    "\n",
+    "from mlip_arena.models import MLIPEnum\n",
+    "\n",
+    "thres = -1e-7\n",
+    "\n",
+    "select_models = [\n",
+    "    \"ALIGNN\",\n",
+    "    \"CHGNet\",\n",
+    "    \"M3GNet\",\n",
+    "    \"MACE-MP(M)\",\n",
+    "    \"MACE-MPA\",\n",
+    "    \"MatterSim\",\n",
+    "    \"ORBv2\",\n",
+    "    \"SevenNet\",\n",
+    "]\n",
+    "\n",
+    "with plt.style.context(\"default\"):\n",
+    "\n",
+    "    SMALL_SIZE = 8\n",
+    "    MEDIUM_SIZE = 10\n",
+    "    BIGGER_SIZE = 12\n",
+    "    \n",
+    "    plt.rcParams.update(\n",
+    "        {\n",
+    "            \"font.size\": SMALL_SIZE,\n",
+    "            \"axes.titlesize\": MEDIUM_SIZE,\n",
+    "            \"axes.labelsize\": MEDIUM_SIZE,\n",
+    "            \"xtick.labelsize\": MEDIUM_SIZE,\n",
+    "            \"ytick.labelsize\": MEDIUM_SIZE,\n",
+    "            \"legend.fontsize\": SMALL_SIZE,\n",
+    "            \"figure.titlesize\": BIGGER_SIZE,\n",
+    "        }\n",
+    "    )\n",
+    "\n",
+    "    fig, axs = plt.subplots(\n",
+    "        nrows=int(np.ceil(len(MLIPEnum) / 4)),\n",
+    "        ncols=4,\n",
+    "        figsize=(6, 3 * int(np.ceil(len(select_models) / 4))),\n",
+    "        sharey=True,\n",
+    "        sharex=True,\n",
+    "        layout=\"constrained\",\n",
+    "    )\n",
+    "    axs = axs.flatten()\n",
+    "    plot_idx = 0\n",
+    "\n",
+    "    for model in MLIPEnum:\n",
+    "        fpath = DATA_DIR / f\"{model.name}.parquet\"\n",
+    "        if not fpath.exists():\n",
+    "            continue\n",
+    "\n",
+    "        if model.name not in select_models:\n",
+    "            continue\n",
+    "\n",
+    "        df = pd.read_parquet(fpath)\n",
+    "        df[\"eigval_min\"] = df[\"eigenvalues\"].apply(\n",
+    "            lambda x: x.min() if np.isreal(x).all() else thres\n",
+    "        )\n",
+    "        df[\"freq_min\"] = df[\"frequencies\"].apply(\n",
+    "            lambda x: x.min() if np.isreal(x).all() else thres\n",
+    "        )\n",
+    "        df[\"dyn_stab\"] = ~np.logical_or(\n",
+    "            df[\"eigval_min\"] < thres, df[\"freq_min\"] < thres\n",
+    "        )\n",
+    "\n",
+    "        arg = np.argsort(uids)\n",
+    "        uids_sorted = np.array(uids)[arg]\n",
+    "        stabilities_sorted = stabilities[arg]\n",
+    "\n",
+    "        sorted_df = (\n",
+    "            df[df[\"uid\"].isin(uids_sorted)].set_index(\"uid\").reindex(uids_sorted)\n",
+    "        )\n",
+    "        mask = ~(stabilities_sorted == None)\n",
+    "\n",
+    "        y_true = stabilities_sorted[mask].astype(\"int\")\n",
+    "        y_pred = sorted_df[\"dyn_stab\"][mask].fillna(-1).astype(\"int\")\n",
+    "        cm = confusion_matrix(y_true, y_pred, labels=[1, 0, -1])\n",
+    "\n",
+    "        ax = axs[plot_idx]\n",
+    "        ConfusionMatrixDisplay(\n",
+    "            cm, display_labels=[\"stable\", \"unstable\", \"missing\"]\n",
+    "        ).plot(ax=ax, cmap=\"Blues\", colorbar=False)\n",
+    "\n",
+    "        ax.set_title(model.name)\n",
+    "        ax.set_xlabel(\"Predicted\")\n",
+    "        ax.set_ylabel(\"True\")\n",
+    "        ax.set_xticks([0, 1, 2])\n",
+    "        ax.set_xticklabels([\"stable\", \"unstable\", \"missing\"])\n",
+    "        ax.set_yticks([0, 1, 2])\n",
+    "        ax.set_yticklabels([\"stable\", \"unstable\", \"missing\"])\n",
+    "\n",
+    "        plot_idx += 1\n",
+    "\n",
+    "    # Hide unused subplots\n",
+    "    for i in range(plot_idx, len(axs)):\n",
+    "        fig.delaxes(axs[i])\n",
+    "\n",
+    "    # plt.tight_layout()\n",
+    "    plt.savefig(\"c2db-confusion_matrices.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "id": "573b3c38",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from sklearn.metrics import confusion_matrix\n",
+    "\n",
+    "from mlip_arena.models import MLIPEnum\n",
+    "\n",
+    "thres = -1e-7\n",
+    "\n",
+    "summary_df = pd.DataFrame(columns=[\"Model\", \"Stable F1\", \"Unstable F1\", \"Weighted F1\"])\n",
+    "\n",
+    "for model in MLIPEnum:\n",
+    "    fpath = DATA_DIR / f\"{model.name}.parquet\"\n",
+    "\n",
+    "    if not fpath.exists() or model.name not in select_models:\n",
+    "        # print(f\"File {fpath} does not exist\")\n",
+    "        continue\n",
+    "    df = pd.read_parquet(fpath)\n",
+    "\n",
+    "    df[\"eigval_min\"] = df[\"eigenvalues\"].apply(\n",
+    "        lambda x: x.min() if np.isreal(x).all() else thres\n",
+    "    )\n",
+    "    df[\"freq_min\"] = df[\"frequencies\"].apply(\n",
+    "        lambda x: x.min() if np.isreal(x).all() else thres\n",
+    "    )\n",
+    "    df[\"dyn_stab\"] = ~np.logical_or(df[\"eigval_min\"] < thres, df[\"freq_min\"] < thres)\n",
+    "\n",
+    "    arg = np.argsort(uids)\n",
+    "    uids = np.array(uids)[arg]\n",
+    "    stabilities = stabilities[arg]\n",
+    "\n",
+    "    sorted_df = df[df[\"uid\"].isin(uids)].sort_values(by=\"uid\")\n",
+    "\n",
+    "    # sorted_df = sorted_df.reindex(uids).reset_index()\n",
+    "    sorted_df = sorted_df.set_index(\"uid\").reindex(uids)  # .loc[uids].reset_index()\n",
+    "\n",
+    "    sorted_df = sorted_df.loc[uids]\n",
+    "    # mask = ~np.logical_or(sorted_df['dyn_stab'].isna().values, stabilities == None)\n",
+    "    mask = ~(stabilities == None)\n",
+    "\n",
+    "    y_true = stabilities[mask].astype(\"int\")\n",
+    "    y_pred = sorted_df[\"dyn_stab\"][mask].fillna(-1).astype(\"int\")\n",
+    "    cm = confusion_matrix(y_true, y_pred, labels=[1, 0, -1])\n",
+    "    # print(model)\n",
+    "    # print(cm)\n",
+    "    # print(classification_report(y_true, y_pred, labels=[1, 0], target_names=['stable', 'unstable'], digits=3, output_dict=False))\n",
+    "\n",
+    "    report = classification_report(\n",
+    "        y_true,\n",
+    "        y_pred,\n",
+    "        labels=[1, 0],\n",
+    "        target_names=[\"stable\", \"unstable\"],\n",
+    "        digits=3,\n",
+    "        output_dict=True,\n",
+    "    )\n",
+    "\n",
+    "    summary_df = pd.concat(\n",
+    "        [\n",
+    "            summary_df,\n",
+    "            pd.DataFrame(\n",
+    "                [\n",
+    "                    {\n",
+    "                        \"Model\": model.name,\n",
+    "                        \"Stable F1\": report[\"stable\"][\"f1-score\"],\n",
+    "                        \"Unstable F1\": report[\"unstable\"][\"f1-score\"],\n",
+    "                        \"Macro F1\": report[\"macro avg\"][\"f1-score\"],\n",
+    "                        # 'Micro F1': report['micro avg']['f1-score'],\n",
+    "                        \"Weighted F1\": report[\"weighted avg\"][\"f1-score\"],\n",
+    "                    }\n",
+    "                ]\n",
+    "            ),\n",
+    "        ],\n",
+    "        ignore_index=True,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 85,
+   "id": "df660870",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summary_df = summary_df.sort_values(by=[\"Macro F1\", \"Weighted F1\"], ascending=False)\n",
+    "summary_df.to_latex(\"c2db_summary_table.tex\", index=False, float_format=\"%.3f\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 103,
+   "id": "18f4a59b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from matplotlib import cm\n",
+    "\n",
+    "# Metrics and bar settings\n",
+    "metrics = [\"Stable F1\", \"Unstable F1\", \"Macro F1\", \"Weighted F1\"]\n",
+    "bar_width = 0.2\n",
+    "x = np.arange(len(summary_df))\n",
+    "\n",
+    "# Get Set2 colormap (as RGBA)\n",
+    "cmap = plt.get_cmap(\"tab20\")\n",
+    "colors = {metric: cmap(i) for i, metric in enumerate(metrics)}\n",
+    "\n",
+    "with plt.style.context(\"default\"):\n",
+    "    plt.rcParams.update(\n",
+    "        {\n",
+    "            \"font.size\": SMALL_SIZE,\n",
+    "            \"axes.titlesize\": MEDIUM_SIZE,\n",
+    "            \"axes.labelsize\": MEDIUM_SIZE,\n",
+    "            \"xtick.labelsize\": MEDIUM_SIZE,\n",
+    "            \"ytick.labelsize\": MEDIUM_SIZE,\n",
+    "            \"legend.fontsize\": SMALL_SIZE,\n",
+    "            \"figure.titlesize\": BIGGER_SIZE,\n",
+    "        }\n",
+    "    )\n",
+    "\n",
+    "    fig, ax = plt.subplots(figsize=(4, 3), layout=\"constrained\")\n",
+    "\n",
+    "    # Bar positions\n",
+    "    positions = {\n",
+    "        \"Stable F1\": x - 1.5 * bar_width,\n",
+    "        \"Unstable F1\": x - 0.5 * bar_width,\n",
+    "        \"Macro F1\": x + 0.5 * bar_width,\n",
+    "        \"Weighted F1\": x + 1.5 * bar_width,\n",
+    "    }\n",
+    "\n",
+    "    # Plot each metric with assigned color\n",
+    "    for metric, pos in positions.items():\n",
+    "        ax.bar(\n",
+    "            pos, summary_df[metric], width=bar_width, label=metric, color=colors[metric]\n",
+    "        )\n",
+    "\n",
+    "    ax.set_xlabel(\"Model\")\n",
+    "    ax.set_ylabel(\"F1 Score\")\n",
+    "    # ax.set_title('F1 Scores by Model and Class')\n",
+    "    ax.set_xticks(x)\n",
+    "    ax.set_xticklabels(summary_df[\"Model\"], rotation=45, ha=\"right\")\n",
+    "    ax.legend(ncols=2, bbox_to_anchor=(0.5, 1), loc=\"upper center\", fontsize=SMALL_SIZE)\n",
+    "    # ax.legend(ncols=2, fontsize=SMALL_SIZE)\n",
+    "    ax.spines[[\"top\", \"right\"]].set_visible(False)\n",
+    "    plt.tight_layout()\n",
+    "    plt.ylim(0, 0.9)\n",
+    "    plt.grid(axis=\"y\", linestyle=\"--\", alpha=0.6)\n",
+    "\n",
+    "    plt.savefig(\"c2db_f1_bar.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1c50f705",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "mlip-arena",
+   "language": "python",
+   "name": "mlip-arena"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

benchmarks/c2db/c2db-confusion_matrices.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:463968f63e87ca0a7acd2e719cc481d0e3c5f5dd69ccf8f8659bddf6aa3b1e93
+size 21238

benchmarks/c2db/c2db-f1_bar.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d0c862d4efa2d9c83ac4fbe26eeef66a8f8017b37d955b70e414fdbea94aabd
+size 17883

benchmarks/c2db/c2db.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caf58205692de480e06149ac43a437385f18e14582e7d9a8dab8b3cb5d4bd678
+size 70762496

benchmarks/c2db/run.py

@@ -0,0 +1,214 @@
+from itertools import product
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from dask.distributed import Client
+from dask_jobqueue import SLURMCluster
+from mlip_arena.models import MLIPEnum
+from mlip_arena.tasks import ELASTICITY, OPT, PHONON
+from mlip_arena.tasks.optimize import run as OPT
+from mlip_arena.tasks.utils import get_calculator
+from numpy import linalg as LA
+from prefect import flow, task
+from prefect_dask import DaskTaskRunner
+from tqdm.auto import tqdm
+
+from ase.db import connect
+
+select_models = [
+    "ALIGNN",
+    "CHGNet",
+    "M3GNet",
+    "MACE-MP(M)",
+    "MACE-MPA",
+    "MatterSim",
+    "ORBv2",
+    "SevenNet",
+]
+
+
+def elastic_tensor_to_voigt(C):
+    """
+    Convert a rank-4 (3x3x3x3) elastic tensor into a rank-2 (6x6) tensor using Voigt notation.
+
+    Parameters:
+    C (numpy.ndarray): A 3x3x3x3 elastic tensor.
+
+    Returns:
+    numpy.ndarray: A 6x6 elastic tensor in Voigt notation.
+    """
+    # voigt_map = {
+    #     (0, 0): 0, (1, 1): 1, (2, 2): 2,  # Normal components
+    #     (1, 2): 3, (2, 1): 3,  # Shear components
+    #     (0, 2): 4, (2, 0): 4,
+    #     (0, 1): 5, (1, 0): 5
+    # }
+    voigt_map = {
+        (0, 0): 0,
+        (1, 1): 1,
+        (2, 2): -1,  # Normal components
+        (1, 2): -1,
+        (2, 1): -1,  # Shear components
+        (0, 2): -1,
+        (2, 0): -1,
+        (0, 1): 2,
+        (1, 0): 2,
+    }
+
+    C_voigt = np.zeros((3, 3))
+
+    for i in range(3):
+        for j in range(3):
+            for k in range(3):
+                for l in range(3):
+                    alpha = voigt_map[(i, j)]
+                    beta = voigt_map[(k, l)]
+
+                    if alpha == -1 or beta == -1:
+                        continue
+
+                    factor = 1
+                    # if alpha in [3, 4, 5]:
+                    if alpha == 2:
+                        factor = factor * (2**0.5)
+                    if beta == 2:
+                        factor = factor * (2**0.5)
+
+                    C_voigt[alpha, beta] = C[i, j, k, l] * factor
+
+    return C_voigt
+
+
+# -
+
+
+@task
+def run_one(model, row):
+    if Path(f"{model.name}.pkl").exists():
+        df = pd.read_pickle(f"{model.name}.pkl")
+
+        # if row.key_value_pairs.get('uid', None) in df['uid'].unique():
+        #     pass
+    else:
+        df = pd.DataFrame(columns=["model", "uid", "eigenvalues", "frequencies"])
+
+    atoms = row.toatoms()
+    # print(data := row.key_value_pairs)
+
+    calc = get_calculator(model)
+
+    result_opt = OPT(
+        atoms,
+        calc,
+        optimizer="FIRE",
+        criterion=dict(fmax=0.05, steps=500),
+        symmetry=True,
+    )
+
+    atoms = result_opt["atoms"]
+
+    result_elastic = ELASTICITY(
+        atoms,
+        calc,
+        optimizer="FIRE",
+        criterion=dict(fmax=0.05, steps=500),
+        pre_relax=False,
+    )
+
+    elastic_tensor = elastic_tensor_to_voigt(result_elastic["elastic_tensor"])
+    eigenvalues, eigenvectors = LA.eig(elastic_tensor)
+
+    outdir = Path(f"{model.name}") / row.key_value_pairs.get(
+        "uid", atoms.get_chemical_formula()
+    )
+    outdir.mkdir(parents=True, exist_ok=True)
+
+    np.savez(outdir / "elastic.npz", tensor=elastic_tensor, eigenvalues=eigenvalues)
+
+    result_phonon = PHONON(
+        atoms,
+        calc,
+        supercell_matrix=(2, 2, 1),
+        outdir=outdir,
+    )
+
+    frequencies = result_phonon["phonon"].get_frequencies(q=(0, 0, 0))
+
+    new_row = pd.DataFrame(
+        [
+            {
+                "model": model.name,
+                "uid": row.key_value_pairs.get("uid", None),
+                "eigenvalues": eigenvalues,
+                "frequencies": frequencies,
+            }
+        ]
+    )
+
+    df = pd.concat([df, new_row], ignore_index=True)
+    df.drop_duplicates(subset=["model", "uid"], keep="last", inplace=True)
+
+    df.to_pickle(f"{model.name}.pkl")
+
+
+@flow
+def run_all():
+    import random
+
+    random.seed(0)
+
+    futures = []
+    with connect("c2db.db") as db:
+        random_indices = random.sample(range(1, len(db) + 1), 1000)
+        for row, model in tqdm(
+            product(db.select(filter=lambda r: r["id"] in random_indices), MLIPEnum)
+        ):
+            if model.name not in select_models:
+                continue
+            future = run_one.submit(model, row)
+            futures.append(future)
+    return [f.result(raise_on_failure=False) for f in futures]
+
+
+# +
+
+
+if __name__ == "__main__":
+    nodes_per_alloc = 1
+    gpus_per_alloc = 1
+    ntasks = 1
+
+    cluster_kwargs = dict(
+        cores=1,
+        memory="64 GB",
+        processes=1,
+        shebang="#!/bin/bash",
+        account="matgen",
+        walltime="00:30:00",
+        # job_cpu=128,
+        job_mem="0",
+        job_script_prologue=[
+            "source ~/.bashrc",
+            "module load python",
+            "source activate /pscratch/sd/c/cyrusyc/.conda/dev",
+        ],
+        job_directives_skip=["-n", "--cpus-per-task", "-J"],
+        job_extra_directives=[
+            "-J c2db",
+            "-q regular",
+            f"-N {nodes_per_alloc}",
+            "-C gpu",
+            f"-G {gpus_per_alloc}",
+        ],
+    )
+
+    cluster = SLURMCluster(**cluster_kwargs)
+    print(cluster.job_script())
+    cluster.adapt(minimum_jobs=25, maximum_jobs=50)
+    client = Client(cluster)
+    # -
+
+    run_all.with_options(
+        task_runner=DaskTaskRunner(address=client.scheduler.address), log_prints=True
+    )()

benchmarks/combustion/H256O128.extxyz

@@ -0,0 +1,386 @@
+384
+Lattice="30.0 0.0 0.0 0.0 30.0 0.0 0.0 0.0 30.0" Properties=species:S:1:pos:R:3 Built=T with=T Packmol=T pbc="T T T"
+H       16.87897000      14.36340900       5.78639500
+H       17.43576500      14.42974000       6.26492700
+H        3.55483600       6.05817200      17.69903100
+H        3.93076700       5.56617900      17.29899100
+H        9.49657700      24.94365100      23.70565700
+H        8.79155100      24.79399100      23.55088300
+H       23.86076500      25.42809100       5.93695500
+H       24.14102300      25.10486100       6.53728200
+H        3.01569800      26.19656300      15.08020000
+H        2.75520000      25.98067600      14.42525900
+H        9.67788700      17.96928400       5.86481000
+H       10.22656200      18.46063200       5.83403500
+H       12.57870500      16.09171300      25.13668100
+H       12.52257400      16.32590900      24.43996300
+H       12.70913000       6.58341200      15.79997200
+H       13.14484100       6.28817300      16.31611500
+H       21.57221500      26.22007700       8.20775100
+H       21.49858000      25.76305400       7.63406000
+H        5.23150000       2.79202900      17.94668200
+H        5.34160000       2.30038000      17.40856200
+H        3.20140800      24.30032700      16.20596600
+H        3.01866600      23.59120400      16.29060400
+H       24.14000600      28.98000500      16.44843000
+H       24.06795800      28.97179200      15.71483900
+H        5.06521500      20.71716800       5.13534300
+H        4.35233500      20.56737200       5.02230400
+H       20.35575900      22.80574000      20.92112100
+H       20.33775300      22.82449400      20.18441300
+H        9.44001100      14.49775500      28.22965700
+H        8.79780000      14.40101700      27.88091500
+H        9.85111200      22.18687800       5.65961300
+H       10.23873200      22.52486700       5.13147700
+H        2.43073100      14.23882800       9.88843100
+H        2.41513200      14.74593300      10.42323500
+H       24.12085900      18.37439500      13.96560000
+H       24.11116500      17.72761900      14.31915600
+H        8.20116600      27.96832200      10.13677000
+H        7.93113800      28.48718300      10.58541500
+H        9.12140400      26.33764100       3.10957700
+H        9.77423300      26.15812100       3.40112700
+H        4.61212100       6.00445300       4.16492600
+H        4.64955500       6.71113500       4.37135300
+H       11.25926100      17.21528300      14.18800300
+H       11.45743100      17.84358800      14.51872400
+H        8.30883000      10.16305200      12.29874000
+H        7.72273600      10.57641900      12.12834100
+H       27.50695500      17.01494000       3.57254600
+H       26.93803700      16.58868000       3.76760800
+H        5.77076100       2.49322200      14.46871700
+H        5.62762700       2.75411500      15.14315000
+H        6.51732300      25.84318300      26.82939800
+H        6.55807500      25.12027600      26.69098100
+H       12.12323800      24.13458100      24.41591800
+H       12.45973600      24.45908800      23.84593700
+H        5.17331100      23.16438400      16.63650900
+H        4.88027500      22.87390600      17.24738200
+H        8.10959100      21.29987000       4.46261500
+H        7.63531700      20.73605600       4.43829600
+H       22.74497200      17.55560800       1.89494000
+H       22.79182400      17.39285800       2.61238800
+H       10.18128000      19.85571100      21.25732700
+H       10.60903100      20.45053300      21.17590400
+H        7.36765600      22.32656800      11.62091000
+H        7.36366200      22.32242700      12.35805300
+H       22.11051700      17.35754100      10.70421900
+H       21.63342400      16.93939000      10.32879200
+H       17.11800800      14.46985000      18.41272300
+H       17.76175400      14.14922200      18.57459600
+H       23.88598700      25.00383500      10.24699000
+H       24.23686000      25.54091700       9.88388500
+H       23.07268800      28.13033100      13.42606000
+H       22.86236700      28.70230100      13.84082400
+H       19.71929900      26.69691800      16.96391200
+H       19.77256100      26.69242100      16.22868700
+H       10.20897000       5.12509300      26.31992400
+H       10.27351800       5.67934800      25.83820900
+H        7.74945200      21.75294300      26.96569600
+H        7.85650000      21.94562800      26.26225700
+H       28.99846900       5.82412900       4.51948000
+H       28.84661100       5.17571900       4.20338400
+H       14.00669600      21.28067600      10.51077400
+H       14.02815700      20.57927500      10.28496600
+H       24.56688200       8.44802400      10.18169000
+H       24.87884700       8.74970200      10.77757700
+H       19.82097000       6.71060100      21.14119900
+H       20.20300600       6.57543700      20.52541300
+H       12.50680000      16.29448300      21.20002800
+H       12.58574100      16.94571300      21.53630400
+H        2.88750900       6.48194900      12.41992700
+H        2.63814400       5.97929100      11.94184100
+H       18.31924100      27.95926100      11.93343800
+H       18.08127600      28.28411700      12.55089700
+H        8.80421900      21.80835300      22.53574400
+H        8.19015200      22.18697500      22.38414900
+H       16.51146900      26.58804000      12.13737900
+H       16.51397100      26.60180800      11.40034500
+H        3.01154500       1.61542600      13.17298500
+H        2.31442300       1.37599500      13.16260300
+H       14.38141000      16.74073800      22.79765000
+H       14.54283900      16.49896600      23.47507200
+H       27.32062300      13.13890000       3.91277400
+H       27.68551200      13.68559000       4.24654100
+H       14.17603100      27.61355700       6.47229000
+H       14.22488300      28.10975000       5.92931700
+H        9.46812300       7.70071300       3.45368100
+H        9.61903800       8.06415600       4.07701700
+H        2.18384900       3.88122900      12.55886400
+H        2.16274500       3.79907900      11.82659400
+H        8.39628800      28.36448200      27.01582200
+H        7.83759500      27.92933600      26.81106900
+H       12.15799700      14.43608700      11.46881200
+H       11.91985600      14.40143500      10.77203200
+H       13.07421100      20.72377800      27.23797600
+H       12.38307500      20.94338900      27.37032300
+H        5.74890500      20.92045300      10.67404300
+H        6.07702100      21.30697500      10.13892200
+H        1.20101800       3.91867000       5.21703800
+H        1.33343400       3.87199400       5.94071000
+H       15.87910900       3.60421600      17.51325200
+H       16.55088000       3.81033800      17.29042700
+H       12.77118400      24.89708700      14.04640100
+H       12.93989800      25.26872400      13.43253200
+H        9.17302000       6.59818400       6.60346000
+H        9.20243000       6.69949200       7.33303900
+H        3.82873400      21.34798500      18.48337600
+H        4.44631900      21.08569500      18.78867100
+H        6.64065400      26.32535100       2.13715800
+H        7.07217200      25.92015200       1.69781700
+H       23.61875500      28.68096800       2.12527200
+H       24.20152800      28.50119700       1.71118000
+H       28.97955000      28.31110000      20.77510000
+H       28.51757300      27.79654500      21.03049100
+H        5.03608000       8.09600900      15.70374700
+H        5.01029200       8.77247600      15.41195800
+H        6.38181200      23.23894300      14.67726800
+H        6.72455700      23.67837800      15.15980100
+H        4.17225800      15.87846200      26.32328700
+H        3.60033300      16.10995900      25.91989500
+H        9.98548800      27.03736900      11.34257400
+H        9.89827200      26.31071900      11.43082100
+H       11.08624000       3.40906900      27.43653900
+H       11.03887400       3.46845400      28.16978100
+H       18.17462600      26.60855300      18.64341900
+H       17.84284900      27.25533700      18.76592900
+H       24.56041800      23.73132300      15.45710400
+H       24.15216800      23.55566200      16.04522700
+H       12.42149000       8.51852900      26.92738800
+H       12.85187200       8.54739800      26.32959800
+H       12.58494400      23.66894000      26.51841800
+H       11.87423100      23.85271600      26.58569800
+H       25.66776400      11.38987800       3.60979300
+H       25.04312400      11.04530700       3.79556200
+H       26.02410100      25.69683300       2.37428800
+H       26.49306900      25.51267100       1.83617300
+H        1.17207300      13.56847700      17.81787500
+H        1.09749900      13.56957200      17.08449200
+H       16.34427300      15.52582100      25.01154500
+H       16.21113800      15.84954200      25.66030800
+H        8.34844300       4.44909300      22.93722500
+H        7.86249000       4.49217500      23.48986100
+H       24.87485300       5.31864800       1.40835600
+H       25.19256300       5.84446800       1.00094100
+H        3.42570400      18.20815000      27.19415500
+H        2.77521600      18.34972600      26.87755500
+H       26.13269700      10.18845100      14.93319100
+H       25.85987900       9.71045000      14.44278400
+H       16.37800300       2.82461800      14.94014800
+H       16.88769100       2.31071600      14.80037500
+H        2.68461500      21.47767100      25.65109500
+H        2.86206000      21.90699200      25.07872200
+H       12.04855900       6.95917200      11.68418900
+H       12.39801200       7.51168600      11.34356500
+H       17.65835900      17.90689000      21.24729900
+H       17.57119400      17.94728700      20.51642000
+H       19.13123000       3.51526100      22.77599000
+H       18.50245100       3.89609300      22.72106000
+H       17.59014300      22.68251700       7.89677900
+H       17.94157500      23.30906000       7.73138900
+H       17.80517200      27.73224100      26.14671300
+H       17.17556300      27.67636700      25.76739200
+H        1.23383900      16.30952500      11.48348600
+H        1.21410800      16.78863900      10.92359800
+H        5.14106900       3.12858300      22.32613900
+H        4.84773300       3.33214200      22.97106700
+H        1.54627400      27.32284900      21.64647900
+H        1.89536100      27.35916600      22.29473300
+H       23.61494100       4.81278600      28.82197500
+H       23.15278300       4.71015800      28.25691600
+H       16.55009200       9.41232000       9.23163100
+H       16.39011400       8.95635500       9.78833300
+H        9.24672000      17.04310600      22.97075500
+H        9.69788100      16.57902500      23.32359500
+H        5.92999000       6.70344600      12.54324900
+H        6.12013700       6.02943400      12.77339200
+H       20.55806500      20.91865300      27.16038800
+H       20.17869900      21.08398200      27.77043800
+H        6.86904800      16.04280700      13.61109400
+H        7.39742300      16.55244000      13.54396000
+H        2.56041500      20.50409800       7.85659000
+H        1.94972600      20.22580200       8.16158900
+H       10.30946400      26.76199200      22.35478000
+H        9.74930900      26.93466800      22.80179500
+H        9.37796100       7.80951200      12.91974400
+H        8.83394300       8.02560600      12.47167900
+H        8.87347500      15.06285800       6.33146400
+H        8.47422200      14.92537200       6.93570600
+H        7.76353600       8.45873000      15.11360900
+H        7.75143800       8.55689900      14.38310900
+H        3.37293300      18.43420600      23.45136000
+H        3.96942500      18.08124700      23.70242100
+H       26.76422700      16.17472200      17.31888500
+H       26.40881800      16.38505300      17.92950700
+H       23.72340800      16.16827600      17.16088600
+H       23.54079100      15.74444300      16.58605400
+H        9.04422200      24.03668900      13.69386900
+H        8.77587200      23.93037800      14.37217600
+H       26.63067400      17.02388700       6.06368500
+H       26.82086300      16.77080800       6.72941200
+H       11.89740000       7.80260100      21.08513000
+H       12.35463200       8.38081400      21.08994900
+H       18.88034700      13.04076800       4.44410900
+H       18.99432900      13.09305500       5.17053000
+H       12.33984600      11.69259200      22.86388500
+H       12.74815800      11.21413800      23.24830300
+H        5.80287000      16.81823400      16.93960100
+H        5.49624100      17.28191100      17.42374600
+H       28.49793800       7.81327600      23.13893400
+H       28.47600900       7.97048500      22.41906000
+H       12.83557500      11.24373500      25.45085300
+H       12.41881800      11.24066000      26.05889700
+H        8.27635600      24.04023300       1.68334600
+H        7.96194000      23.42057200       1.43722300
+H       19.77199600      11.69701700       8.46538200
+H       19.78074500      11.70307100       9.20247100
+H       11.31673000      12.37398400      14.55721300
+H       10.84297500      12.47166100      14.00094900
+H       10.79125000      20.69161200       3.40245000
+H       11.27250900      20.38956300       3.87209900
+H       12.69025500      11.48337700       5.00482800
+H       11.98627900      11.67412000       4.89783200
+H       26.72371500       1.50329200       2.91105000
+H       26.51636700       2.14519300       3.20834400
+H        9.52738300      16.34429400      15.41539000
+H        8.94769200      16.45923000      15.85602700
+H       22.92343200      13.91655100      28.55448400
+H       22.91144200      13.99598900      27.82170900
+H       19.42780900       1.69485400       3.41368800
+H       18.91557000       1.36746900       3.83063400
+H        7.19465000      19.03417100      20.30585200
+H        6.81579600      18.83473100      20.90594100
+H        5.76403400       2.66353000      26.49024900
+H        6.17055300       2.09977300      26.24461900
+H       22.76304500       7.37905700      21.38041300
+H       22.27422500       7.45776800      21.92655800
+H       20.87103900       4.95557800      21.95393100
+H       20.54276300       4.55466100      21.42960800
+O        2.79836800       3.68628500      15.85501600
+O        2.81086000       3.61004300      17.09857400
+O       16.47940800       2.53818900      22.88295600
+O       16.11951200       2.71624200      24.06243800
+O       16.11955600      27.75767800      21.06500300
+O       16.53710300      26.62354600      21.36799700
+O        2.28922000       5.17821400      26.49769500
+O        2.38796000       5.21061100      25.25608000
+O       23.35821600      23.78530000       2.30561500
+O       22.62749600      23.99025500       3.29376900
+O        8.01645400       1.87918100      22.34390100
+O        7.71565300       1.00097600      23.17497300
+O       24.09262600      23.80245300      18.50597300
+O       25.13119000      23.75358600      19.19256200
+O       23.43919000       8.32924800       7.70010800
+O       22.65907800       8.89107400       8.49268800
+O       23.97254400      22.00311700       5.43698700
+O       23.89487400      22.49778100       6.57789900
+O       27.56254300       8.43488800       7.35581200
+O       27.43055900       8.25837300       6.12950500
+O       20.87006100      27.63986300      11.38162200
+O       20.86380900      27.63798600      12.62756100
+O        1.73533900      16.60794700       2.29549200
+O        1.55322100      16.46304200       1.07146500
+O        2.03508400      26.37263200       7.32515600
+O        1.51280300      27.39855300       7.80172200
+O        9.85776800      20.59545000      27.48500700
+O        9.78700000      20.58172200      26.24113800
+O        6.50961300      12.26791300      10.82158600
+O        6.77142400      11.85977400       9.67385600
+O       27.41406500      13.33114600      25.89152900
+O       27.64803100      14.38606200      25.27119400
+O       10.65841500      26.52178800      25.03188900
+O       11.32208500      27.05107500      25.94392100
+O       19.37531500      27.41779100      27.77152100
+O       19.50655800      28.65269600      27.87247500
+O       15.16713800      28.62232100      25.08493400
+O       15.07154300      28.26860400      26.27579700
+O       17.00751400      16.63266600      17.58492000
+O       17.59502600      16.71087600      16.48896400
+O       11.43827100      14.97615400       3.72092700
+O       10.81544400      15.61953100       4.58727500
+O       15.49533900      18.92456600       3.73508000
+O       15.41648300      19.38131500       2.57854700
+O       24.31615500       8.87176200       3.90336400
+O       24.24717100       8.63758100       2.68155900
+O        7.28827800      12.82799800      18.71906300
+O        6.55542500      13.39545800      19.55172300
+O       11.08744900       6.55884500      22.72049900
+O       12.06607400       6.27858300      23.43893700
+O        6.38492100      21.01048900      22.84221300
+O        6.12927500      19.99335000      23.51488000
+O       24.06443800       6.57624000       5.98481500
+O       24.52114300       5.76588200       6.81375800
+O       10.04719800       9.05734900       6.05599500
+O       10.94084300       9.28468500       6.89392400
+O       14.96821700      27.18796600      18.05445400
+O       15.07988500      27.85573600      17.00850000
+O        3.86508500       1.34322900      24.91310100
+O        3.95599400       1.18703300      26.14588000
+O       20.55715100      20.45753800      21.54897500
+O       19.73819000      20.46822400      22.48790900
+O       15.54716700      11.17956000      10.54123500
+O       15.52405700      11.70380700      11.67129500
+O       27.57196100      18.81697000      18.10230200
+O       28.42915400      18.07345000      18.61689400
+O       13.08594500      21.54734200      24.54405200
+O       14.11395800      21.16325000      25.13402600
+O       26.73666800      18.41630100      23.76197500
+O       26.39312500      17.34817200      24.30371700
+O        3.75150300      18.39675300       7.29152900
+O        3.53600500      18.86067200       6.15541900
+O       17.47592700      22.97389700      14.10462800
+O       17.52163900      23.02279100      15.34878500
+O       18.26366400       1.00092300      19.49677400
+O       18.47839600       1.21883800      20.70458600
+O       13.79429800      11.31045500      17.14879000
+O       14.75715400      11.10528500      16.38510100
+O       23.04461100      15.79867400      21.94317000
+O       24.21186100      16.06476100      22.28832400
+O       18.39899300      20.45438100       6.25233500
+O       18.17334100      19.38368000       5.65644400
+O       12.45708500      28.00076400      22.87959600
+O       12.89052700      26.91111300      23.30054100
+O       26.92707100       5.06905500       5.23373400
+O       25.77848800       5.33532200       4.83091500
+O       21.42684100      24.25676000      11.83089700
+O       21.61828400      25.40374200      11.38347100
+O        5.87933600      27.53672900      16.61577900
+O        4.73532800      27.38741900      17.08626700
+O       24.28470500       3.34526900       2.03178600
+O       24.76461000       2.73333800       1.05832000
+O       28.95365000       2.62596100      13.80868600
+O       28.43270400       3.61259000      13.25408200
+O        4.22171400      14.80081000      20.83480200
+O        4.21389800      14.77292800      19.58918200
+O       22.63711100       1.37775300      19.58937200
+O       23.87485500       1.47510200      19.48488400
+O       24.56301300      18.58751900      21.22319300
+O       23.67458500      19.30549400      20.72558500
+O        8.94762000      14.49730700      12.31189400
+O        9.38425200      15.14940800      13.27963600
+O       16.95516300       6.13080300      19.32331700
+O       16.15902300       5.87750900      18.39897300
+O        8.63504700      19.77961300       8.44117000
+O        9.68469700      20.38220800       8.73700800
+O       11.31282500       2.46998800       2.58334500
+O       11.07470700       1.38104200       2.02665600
+O       27.39479700      24.64841500       9.77024200
+O       26.71049700      25.67139800       9.57621600
+O        2.05963700       8.45476400      16.28023600
+O        2.01230900       8.30200000      17.51588500
+O        9.59314000      26.93822800       8.47063300
+O       10.09094600      25.91018600       7.97291300
+O       25.13932300       4.62957000       8.63843500
+O       25.91372600       5.60386200       8.57958800
+O       21.90534500       3.90912800      17.80877300
+O       21.81900900       3.92307100      19.05165600
+O       19.73746700       6.78254100       2.45368300
+O       19.39816400       6.01988000       3.37868500
+O        6.15237600      26.10484900      18.61638000
+O        4.93823500      25.98411400      18.86874900
+O        7.89451900       2.41931000      18.51611600
+O        7.61766700       2.45592400      17.30185900
+O       24.96184600       3.83646000      17.86831300
+O       24.40660700       3.89389800      16.75439300
+O       19.23185100       4.23752100       5.63971900
+O       20.27904100       4.39660700       4.98360000

benchmarks/combustion/README.md

@@ -0,0 +1,8 @@
+# MD reactivity with hydrogen combustion (A.7)
+
+The workflow and analysis are defined in a single Prefect flow, which can be imported by 
+
+```python
+from mlip_arena.tasks.combustion.flow import hydrogen_combustion
+```
+. See [run.ipynb](./run.ipynb) for detials.

benchmarks/combustion/chgnet/CHGNet_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ab2f6bd802d8f9491e03dcc15e1fe179f593921d1038101d588cd0bee22d54a
+size 58345

benchmarks/combustion/equiformer/EquiformerV2(OC20)_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b99da993c15df5700e06781eb85e46541557f33df40f6b8984d89add4b24cce
+size 67780

benchmarks/combustion/escn/eSCN(OC20)_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab10167bece4688dbb6ca867bc443219333f8dfef678eee891d535d5e5ceaa30
+size 138692

benchmarks/combustion/mace-mp/MACE-MP(M)_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fee4011f63fe132bb65c9dfdad1c0e13f9db937675bc49f077c4fba1ffeb874
+size 229391

benchmarks/combustion/mace-mp/MACE-MPA_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d54c6f3afbd7cebed96d7985990725b7311fb0ff2f0a747ef60c0467df1d92fd
+size 225547

benchmarks/combustion/matgl/M3GNet_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430d50172964ae7e03b4d0879e1d09daefea8d413e5169b9c7dd3271907e3196
+size 50493

benchmarks/combustion/mattersim/MatterSim_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c0c38af5321151ff4a3fc64935df168689030ba31cad0be2589379360b333b
+size 226556

benchmarks/combustion/orb/ORB_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6ebb15669863ba6c214444dea3041d9745dc9fe50c59d7ad2f21211404cfb6e
+size 239943

benchmarks/combustion/orb/ORBv2_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daf6ea77d138287df70ac9cba580407e3446c8bc0c85142761cea22349dbf5a8
+size 225159

benchmarks/combustion/run.ipynb

@@ -0,0 +1,228 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ab579ff1",
+   "metadata": {},
+   "source": [
+    "## Option 1: Run single flow with Prefect `ThreadPoolTaskRunner`.\n",
+    "\n",
+    "This method will start Prefect ThreadPoolTaskRunner by default and run tasks concurrently. Here only one single task is being run, so we won't see multiple tasks being run concurrently. See https://docs.prefect.io/v3/concepts/task-runners#available-task-runners for details."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa42c417-7377-4113-b4a1-2fc1d522a3e6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "from mlip_arena.models import REGISTRY\n",
+    "from mlip_arena.tasks.combustion.flow import hydrogen_combustion\n",
+    "\n",
+    "model = \"MACE-MPA\"\n",
+    "run_dir = Path(\".\").parent / REGISTRY[model][\"family\"]\n",
+    "\n",
+    "hydrogen_combustion.with_options(persist_result=True)(\n",
+    "    model=model,  # can be MLIPEnum name or ASE calculator object\n",
+    "    run_dir=run_dir,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b1c8b50e",
+   "metadata": {},
+   "source": [
+    "## Option 2: Run single flow with `DaskTaskRunner`.\n",
+    "\n",
+    "To achieve true parallelism, we can use `DaskTaskRunner` or `RayTaskRunner` to ask HPC resourcces and submit tasks to the cluster for queuing and execution. The task are automatically resubmitted if killed or running overtime. This might be more desirable option becase this task takes longer. See https://docs.prefect.io/integrations/prefect-dask/index#prefect-dask for details."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7bec78f8-0bd0-4a5f-a3bf-fbd2be98f9db",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.distributed import Client\n",
+    "from dask_jobqueue import SLURMCluster\n",
+    "from prefect_dask import DaskTaskRunner\n",
+    "\n",
+    "nodes_per_alloc = 1\n",
+    "gpus_per_alloc = 1\n",
+    "\n",
+    "cluster_kwargs = dict(\n",
+    "    cores=1,\n",
+    "    memory=\"64 GB\",\n",
+    "    processes=1,\n",
+    "    shebang=\"#!/bin/bash\",\n",
+    "    account=\"matgen\",\n",
+    "    walltime=\"04:00:00\",\n",
+    "    job_mem=\"0\",\n",
+    "    job_script_prologue=[\n",
+    "        \"source ~/.bashrc\",\n",
+    "        \"module load python\",\n",
+    "        \"source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena\",\n",
+    "    ],\n",
+    "    job_directives_skip=[\"-n\", \"--cpus-per-task\", \"-J\"],\n",
+    "    job_extra_directives=[\n",
+    "        \"-J arena-combustion\",\n",
+    "        \"-q preempt\",\n",
+    "        \"--time-min=00:30:00\",\n",
+    "        \"--comment=12:00:00\",\n",
+    "        f\"-N {nodes_per_alloc}\",\n",
+    "        \"-C gpu\",\n",
+    "        f\"-G {gpus_per_alloc}\",\n",
+    "    ],\n",
+    ")\n",
+    "cluster = SLURMCluster(**cluster_kwargs)\n",
+    "print(cluster.job_script())\n",
+    "cluster.adapt(minimum_jobs=1, maximum_jobs=2)\n",
+    "client = Client(cluster)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9fbb5f61-7940-4d37-a4ef-ec1b407b9c5d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hydrogen_combustion.with_options(\n",
+    "    persist_result=True,\n",
+    "    task_runner=DaskTaskRunner(address=client.scheduler.address),\n",
+    "    log_prints=True,\n",
+    ")(model=model, run_dir=run_dir)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {
+     "22140084d8b047aa8b04a991d81c30ac": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "29b37f8fd00546e6aee3f1cd8a1a9438": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "3b9b53f3e9e640c78cd15c16042fdb24": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "5396dbda76074f06a3ab8a978a572821": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     },
+     "54364b0188b042a9ac68590f8ca5c69e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "bar_style": "danger",
+       "layout": "IPY_MODEL_3b9b53f3e9e640c78cd15c16042fdb24",
+       "max": 2000000,
+       "style": "IPY_MODEL_b6307bc40b174c48b80d1d447d8fdaa5"
+      }
+     },
+     "652cfcaa7e8542b5b3a33ffce36d84f6": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_29b37f8fd00546e6aee3f1cd8a1a9438",
+       "style": "IPY_MODEL_ba4005a4dcc448b2be982387c82a91be",
+       "value": " 0/2000000 [02:32<?, ?it/s]"
+      }
+     },
+     "ac2c57e840774a598cae608b15d0b1d2": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "b41147f209144946914d5955fa380dfa": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "children": [
+        "IPY_MODEL_cacc853eb75e42b7a78b3e5e011b580f",
+        "IPY_MODEL_54364b0188b042a9ac68590f8ca5c69e",
+        "IPY_MODEL_652cfcaa7e8542b5b3a33ffce36d84f6"
+       ],
+       "layout": "IPY_MODEL_5396dbda76074f06a3ab8a978a572821"
+      }
+     },
+     "b6307bc40b174c48b80d1d447d8fdaa5": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "description_width": ""
+      }
+     },
+     "ba4005a4dcc448b2be982387c82a91be": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "cacc853eb75e42b7a78b3e5e011b580f": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_22140084d8b047aa8b04a991d81c30ac",
+       "style": "IPY_MODEL_ac2c57e840774a598cae608b15d0b1d2",
+       "value": "MD H256O128:   0%"
+      }
+     }
+    },
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

benchmarks/combustion/sevennet/SevenNet_H256O128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc465c2bb3212f9fb71f4d8d76b4b9db86bef4553f3434a52ca792cca6e4d3d7
+size 225986

benchmarks/diatomics/alignn/ALIGNN.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:325979c740279016c3652f570b742d0ed28065156e553ae6c878a775951c344b
+size 2383784

benchmarks/diatomics/ani/ANI2x.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f622fde1f33128568baa2beb374e33e112557581a2a5dc675ce8b6273a840a17
+size 121195

benchmarks/diatomics/chgnet/CHGNet.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3595e1ae02dbc7c30a8ff39227b2a2e46ea940be17e482200f8b3d518cda01
+size 2003404

benchmarks/diatomics/equiformer/EquiformerV2(OC20).json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b04fa7b4183d7005a68cb978645a7c8c7759629a58dc5143a0df9ae9834e13a
+size 1793152

benchmarks/diatomics/equiformer/EquiformerV2(OC22).json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab517a56946d8fde0a6f8c1af140d731698419a33a6438ebecca0ee6edafc195
+size 1971360

benchmarks/diatomics/escn/eSCN(OC20).json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bfb8f0a5424259c9e9686e3e8778adb36e9b08937296e2a127d6007a1adf3bf
+size 1960063