# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %%capture
# %cd ..
from pprint import pprint
import matplotlib.pyplot as plt

# # Importing

from api import Benchmark

bench_dir = "cached/six_datasets.json"
bench = Benchmark(bench_dir, cache=False)

# # API exploration

# ### Queryable tags
#
# Tags starting with "Train/" indicate metrics which are logged every epoch.

queriable_tags = bench.get_queriable_tags()
pprint(queriable_tags)

# ### Datasets

# +
dataset_names = bench.get_dataset_names()
openml_task_ids = bench.get_openml_task_ids()

print(dataset_names)
print(openml_task_ids)
# -

# ### Querying

# +
# Get an example for a loss log
example_loss = bench.query(dataset_name="higgs", tag="Train/loss", config_id=0)

# Get the log of the accuracy for the run with the best peak accuracy
example_best_acc = bench.query_best("higgs", "Train/val_accuracy", "Train/val_accuracy", 0)

# Get the configuration of the best performing configuration
example_best_config = bench.query_best("higgs", "config", "Train/val_accuracy", 1)

print("Example loss log:\n", example_loss)
print("Best validation accuracy log:\n", example_best_acc)
print("Best config with regard to validation accuracy:\n", example_best_config)
# -

# ### Plotting
#
# The _plot\__by\__name_ method allows you to quickly look at some logs for a number of datasets.

help(bench.plot_by_name)

bench.plot_by_name(dataset_names=["higgs", "jasmine"],
                   x_col="epoch",
                   y_col="Train/val_accuracy",
                   n_configs=10,
                   xscale='linear',
                   yscale='linear',
                   show_best=False)

StarcoderdataJupyterScriptsDedupFiltered