We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
The plot and report of Bayesmark scripts are inconsistent
### Expected behavior

Consistent results in the plots and `report.md`.

### Environment

GitHub Actions: https://github.com/optuna/optuna/blob/1a520bd5daa9ff0af09fb060464bb157f8af891b/.github/workflows/performance-benchmarks-bayesmark.yml

### Error messages, stack traces, or logs

```shell
https://github.com/optuna/optuna/actions/runs/3143078497

(See Additional context)
```


### Steps to reproduce

Run `Actions -> Performance benchmarks with bayesmark` with the default arguments.

### Additional context (optional)

The values in the plots and `report.md` seem to be inconsistent.
Example:
![image](https://user-images.githubusercontent.com/23695091/195053993-9fa16f86-25d8-4252-ad53-bc2f7ddff925.png)
![image](https://user-images.githubusercontent.com/23695091/195054314-8daf8e63-94bf-4615-a6f9-6221b5b59dbe.png)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `benchmarks/run_bayesmark.py`
Content:
```
1 import argparse
2 import json
3 import os
4 import subprocess
5 from typing import Dict
6 from typing import List
7 
8 from matplotlib import cm
9 from matplotlib import colors
10 from matplotlib.axes import Axes
11 import matplotlib.pyplot as plt
12 import numpy as np
13 import pandas as pd
14 from xarray import Dataset
15 
16 
17 _DB = "bo_optuna_run"
18 
19 
20 def run_benchmark(args: argparse.Namespace) -> None:
21 
22     sampler_list = args.sampler_list.split()
23     sampler_kwargs_list = args.sampler_kwargs_list.split()
24     pruner_list = args.pruner_list.split()
25     pruner_kwargs_list = args.pruner_kwargs_list.split()
26 
27     config = dict()
28     for sampler, sampler_kwargs in zip(sampler_list, sampler_kwargs_list):
29         for pruner, pruner_kwargs in zip(pruner_list, pruner_kwargs_list):
30             optimizer_name = f"{sampler}-{pruner}"
31             optimizer_kwargs = {
32                 "sampler": sampler,
33                 "sampler_kwargs": json.loads(sampler_kwargs),
34                 "pruner": pruner,
35                 "pruner_kwargs": json.loads(pruner_kwargs),
36             }
37             # We need to dynamically generate config.json sice sampler pruner combos (solvers)
38             # are parametrized by user. Following sample config schema.
39             # https://github.com/uber/bayesmark/blob/master/example_opt_root/config.json
40             config[optimizer_name] = ["optuna_optimizer.py", optimizer_kwargs]
41 
42     with open(os.path.join("benchmarks", "bayesmark", "config.json"), "w") as file:
43         json.dump(config, file, indent=4)
44 
45     samplers = " ".join(config.keys())
46     metric = "nll" if args.dataset in ["breast", "iris", "wine", "digits"] else "mse"
47     cmd = (
48         f"bayesmark-launch -n {args.budget} -r {args.n_runs} "
49         f"-dir runs -b {_DB} "
50         f"-o {samplers} "
51         f"-c {args.model} -d {args.dataset} "
52         f"-m {metric} --opt-root benchmarks/bayesmark"
53     )
54     subprocess.run(cmd, shell=True)
55 
56 
57 def make_plots(args: argparse.Namespace) -> None:
58 
59     filename = f"{args.dataset}-{args.model}-partial-report.json"
60     df = pd.read_json(os.path.join("partial", filename))
61     summaries = (
62         df.groupby(["opt", "iter"])
63         .generalization.agg(["mean", "std"])
64         .rename(columns={"mean": "best_mean", "std": "best_std"})
65         .reset_index()
66     )
67 
68     fig, ax = plt.subplots()
69     fig.set_size_inches(12, 8)
70     warmup = json.loads(args.plot_warmup)
71     metric = df.metric[0]
72     color_lookup = build_color_dict(sorted(df["opt"].unique()))
73 
74     for optimizer, summary in summaries.groupby("opt"):
75         color = color_lookup[optimizer]
76         make_plot(summary, ax, optimizer, metric, warmup, color)
77 
78     handles, labels = ax.get_legend_handles_labels()
79     fig.legend(handles, labels)
80     fig.suptitle(f"Bayesmark-{args.dataset.capitalize()}-{args.model}")
81     fig.savefig(os.path.join("plots", f"optuna-{args.dataset}-{args.model}-sumamry.png"))
82 
83 
84 def make_plot(
85     summary: pd.DataFrame,
86     ax: Axes,
87     optimizer: str,
88     metric: str,
89     plot_warmup: bool,
90     color: np.ndarray,
91 ) -> None:
92 
93     start = 0 if plot_warmup else 10
94     argpos = summary.best_mean.expanding().apply(np.argmin).astype(int)
95     best_found = summary.best_mean.values[argpos.values]
96     sdev = summary.best_std.values[argpos.values]
97 
98     if len(best_found) <= start:
99         return
100 
101     ax.fill_between(
102         np.arange(len(best_found))[start:],
103         (best_found - sdev)[start:],
104         (best_found + sdev)[start:],
105         color=color,
106         alpha=0.25,
107         step="mid",
108     )
109 
110     ax.plot(
111         np.arange(len(best_found))[start:],
112         best_found[start:],
113         color=color,
114         label=optimizer,
115         drawstyle="steps-mid",
116     )
117 
118     ax.set_xlabel("Budget", fontsize=10)
119     ax.set_ylabel(f"Validation {metric.upper()}", fontsize=10)
120     ax.grid(alpha=0.2)
121 
122 
123 def build_color_dict(names: List[str]) -> Dict[str, np.ndarray]:
124 
125     norm = colors.Normalize(vmin=0, vmax=1)
126     m = cm.ScalarMappable(norm, cm.tab20)
127     color_dict = m.to_rgba(np.linspace(0, 1, len(names)))
128     color_dict = dict(zip(names, color_dict))
129 
130     return color_dict
131 
132 
133 def partial_report(args: argparse.Namespace) -> None:
134 
135     eval_path = os.path.join("runs", _DB, "eval")
136     time_path = os.path.join("runs", _DB, "time")
137     studies = os.listdir(eval_path)
138     summaries: List[pd.DataFrame] = []
139 
140     for study in studies:
141         table_buffer: List[pd.DataFrame] = []
142         column_buffer: List[str] = []
143         for path in [eval_path, time_path]:
144             with open(os.path.join(path, study), "r") as file:
145                 data = json.load(file)
146                 df = Dataset.from_dict(data["data"]).to_dataframe()
147                 df = df.droplevel("suggestion")
148 
149             for argument, meatadata in data["meta"]["args"].items():
150                 colname = argument[2:] if argument.startswith("--") else argument
151                 if colname not in column_buffer:
152                     df[colname] = meatadata
153                     column_buffer.append(colname)
154 
155             table_buffer.append(df)
156 
157         summary = pd.merge(*table_buffer, left_index=True, right_index=True)
158         summaries.append(summary.reset_index())
159 
160     filename = f"{args.dataset}-{args.model}-partial-report.json"
161     pd.concat(summaries).reset_index(drop=True).to_json(os.path.join("partial", filename))
162 
163 
164 if __name__ == "__main__":
165     parser = argparse.ArgumentParser()
166     parser.add_argument("--dataset", type=str, default="iris")
167     parser.add_argument("--model", type=str, default="kNN")
168     parser.add_argument("--budget", type=int, default=80)
169     parser.add_argument("--n-runs", type=int, default=10)
170     parser.add_argument("--sampler-list", type=str, default="TPESampler CmaEsSampler")
171     parser.add_argument(
172         "--sampler-kwargs-list",
173         type=str,
174         default='{"multivariate":true,"constant_liar":true} {}',
175     )
176     parser.add_argument("--pruner-list", type=str, default="NopPruner")
177     parser.add_argument("--pruner-kwargs-list", type=str, default="{}")
178     parser.add_argument("--plot-warmup", type=str, default="true")
179 
180     args = parser.parse_args()
181     os.makedirs("runs", exist_ok=True)
182     os.makedirs("plots", exist_ok=True)
183     os.makedirs("partial", exist_ok=True)
184 
185     run_benchmark(args)
186     partial_report(args)
187     make_plots(args)
188
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/benchmarks/run_bayesmark.py b/benchmarks/run_bayesmark.py
--- a/benchmarks/run_bayesmark.py
+++ b/benchmarks/run_bayesmark.py
@@ -58,9 +58,10 @@
 
     filename = f"{args.dataset}-{args.model}-partial-report.json"
     df = pd.read_json(os.path.join("partial", filename))
+    df["best_value"] = df.groupby(["opt", "uuid"]).generalization.cummin()
     summaries = (
         df.groupby(["opt", "iter"])
-        .generalization.agg(["mean", "std"])
+        .best_value.agg(["mean", "std"])
         .rename(columns={"mean": "best_mean", "std": "best_std"})
         .reset_index()
     )
@@ -91,25 +92,21 @@
 ) -> None:
 
     start = 0 if plot_warmup else 10
-    argpos = summary.best_mean.expanding().apply(np.argmin).astype(int)
-    best_found = summary.best_mean.values[argpos.values]
-    sdev = summary.best_std.values[argpos.values]
-
-    if len(best_found) <= start:
+    if len(summary.best_mean) <= start:
         return
 
     ax.fill_between(
-        np.arange(len(best_found))[start:],
-        (best_found - sdev)[start:],
-        (best_found + sdev)[start:],
+        np.arange(len(summary.best_mean))[start:],
+        (summary.best_mean - summary.best_std)[start:],
+        (summary.best_mean + summary.best_std)[start:],
         color=color,
         alpha=0.25,
         step="mid",
     )
 
     ax.plot(
-        np.arange(len(best_found))[start:],
-        best_found[start:],
+        np.arange(len(summary.best_mean))[start:],
+        summary.best_mean[start:],
         color=color,
         label=optimizer,
         drawstyle="steps-mid",

{"golden_diff": "diff --git a/benchmarks/run_bayesmark.py b/benchmarks/run_bayesmark.py\n--- a/benchmarks/run_bayesmark.py\n+++ b/benchmarks/run_bayesmark.py\n@@ -58,9 +58,10 @@\n \n     filename = f\"{args.dataset}-{args.model}-partial-report.json\"\n     df = pd.read_json(os.path.join(\"partial\", filename))\n+    df[\"best_value\"] = df.groupby([\"opt\", \"uuid\"]).generalization.cummin()\n     summaries = (\n         df.groupby([\"opt\", \"iter\"])\n-        .generalization.agg([\"mean\", \"std\"])\n+        .best_value.agg([\"mean\", \"std\"])\n         .rename(columns={\"mean\": \"best_mean\", \"std\": \"best_std\"})\n         .reset_index()\n     )\n@@ -91,25 +92,21 @@\n ) -> None:\n \n     start = 0 if plot_warmup else 10\n-    argpos = summary.best_mean.expanding().apply(np.argmin).astype(int)\n-    best_found = summary.best_mean.values[argpos.values]\n-    sdev = summary.best_std.values[argpos.values]\n-\n-    if len(best_found) <= start:\n+    if len(summary.best_mean) <= start:\n         return\n \n     ax.fill_between(\n-        np.arange(len(best_found))[start:],\n-        (best_found - sdev)[start:],\n-        (best_found + sdev)[start:],\n+        np.arange(len(summary.best_mean))[start:],\n+        (summary.best_mean - summary.best_std)[start:],\n+        (summary.best_mean + summary.best_std)[start:],\n         color=color,\n         alpha=0.25,\n         step=\"mid\",\n     )\n \n     ax.plot(\n-        np.arange(len(best_found))[start:],\n-        best_found[start:],\n+        np.arange(len(summary.best_mean))[start:],\n+        summary.best_mean[start:],\n         color=color,\n         label=optimizer,\n         drawstyle=\"steps-mid\",\n", "issue": "The plot and report of Bayesmark scripts are inconsistent\n### Expected behavior\r\n\r\nConsistent results in the plots and `report.md`.\r\n\r\n### Environment\r\n\r\nGitHub Actions: https://github.com/optuna/optuna/blob/1a520bd5daa9ff0af09fb060464bb157f8af891b/.github/workflows/performance-benchmarks-bayesmark.yml\r\n\r\n### Error messages, stack traces, or logs\r\n\r\n```shell\r\nhttps://github.com/optuna/optuna/actions/runs/3143078497\r\n\r\n(See Additional context)\r\n```\r\n\r\n\r\n### Steps to reproduce\r\n\r\nRun `Actions -> Performance benchmarks with bayesmark` with the default arguments.\r\n\r\n### Additional context (optional)\r\n\r\nThe values in the plots and `report.md` seem to be inconsistent.\r\nExample:\r\n![image](https://user-images.githubusercontent.com/23695091/195053993-9fa16f86-25d8-4252-ad53-bc2f7ddff925.png)\r\n![image](https://user-images.githubusercontent.com/23695091/195054314-8daf8e63-94bf-4615-a6f9-6221b5b59dbe.png)\n", "before_files": [{"content": "import argparse\nimport json\nimport os\nimport subprocess\nfrom typing import Dict\nfrom typing import List\n\nfrom matplotlib import cm\nfrom matplotlib import colors\nfrom matplotlib.axes import Axes\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom xarray import Dataset\n\n\n_DB = \"bo_optuna_run\"\n\n\ndef run_benchmark(args: argparse.Namespace) -> None:\n\n    sampler_list = args.sampler_list.split()\n    sampler_kwargs_list = args.sampler_kwargs_list.split()\n    pruner_list = args.pruner_list.split()\n    pruner_kwargs_list = args.pruner_kwargs_list.split()\n\n    config = dict()\n    for sampler, sampler_kwargs in zip(sampler_list, sampler_kwargs_list):\n        for pruner, pruner_kwargs in zip(pruner_list, pruner_kwargs_list):\n            optimizer_name = f\"{sampler}-{pruner}\"\n            optimizer_kwargs = {\n                \"sampler\": sampler,\n                \"sampler_kwargs\": json.loads(sampler_kwargs),\n                \"pruner\": pruner,\n                \"pruner_kwargs\": json.loads(pruner_kwargs),\n            }\n            # We need to dynamically generate config.json sice sampler pruner combos (solvers)\n            # are parametrized by user. Following sample config schema.\n            # https://github.com/uber/bayesmark/blob/master/example_opt_root/config.json\n            config[optimizer_name] = [\"optuna_optimizer.py\", optimizer_kwargs]\n\n    with open(os.path.join(\"benchmarks\", \"bayesmark\", \"config.json\"), \"w\") as file:\n        json.dump(config, file, indent=4)\n\n    samplers = \" \".join(config.keys())\n    metric = \"nll\" if args.dataset in [\"breast\", \"iris\", \"wine\", \"digits\"] else \"mse\"\n    cmd = (\n        f\"bayesmark-launch -n {args.budget} -r {args.n_runs} \"\n        f\"-dir runs -b {_DB} \"\n        f\"-o {samplers} \"\n        f\"-c {args.model} -d {args.dataset} \"\n        f\"-m {metric} --opt-root benchmarks/bayesmark\"\n    )\n    subprocess.run(cmd, shell=True)\n\n\ndef make_plots(args: argparse.Namespace) -> None:\n\n    filename = f\"{args.dataset}-{args.model}-partial-report.json\"\n    df = pd.read_json(os.path.join(\"partial\", filename))\n    summaries = (\n        df.groupby([\"opt\", \"iter\"])\n        .generalization.agg([\"mean\", \"std\"])\n        .rename(columns={\"mean\": \"best_mean\", \"std\": \"best_std\"})\n        .reset_index()\n    )\n\n    fig, ax = plt.subplots()\n    fig.set_size_inches(12, 8)\n    warmup = json.loads(args.plot_warmup)\n    metric = df.metric[0]\n    color_lookup = build_color_dict(sorted(df[\"opt\"].unique()))\n\n    for optimizer, summary in summaries.groupby(\"opt\"):\n        color = color_lookup[optimizer]\n        make_plot(summary, ax, optimizer, metric, warmup, color)\n\n    handles, labels = ax.get_legend_handles_labels()\n    fig.legend(handles, labels)\n    fig.suptitle(f\"Bayesmark-{args.dataset.capitalize()}-{args.model}\")\n    fig.savefig(os.path.join(\"plots\", f\"optuna-{args.dataset}-{args.model}-sumamry.png\"))\n\n\ndef make_plot(\n    summary: pd.DataFrame,\n    ax: Axes,\n    optimizer: str,\n    metric: str,\n    plot_warmup: bool,\n    color: np.ndarray,\n) -> None:\n\n    start = 0 if plot_warmup else 10\n    argpos = summary.best_mean.expanding().apply(np.argmin).astype(int)\n    best_found = summary.best_mean.values[argpos.values]\n    sdev = summary.best_std.values[argpos.values]\n\n    if len(best_found) <= start:\n        return\n\n    ax.fill_between(\n        np.arange(len(best_found))[start:],\n        (best_found - sdev)[start:],\n        (best_found + sdev)[start:],\n        color=color,\n        alpha=0.25,\n        step=\"mid\",\n    )\n\n    ax.plot(\n        np.arange(len(best_found))[start:],\n        best_found[start:],\n        color=color,\n        label=optimizer,\n        drawstyle=\"steps-mid\",\n    )\n\n    ax.set_xlabel(\"Budget\", fontsize=10)\n    ax.set_ylabel(f\"Validation {metric.upper()}\", fontsize=10)\n    ax.grid(alpha=0.2)\n\n\ndef build_color_dict(names: List[str]) -> Dict[str, np.ndarray]:\n\n    norm = colors.Normalize(vmin=0, vmax=1)\n    m = cm.ScalarMappable(norm, cm.tab20)\n    color_dict = m.to_rgba(np.linspace(0, 1, len(names)))\n    color_dict = dict(zip(names, color_dict))\n\n    return color_dict\n\n\ndef partial_report(args: argparse.Namespace) -> None:\n\n    eval_path = os.path.join(\"runs\", _DB, \"eval\")\n    time_path = os.path.join(\"runs\", _DB, \"time\")\n    studies = os.listdir(eval_path)\n    summaries: List[pd.DataFrame] = []\n\n    for study in studies:\n        table_buffer: List[pd.DataFrame] = []\n        column_buffer: List[str] = []\n        for path in [eval_path, time_path]:\n            with open(os.path.join(path, study), \"r\") as file:\n                data = json.load(file)\n                df = Dataset.from_dict(data[\"data\"]).to_dataframe()\n                df = df.droplevel(\"suggestion\")\n\n            for argument, meatadata in data[\"meta\"][\"args\"].items():\n                colname = argument[2:] if argument.startswith(\"--\") else argument\n                if colname not in column_buffer:\n                    df[colname] = meatadata\n                    column_buffer.append(colname)\n\n            table_buffer.append(df)\n\n        summary = pd.merge(*table_buffer, left_index=True, right_index=True)\n        summaries.append(summary.reset_index())\n\n    filename = f\"{args.dataset}-{args.model}-partial-report.json\"\n    pd.concat(summaries).reset_index(drop=True).to_json(os.path.join(\"partial\", filename))\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--dataset\", type=str, default=\"iris\")\n    parser.add_argument(\"--model\", type=str, default=\"kNN\")\n    parser.add_argument(\"--budget\", type=int, default=80)\n    parser.add_argument(\"--n-runs\", type=int, default=10)\n    parser.add_argument(\"--sampler-list\", type=str, default=\"TPESampler CmaEsSampler\")\n    parser.add_argument(\n        \"--sampler-kwargs-list\",\n        type=str,\n        default='{\"multivariate\":true,\"constant_liar\":true} {}',\n    )\n    parser.add_argument(\"--pruner-list\", type=str, default=\"NopPruner\")\n    parser.add_argument(\"--pruner-kwargs-list\", type=str, default=\"{}\")\n    parser.add_argument(\"--plot-warmup\", type=str, default=\"true\")\n\n    args = parser.parse_args()\n    os.makedirs(\"runs\", exist_ok=True)\n    os.makedirs(\"plots\", exist_ok=True)\n    os.makedirs(\"partial\", exist_ok=True)\n\n    run_benchmark(args)\n    partial_report(args)\n    make_plots(args)\n", "path": "benchmarks/run_bayesmark.py"}], "after_files": [{"content": "import argparse\nimport json\nimport os\nimport subprocess\nfrom typing import Dict\nfrom typing import List\n\nfrom matplotlib import cm\nfrom matplotlib import colors\nfrom matplotlib.axes import Axes\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom xarray import Dataset\n\n\n_DB = \"bo_optuna_run\"\n\n\ndef run_benchmark(args: argparse.Namespace) -> None:\n\n    sampler_list = args.sampler_list.split()\n    sampler_kwargs_list = args.sampler_kwargs_list.split()\n    pruner_list = args.pruner_list.split()\n    pruner_kwargs_list = args.pruner_kwargs_list.split()\n\n    config = dict()\n    for sampler, sampler_kwargs in zip(sampler_list, sampler_kwargs_list):\n        for pruner, pruner_kwargs in zip(pruner_list, pruner_kwargs_list):\n            optimizer_name = f\"{sampler}-{pruner}\"\n            optimizer_kwargs = {\n                \"sampler\": sampler,\n                \"sampler_kwargs\": json.loads(sampler_kwargs),\n                \"pruner\": pruner,\n                \"pruner_kwargs\": json.loads(pruner_kwargs),\n            }\n            # We need to dynamically generate config.json sice sampler pruner combos (solvers)\n            # are parametrized by user. Following sample config schema.\n            # https://github.com/uber/bayesmark/blob/master/example_opt_root/config.json\n            config[optimizer_name] = [\"optuna_optimizer.py\", optimizer_kwargs]\n\n    with open(os.path.join(\"benchmarks\", \"bayesmark\", \"config.json\"), \"w\") as file:\n        json.dump(config, file, indent=4)\n\n    samplers = \" \".join(config.keys())\n    metric = \"nll\" if args.dataset in [\"breast\", \"iris\", \"wine\", \"digits\"] else \"mse\"\n    cmd = (\n        f\"bayesmark-launch -n {args.budget} -r {args.n_runs} \"\n        f\"-dir runs -b {_DB} \"\n        f\"-o {samplers} \"\n        f\"-c {args.model} -d {args.dataset} \"\n        f\"-m {metric} --opt-root benchmarks/bayesmark\"\n    )\n    subprocess.run(cmd, shell=True)\n\n\ndef make_plots(args: argparse.Namespace) -> None:\n\n    filename = f\"{args.dataset}-{args.model}-partial-report.json\"\n    df = pd.read_json(os.path.join(\"partial\", filename))\n    df[\"best_value\"] = df.groupby([\"opt\", \"uuid\"]).generalization.cummin()\n    summaries = (\n        df.groupby([\"opt\", \"iter\"])\n        .best_value.agg([\"mean\", \"std\"])\n        .rename(columns={\"mean\": \"best_mean\", \"std\": \"best_std\"})\n        .reset_index()\n    )\n\n    fig, ax = plt.subplots()\n    fig.set_size_inches(12, 8)\n    warmup = json.loads(args.plot_warmup)\n    metric = df.metric[0]\n    color_lookup = build_color_dict(sorted(df[\"opt\"].unique()))\n\n    for optimizer, summary in summaries.groupby(\"opt\"):\n        color = color_lookup[optimizer]\n        make_plot(summary, ax, optimizer, metric, warmup, color)\n\n    handles, labels = ax.get_legend_handles_labels()\n    fig.legend(handles, labels)\n    fig.suptitle(f\"Bayesmark-{args.dataset.capitalize()}-{args.model}\")\n    fig.savefig(os.path.join(\"plots\", f\"optuna-{args.dataset}-{args.model}-sumamry.png\"))\n\n\ndef make_plot(\n    summary: pd.DataFrame,\n    ax: Axes,\n    optimizer: str,\n    metric: str,\n    plot_warmup: bool,\n    color: np.ndarray,\n) -> None:\n\n    start = 0 if plot_warmup else 10\n    if len(summary.best_mean) <= start:\n        return\n\n    ax.fill_between(\n        np.arange(len(summary.best_mean))[start:],\n        (summary.best_mean - summary.best_std)[start:],\n        (summary.best_mean + summary.best_std)[start:],\n        color=color,\n        alpha=0.25,\n        step=\"mid\",\n    )\n\n    ax.plot(\n        np.arange(len(summary.best_mean))[start:],\n        summary.best_mean[start:],\n        color=color,\n        label=optimizer,\n        drawstyle=\"steps-mid\",\n    )\n\n    ax.set_xlabel(\"Budget\", fontsize=10)\n    ax.set_ylabel(f\"Validation {metric.upper()}\", fontsize=10)\n    ax.grid(alpha=0.2)\n\n\ndef build_color_dict(names: List[str]) -> Dict[str, np.ndarray]:\n\n    norm = colors.Normalize(vmin=0, vmax=1)\n    m = cm.ScalarMappable(norm, cm.tab20)\n    color_dict = m.to_rgba(np.linspace(0, 1, len(names)))\n    color_dict = dict(zip(names, color_dict))\n\n    return color_dict\n\n\ndef partial_report(args: argparse.Namespace) -> None:\n\n    eval_path = os.path.join(\"runs\", _DB, \"eval\")\n    time_path = os.path.join(\"runs\", _DB, \"time\")\n    studies = os.listdir(eval_path)\n    summaries: List[pd.DataFrame] = []\n\n    for study in studies:\n        table_buffer: List[pd.DataFrame] = []\n        column_buffer: List[str] = []\n        for path in [eval_path, time_path]:\n            with open(os.path.join(path, study), \"r\") as file:\n                data = json.load(file)\n                df = Dataset.from_dict(data[\"data\"]).to_dataframe()\n                df = df.droplevel(\"suggestion\")\n\n            for argument, meatadata in data[\"meta\"][\"args\"].items():\n                colname = argument[2:] if argument.startswith(\"--\") else argument\n                if colname not in column_buffer:\n                    df[colname] = meatadata\n                    column_buffer.append(colname)\n\n            table_buffer.append(df)\n\n        summary = pd.merge(*table_buffer, left_index=True, right_index=True)\n        summaries.append(summary.reset_index())\n\n    filename = f\"{args.dataset}-{args.model}-partial-report.json\"\n    pd.concat(summaries).reset_index(drop=True).to_json(os.path.join(\"partial\", filename))\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--dataset\", type=str, default=\"iris\")\n    parser.add_argument(\"--model\", type=str, default=\"kNN\")\n    parser.add_argument(\"--budget\", type=int, default=80)\n    parser.add_argument(\"--n-runs\", type=int, default=10)\n    parser.add_argument(\"--sampler-list\", type=str, default=\"TPESampler CmaEsSampler\")\n    parser.add_argument(\n        \"--sampler-kwargs-list\",\n        type=str,\n        default='{\"multivariate\":true,\"constant_liar\":true} {}',\n    )\n    parser.add_argument(\"--pruner-list\", type=str, default=\"NopPruner\")\n    parser.add_argument(\"--pruner-kwargs-list\", type=str, default=\"{}\")\n    parser.add_argument(\"--plot-warmup\", type=str, default=\"true\")\n\n    args = parser.parse_args()\n    os.makedirs(\"runs\", exist_ok=True)\n    os.makedirs(\"plots\", exist_ok=True)\n    os.makedirs(\"partial\", exist_ok=True)\n\n    run_benchmark(args)\n    partial_report(args)\n    make_plots(args)\n", "path": "benchmarks/run_bayesmark.py"}]}