We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Improve multi-objective optimization tutorial regarding `Study.best_trials`
### What is an issue?

## Background

https://github.com/optuna/optuna/issues/3397 added a note about `study.best_trials` to the [reuse the best trial tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/010_reuse_best_trial.html). During the discussion, we found that the [multi-objective optimization tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/002_multi_objective.html) does not fetch the trials on the Pareto front (i.e., `Study.best_trials`) while it visualize the Pareto front.

## Description

I think readers can understand how to use the trials on Pareto front more easily if we mention `Study.best_trials` in the tutorial page. We may have some possible scenarios. For example,

1. Show the Pareto front as can be seen in the current tutorial
2. Explain how to get the trials on the Pareto front by using `Study.best_trials`
3. Demonstrate how to get the desired trial from the Pareto front i.e., the trial whose accuracy is highest, and show the parameter values (or objective values)

![image](https://user-images.githubusercontent.com/3255979/160827518-279a20e7-1c65-4fb7-958c-962e98902864.png)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `tutorial/20_recipes/002_multi_objective.py`
Content:
```
1 """
2 .. _multi_objective:
3 
4 Multi-objective Optimization with Optuna
5 ========================================
6 
7 This tutorial showcases Optuna's multi-objective optimization feature by
8 optimizing the validation accuracy of Fashion MNIST dataset and the FLOPS of the model implemented in PyTorch.
9 
10 We use `thop <https://github.com/Lyken17/pytorch-OpCounter>`_ to measure FLOPS.
11 """
12 
13 import thop
14 import torch
15 import torch.nn as nn
16 import torch.nn.functional as F
17 import torchvision
18 
19 import optuna
20 
21 
22 DEVICE = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
23 DIR = ".."
24 BATCHSIZE = 128
25 N_TRAIN_EXAMPLES = BATCHSIZE * 30
26 N_VALID_EXAMPLES = BATCHSIZE * 10
27 
28 
29 def define_model(trial):
30     n_layers = trial.suggest_int("n_layers", 1, 3)
31     layers = []
32 
33     in_features = 28 * 28
34     for i in range(n_layers):
35         out_features = trial.suggest_int("n_units_l{}".format(i), 4, 128)
36         layers.append(nn.Linear(in_features, out_features))
37         layers.append(nn.ReLU())
38         p = trial.suggest_float("dropout_{}".format(i), 0.2, 0.5)
39         layers.append(nn.Dropout(p))
40 
41         in_features = out_features
42 
43     layers.append(nn.Linear(in_features, 10))
44     layers.append(nn.LogSoftmax(dim=1))
45 
46     return nn.Sequential(*layers)
47 
48 
49 # Defines training and evaluation.
50 def train_model(model, optimizer, train_loader):
51     model.train()
52     for batch_idx, (data, target) in enumerate(train_loader):
53         data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
54         optimizer.zero_grad()
55         F.nll_loss(model(data), target).backward()
56         optimizer.step()
57 
58 
59 def eval_model(model, valid_loader):
60     model.eval()
61     correct = 0
62     with torch.no_grad():
63         for batch_idx, (data, target) in enumerate(valid_loader):
64             data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
65             pred = model(data).argmax(dim=1, keepdim=True)
66             correct += pred.eq(target.view_as(pred)).sum().item()
67 
68     accuracy = correct / N_VALID_EXAMPLES
69 
70     flops, _ = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)
71     return flops, accuracy
72 
73 
74 ###################################################################################################
75 # Define multi-objective objective function.
76 # Objectives are FLOPS and accuracy.
77 def objective(trial):
78     train_dataset = torchvision.datasets.FashionMNIST(
79         DIR, train=True, download=True, transform=torchvision.transforms.ToTensor()
80     )
81     train_loader = torch.utils.data.DataLoader(
82         torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),
83         batch_size=BATCHSIZE,
84         shuffle=True,
85     )
86 
87     val_dataset = torchvision.datasets.FashionMNIST(
88         DIR, train=False, transform=torchvision.transforms.ToTensor()
89     )
90     val_loader = torch.utils.data.DataLoader(
91         torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),
92         batch_size=BATCHSIZE,
93         shuffle=True,
94     )
95     model = define_model(trial).to(DEVICE)
96 
97     optimizer = torch.optim.Adam(
98         model.parameters(), trial.suggest_float("lr", 1e-5, 1e-1, log=True)
99     )
100 
101     for epoch in range(10):
102         train_model(model, optimizer, train_loader)
103     flops, accuracy = eval_model(model, val_loader)
104     return flops, accuracy
105 
106 
107 ###################################################################################################
108 # Run multi-objective optimization
109 # --------------------------------
110 #
111 # If your optimization problem is multi-objective,
112 # Optuna assumes that you will specify the optimization direction for each objective.
113 # Specifically, in this example, we want to minimize the FLOPS (we want a faster model)
114 # and maximize the accuracy. So we set ``directions`` to ``["minimize", "maximize"]``.
115 study = optuna.create_study(directions=["minimize", "maximize"])
116 study.optimize(objective, n_trials=30, timeout=300)
117 
118 print("Number of finished trials: ", len(study.trials))
119 
120 
121 ###################################################################################################
122 # Check trials on pareto front visually
123 optuna.visualization.plot_pareto_front(study, target_names=["FLOPS", "accuracy"])
124 
125 
126 ###################################################################################################
127 # Learn which hyperparameters are affecting the flops most with hyperparameter importance.
128 optuna.visualization.plot_param_importances(
129     study, target=lambda t: t.values[0], target_name="flops"
130 )
131
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/tutorial/20_recipes/002_multi_objective.py b/tutorial/20_recipes/002_multi_objective.py
--- a/tutorial/20_recipes/002_multi_objective.py
+++ b/tutorial/20_recipes/002_multi_objective.py
@@ -119,10 +119,23 @@
 
 
 ###################################################################################################
-# Check trials on pareto front visually
+# Check trials on Pareto front visually.
 optuna.visualization.plot_pareto_front(study, target_names=["FLOPS", "accuracy"])
 
 
+###################################################################################################
+# Fetch the list of trials on the Pareto front with :attr:`~optuna.study.Study.best_trials`.
+#
+# For example, the following code shows the number of trials on the Pareto front and picks the trial with the highest accuracy.
+
+print(f"Number of trials on the Pareto front: {len(study.best_trials)}")
+
+trial_with_highest_accuracy = max(study.best_trials, key=lambda t: t.values[1])
+print(f"Trial with highest accuracy: ")
+print(f"\tnumber: {trial_with_highest_accuracy.number}")
+print(f"\tparams: {trial_with_highest_accuracy.params}")
+print(f"\tvalues: {trial_with_highest_accuracy.values}")
+
 ###################################################################################################
 # Learn which hyperparameters are affecting the flops most with hyperparameter importance.
 optuna.visualization.plot_param_importances(

{"golden_diff": "diff --git a/tutorial/20_recipes/002_multi_objective.py b/tutorial/20_recipes/002_multi_objective.py\n--- a/tutorial/20_recipes/002_multi_objective.py\n+++ b/tutorial/20_recipes/002_multi_objective.py\n@@ -119,10 +119,23 @@\n \n \n ###################################################################################################\n-# Check trials on pareto front visually\n+# Check trials on Pareto front visually.\n optuna.visualization.plot_pareto_front(study, target_names=[\"FLOPS\", \"accuracy\"])\n \n \n+###################################################################################################\n+# Fetch the list of trials on the Pareto front with :attr:`~optuna.study.Study.best_trials`.\n+#\n+# For example, the following code shows the number of trials on the Pareto front and picks the trial with the highest accuracy.\n+\n+print(f\"Number of trials on the Pareto front: {len(study.best_trials)}\")\n+\n+trial_with_highest_accuracy = max(study.best_trials, key=lambda t: t.values[1])\n+print(f\"Trial with highest accuracy: \")\n+print(f\"\\tnumber: {trial_with_highest_accuracy.number}\")\n+print(f\"\\tparams: {trial_with_highest_accuracy.params}\")\n+print(f\"\\tvalues: {trial_with_highest_accuracy.values}\")\n+\n ###################################################################################################\n # Learn which hyperparameters are affecting the flops most with hyperparameter importance.\n optuna.visualization.plot_param_importances(\n", "issue": "Improve multi-objective optimization tutorial regarding `Study.best_trials`\n### What is an issue?\n\n## Background\r\n\r\nhttps://github.com/optuna/optuna/issues/3397 added a note about `study.best_trials` to the [reuse the best trial tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/010_reuse_best_trial.html). During the discussion, we found that the [multi-objective optimization tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/002_multi_objective.html) does not fetch the trials on the Pareto front (i.e., `Study.best_trials`) while it visualize the Pareto front.\r\n\r\n## Description\r\n\r\nI think readers can understand how to use the trials on Pareto front more easily if we mention `Study.best_trials` in the tutorial page. We may have some possible scenarios. For example,\r\n\r\n1. Show the Pareto front as can be seen in the current tutorial\r\n2. Explain how to get the trials on the Pareto front by using `Study.best_trials`\r\n3. Demonstrate how to get the desired trial from the Pareto front i.e., the trial whose accuracy is highest, and show the parameter values (or objective values)\r\n\r\n![image](https://user-images.githubusercontent.com/3255979/160827518-279a20e7-1c65-4fb7-958c-962e98902864.png)\r\n\n", "before_files": [{"content": "\"\"\"\n.. _multi_objective:\n\nMulti-objective Optimization with Optuna\n========================================\n\nThis tutorial showcases Optuna's multi-objective optimization feature by\noptimizing the validation accuracy of Fashion MNIST dataset and the FLOPS of the model implemented in PyTorch.\n\nWe use `thop <https://github.com/Lyken17/pytorch-OpCounter>`_ to measure FLOPS.\n\"\"\"\n\nimport thop\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nimport optuna\n\n\nDEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nDIR = \"..\"\nBATCHSIZE = 128\nN_TRAIN_EXAMPLES = BATCHSIZE * 30\nN_VALID_EXAMPLES = BATCHSIZE * 10\n\n\ndef define_model(trial):\n    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n    layers = []\n\n    in_features = 28 * 28\n    for i in range(n_layers):\n        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 128)\n        layers.append(nn.Linear(in_features, out_features))\n        layers.append(nn.ReLU())\n        p = trial.suggest_float(\"dropout_{}\".format(i), 0.2, 0.5)\n        layers.append(nn.Dropout(p))\n\n        in_features = out_features\n\n    layers.append(nn.Linear(in_features, 10))\n    layers.append(nn.LogSoftmax(dim=1))\n\n    return nn.Sequential(*layers)\n\n\n# Defines training and evaluation.\ndef train_model(model, optimizer, train_loader):\n    model.train()\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n        optimizer.zero_grad()\n        F.nll_loss(model(data), target).backward()\n        optimizer.step()\n\n\ndef eval_model(model, valid_loader):\n    model.eval()\n    correct = 0\n    with torch.no_grad():\n        for batch_idx, (data, target) in enumerate(valid_loader):\n            data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n            pred = model(data).argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    accuracy = correct / N_VALID_EXAMPLES\n\n    flops, _ = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)\n    return flops, accuracy\n\n\n###################################################################################################\n# Define multi-objective objective function.\n# Objectives are FLOPS and accuracy.\ndef objective(trial):\n    train_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=True, download=True, transform=torchvision.transforms.ToTensor()\n    )\n    train_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n\n    val_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=False, transform=torchvision.transforms.ToTensor()\n    )\n    val_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n    model = define_model(trial).to(DEVICE)\n\n    optimizer = torch.optim.Adam(\n        model.parameters(), trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n    )\n\n    for epoch in range(10):\n        train_model(model, optimizer, train_loader)\n    flops, accuracy = eval_model(model, val_loader)\n    return flops, accuracy\n\n\n###################################################################################################\n# Run multi-objective optimization\n# --------------------------------\n#\n# If your optimization problem is multi-objective,\n# Optuna assumes that you will specify the optimization direction for each objective.\n# Specifically, in this example, we want to minimize the FLOPS (we want a faster model)\n# and maximize the accuracy. So we set ``directions`` to ``[\"minimize\", \"maximize\"]``.\nstudy = optuna.create_study(directions=[\"minimize\", \"maximize\"])\nstudy.optimize(objective, n_trials=30, timeout=300)\n\nprint(\"Number of finished trials: \", len(study.trials))\n\n\n###################################################################################################\n# Check trials on pareto front visually\noptuna.visualization.plot_pareto_front(study, target_names=[\"FLOPS\", \"accuracy\"])\n\n\n###################################################################################################\n# Learn which hyperparameters are affecting the flops most with hyperparameter importance.\noptuna.visualization.plot_param_importances(\n    study, target=lambda t: t.values[0], target_name=\"flops\"\n)\n", "path": "tutorial/20_recipes/002_multi_objective.py"}], "after_files": [{"content": "\"\"\"\n.. _multi_objective:\n\nMulti-objective Optimization with Optuna\n========================================\n\nThis tutorial showcases Optuna's multi-objective optimization feature by\noptimizing the validation accuracy of Fashion MNIST dataset and the FLOPS of the model implemented in PyTorch.\n\nWe use `thop <https://github.com/Lyken17/pytorch-OpCounter>`_ to measure FLOPS.\n\"\"\"\n\nimport thop\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nimport optuna\n\n\nDEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nDIR = \"..\"\nBATCHSIZE = 128\nN_TRAIN_EXAMPLES = BATCHSIZE * 30\nN_VALID_EXAMPLES = BATCHSIZE * 10\n\n\ndef define_model(trial):\n    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n    layers = []\n\n    in_features = 28 * 28\n    for i in range(n_layers):\n        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 128)\n        layers.append(nn.Linear(in_features, out_features))\n        layers.append(nn.ReLU())\n        p = trial.suggest_float(\"dropout_{}\".format(i), 0.2, 0.5)\n        layers.append(nn.Dropout(p))\n\n        in_features = out_features\n\n    layers.append(nn.Linear(in_features, 10))\n    layers.append(nn.LogSoftmax(dim=1))\n\n    return nn.Sequential(*layers)\n\n\n# Defines training and evaluation.\ndef train_model(model, optimizer, train_loader):\n    model.train()\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n        optimizer.zero_grad()\n        F.nll_loss(model(data), target).backward()\n        optimizer.step()\n\n\ndef eval_model(model, valid_loader):\n    model.eval()\n    correct = 0\n    with torch.no_grad():\n        for batch_idx, (data, target) in enumerate(valid_loader):\n            data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n            pred = model(data).argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    accuracy = correct / N_VALID_EXAMPLES\n\n    flops, _ = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)\n    return flops, accuracy\n\n\n###################################################################################################\n# Define multi-objective objective function.\n# Objectives are FLOPS and accuracy.\ndef objective(trial):\n    train_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=True, download=True, transform=torchvision.transforms.ToTensor()\n    )\n    train_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n\n    val_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=False, transform=torchvision.transforms.ToTensor()\n    )\n    val_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n    model = define_model(trial).to(DEVICE)\n\n    optimizer = torch.optim.Adam(\n        model.parameters(), trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n    )\n\n    for epoch in range(10):\n        train_model(model, optimizer, train_loader)\n    flops, accuracy = eval_model(model, val_loader)\n    return flops, accuracy\n\n\n###################################################################################################\n# Run multi-objective optimization\n# --------------------------------\n#\n# If your optimization problem is multi-objective,\n# Optuna assumes that you will specify the optimization direction for each objective.\n# Specifically, in this example, we want to minimize the FLOPS (we want a faster model)\n# and maximize the accuracy. So we set ``directions`` to ``[\"minimize\", \"maximize\"]``.\nstudy = optuna.create_study(directions=[\"minimize\", \"maximize\"])\nstudy.optimize(objective, n_trials=30, timeout=300)\n\nprint(\"Number of finished trials: \", len(study.trials))\n\n\n###################################################################################################\n# Check trials on Pareto front visually.\noptuna.visualization.plot_pareto_front(study, target_names=[\"FLOPS\", \"accuracy\"])\n\n\n###################################################################################################\n# Fetch the list of trials on the Pareto front with :attr:`~optuna.study.Study.best_trials`.\n#\n# For example, the following code shows the number of trials on the Pareto front and picks the trial with the highest accuracy.\n\nprint(f\"Number of trials on the Pareto front: {len(study.best_trials)}\")\n\ntrial_with_highest_accuracy = max(study.best_trials, key=lambda t: t.values[1])\nprint(f\"Trial with highest accuracy: \")\nprint(f\"\\tnumber: {trial_with_highest_accuracy.number}\")\nprint(f\"\\tparams: {trial_with_highest_accuracy.params}\")\nprint(f\"\\tvalues: {trial_with_highest_accuracy.values}\")\n\n###################################################################################################\n# Learn which hyperparameters are affecting the flops most with hyperparameter importance.\noptuna.visualization.plot_param_importances(\n    study, target=lambda t: t.values[0], target_name=\"flops\"\n)\n", "path": "tutorial/20_recipes/002_multi_objective.py"}]}