We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Plot the results in cifar10 example
Hi, I ran the experiments (run_2 to run_10 in the cifar10 example) following the steps 3.1-3.6 and would like to plot the results and regenerate the figures you show. However, running this command as advised in step 3.3 to view the results does not work:
  cat ./workspaces/poc_workspace/server/run_2/cross_site_val/cross_site_val.json
Could you advise please on how to regenerate the results after completing all experiments?


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/cifar10/figs/plot_tensorboard_events.py`
Content:
```
1 # Copyright (c) 2021, NVIDIA CORPORATION.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import glob
16 import json
17 import os
18 
19 import matplotlib.pyplot as plt
20 import pandas as pd
21 import seaborn as sns
22 import tensorflow as tf
23 
24 client_results_root = "./workspaces/secure_workspace/site-1"
25 server_results_root = "./workspaces/secure_workspace/localhost"
26 
27 # 4.1 Central vs. FedAvg
28 experiments = {
29     "cifar10_central": {"run": "run_1", "tag": "val_acc_local_model"},
30     "cifar10_fedavg": {"run": "run_2", "tag": "val_acc_global_model"},
31     "cifar10_fedavg_he": {"run": "run_9", "tag": "val_acc_global_model"},
32 }
33 
34 # # 4.2 Impact of client data heterogeneity
35 # experiments = {"cifar10_fedavg (alpha=1.0)": {"run": "run_2", "tag": "val_acc_global_model"},
36 #                 "cifar10_fedavg (alpha=0.5)": {"run": "run_3", "tag": "val_acc_global_model"},
37 #                 "cifar10_fedavg (alpha=0.3)": {"run": "run_4", "tag": "val_acc_global_model"},
38 #                 "cifar10_fedavg (alpha=0.1)": {"run": "run_5", "tag": "val_acc_global_model"}}
39 #
40 # # 4.3 FedProx vs. FedOpt vs. SCAFFOLD
41 # experiments = {"cifar10_fedavg": {"run": "run_5", "tag": "val_acc_global_model"},
42 #                "cifar10_fedprox": {"run": "run_6", "tag": "val_acc_global_model"},
43 #                "cifar10_fedopt": {"run": "run_7", "tag": "val_acc_global_model"},
44 #                "cifar10_scaffold": {"run": "run_8", "tag": "val_acc_global_model"}}
45 
46 add_cross_site_val = True
47 
48 
49 def read_eventfile(filepath, tags=["val_acc_global_model"]):
50     data = {}
51     for summary in tf.compat.v1.train.summary_iterator(filepath):
52         for v in summary.summary.value:
53             if v.tag in tags:
54                 # print(v.tag, summary.step, v.simple_value)
55                 if v.tag in data.keys():
56                     data[v.tag].append([summary.step, v.simple_value])
57                 else:
58                     data[v.tag] = [[summary.step, v.simple_value]]
59     return data
60 
61 
62 def add_eventdata(data, config, filepath, tag="val_acc_global_model"):
63     event_data = read_eventfile(filepath, tags=[tag])
64 
65     assert len(event_data[tag]) > 0, f"No data for key {tag}"
66     # print(event_data)
67     for e in event_data[tag]:
68         # print(e)
69         data["Config"].append(config)
70         data["Step"].append(e[0])
71         data["Accuracy"].append(e[1])
72     print(f"added {len(event_data[tag])} entries for {tag}")
73 
74 
75 def main():
76     data = {"Config": [], "Step": [], "Accuracy": []}
77 
78     if add_cross_site_val:
79         xsite_keys = ["SRV_server", "SRV_server_best"]
80         xsite_data = {"Config": []}
81         for k in xsite_keys:
82             xsite_data.update({k: []})
83     else:
84         xsite_data = None
85         xsite_keys = None
86 
87     # add event files
88     for config, exp in experiments.items():
89         eventfile = glob.glob(os.path.join(client_results_root, exp["run"] + "/**/events.*"), recursive=True)
90         assert len(eventfile) == 1, "No unique event file found!"
91         eventfile = eventfile[0]
92         print("adding", eventfile)
93         add_eventdata(data, config, eventfile, tag=exp["tag"])
94 
95         if add_cross_site_val:
96             xsite_file = glob.glob(
97                 os.path.join(server_results_root, exp["run"] + "/**/cross_site_val.json"), recursive=True
98             )
99             assert len(xsite_file) == 1, "No unique x-site file found!"
100             with open(xsite_file[0], "r") as f:
101                 xsite_results = json.load(f)
102 
103             xsite_data["Config"].append(config)
104             for k in xsite_keys:
105                 xsite_data[k].append(xsite_results["site-1"][k]["val_accuracy"])
106 
107     print("Training TB data:")
108     print(pd.DataFrame(data))
109 
110     if xsite_data:
111         print("Cross-site val data:")
112         print(pd.DataFrame(xsite_data))
113 
114     sns.lineplot(x="Step", y="Accuracy", hue="Config", data=data)
115     plt.show()
116 
117 
118 if __name__ == "__main__":
119     main()
120
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/cifar10/figs/plot_tensorboard_events.py b/examples/cifar10/figs/plot_tensorboard_events.py
--- a/examples/cifar10/figs/plot_tensorboard_events.py
+++ b/examples/cifar10/figs/plot_tensorboard_events.py
@@ -76,7 +76,7 @@
     data = {"Config": [], "Step": [], "Accuracy": []}
 
     if add_cross_site_val:
-        xsite_keys = ["SRV_server", "SRV_server_best"]
+        xsite_keys = ["SRV_FL_global_model.pt", "SRV_best_FL_global_model.pt"]
         xsite_data = {"Config": []}
         for k in xsite_keys:
             xsite_data.update({k: []})
@@ -94,7 +94,7 @@
 
         if add_cross_site_val:
             xsite_file = glob.glob(
-                os.path.join(server_results_root, exp["run"] + "/**/cross_site_val.json"), recursive=True
+                os.path.join(server_results_root, exp["run"] + "/**/cross_val_results.json"), recursive=True
             )
             assert len(xsite_file) == 1, "No unique x-site file found!"
             with open(xsite_file[0], "r") as f:

{"golden_diff": "diff --git a/examples/cifar10/figs/plot_tensorboard_events.py b/examples/cifar10/figs/plot_tensorboard_events.py\n--- a/examples/cifar10/figs/plot_tensorboard_events.py\n+++ b/examples/cifar10/figs/plot_tensorboard_events.py\n@@ -76,7 +76,7 @@\n     data = {\"Config\": [], \"Step\": [], \"Accuracy\": []}\n \n     if add_cross_site_val:\n-        xsite_keys = [\"SRV_server\", \"SRV_server_best\"]\n+        xsite_keys = [\"SRV_FL_global_model.pt\", \"SRV_best_FL_global_model.pt\"]\n         xsite_data = {\"Config\": []}\n         for k in xsite_keys:\n             xsite_data.update({k: []})\n@@ -94,7 +94,7 @@\n \n         if add_cross_site_val:\n             xsite_file = glob.glob(\n-                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_site_val.json\"), recursive=True\n+                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_val_results.json\"), recursive=True\n             )\n             assert len(xsite_file) == 1, \"No unique x-site file found!\"\n             with open(xsite_file[0], \"r\") as f:\n", "issue": "Plot the results in cifar10 example\nHi, I ran the experiments (run_2 to run_10 in the cifar10 example) following the steps 3.1-3.6 and would like to plot the results and regenerate the figures you show. However, running this command as advised in step 3.3 to view the results does not work:\r\n  cat ./workspaces/poc_workspace/server/run_2/cross_site_val/cross_site_val.json\r\nCould you advise please on how to regenerate the results after completing all experiments?\r\n\n", "before_files": [{"content": "# Copyright (c) 2021, NVIDIA CORPORATION.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport json\nimport os\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\n\nclient_results_root = \"./workspaces/secure_workspace/site-1\"\nserver_results_root = \"./workspaces/secure_workspace/localhost\"\n\n# 4.1 Central vs. FedAvg\nexperiments = {\n    \"cifar10_central\": {\"run\": \"run_1\", \"tag\": \"val_acc_local_model\"},\n    \"cifar10_fedavg\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n    \"cifar10_fedavg_he\": {\"run\": \"run_9\", \"tag\": \"val_acc_global_model\"},\n}\n\n# # 4.2 Impact of client data heterogeneity\n# experiments = {\"cifar10_fedavg (alpha=1.0)\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.5)\": {\"run\": \"run_3\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.3)\": {\"run\": \"run_4\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.1)\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"}}\n#\n# # 4.3 FedProx vs. FedOpt vs. SCAFFOLD\n# experiments = {\"cifar10_fedavg\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedprox\": {\"run\": \"run_6\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedopt\": {\"run\": \"run_7\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_scaffold\": {\"run\": \"run_8\", \"tag\": \"val_acc_global_model\"}}\n\nadd_cross_site_val = True\n\n\ndef read_eventfile(filepath, tags=[\"val_acc_global_model\"]):\n    data = {}\n    for summary in tf.compat.v1.train.summary_iterator(filepath):\n        for v in summary.summary.value:\n            if v.tag in tags:\n                # print(v.tag, summary.step, v.simple_value)\n                if v.tag in data.keys():\n                    data[v.tag].append([summary.step, v.simple_value])\n                else:\n                    data[v.tag] = [[summary.step, v.simple_value]]\n    return data\n\n\ndef add_eventdata(data, config, filepath, tag=\"val_acc_global_model\"):\n    event_data = read_eventfile(filepath, tags=[tag])\n\n    assert len(event_data[tag]) > 0, f\"No data for key {tag}\"\n    # print(event_data)\n    for e in event_data[tag]:\n        # print(e)\n        data[\"Config\"].append(config)\n        data[\"Step\"].append(e[0])\n        data[\"Accuracy\"].append(e[1])\n    print(f\"added {len(event_data[tag])} entries for {tag}\")\n\n\ndef main():\n    data = {\"Config\": [], \"Step\": [], \"Accuracy\": []}\n\n    if add_cross_site_val:\n        xsite_keys = [\"SRV_server\", \"SRV_server_best\"]\n        xsite_data = {\"Config\": []}\n        for k in xsite_keys:\n            xsite_data.update({k: []})\n    else:\n        xsite_data = None\n        xsite_keys = None\n\n    # add event files\n    for config, exp in experiments.items():\n        eventfile = glob.glob(os.path.join(client_results_root, exp[\"run\"] + \"/**/events.*\"), recursive=True)\n        assert len(eventfile) == 1, \"No unique event file found!\"\n        eventfile = eventfile[0]\n        print(\"adding\", eventfile)\n        add_eventdata(data, config, eventfile, tag=exp[\"tag\"])\n\n        if add_cross_site_val:\n            xsite_file = glob.glob(\n                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_site_val.json\"), recursive=True\n            )\n            assert len(xsite_file) == 1, \"No unique x-site file found!\"\n            with open(xsite_file[0], \"r\") as f:\n                xsite_results = json.load(f)\n\n            xsite_data[\"Config\"].append(config)\n            for k in xsite_keys:\n                xsite_data[k].append(xsite_results[\"site-1\"][k][\"val_accuracy\"])\n\n    print(\"Training TB data:\")\n    print(pd.DataFrame(data))\n\n    if xsite_data:\n        print(\"Cross-site val data:\")\n        print(pd.DataFrame(xsite_data))\n\n    sns.lineplot(x=\"Step\", y=\"Accuracy\", hue=\"Config\", data=data)\n    plt.show()\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "examples/cifar10/figs/plot_tensorboard_events.py"}], "after_files": [{"content": "# Copyright (c) 2021, NVIDIA CORPORATION.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport json\nimport os\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\n\nclient_results_root = \"./workspaces/secure_workspace/site-1\"\nserver_results_root = \"./workspaces/secure_workspace/localhost\"\n\n# 4.1 Central vs. FedAvg\nexperiments = {\n    \"cifar10_central\": {\"run\": \"run_1\", \"tag\": \"val_acc_local_model\"},\n    \"cifar10_fedavg\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n    \"cifar10_fedavg_he\": {\"run\": \"run_9\", \"tag\": \"val_acc_global_model\"},\n}\n\n# # 4.2 Impact of client data heterogeneity\n# experiments = {\"cifar10_fedavg (alpha=1.0)\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.5)\": {\"run\": \"run_3\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.3)\": {\"run\": \"run_4\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.1)\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"}}\n#\n# # 4.3 FedProx vs. FedOpt vs. SCAFFOLD\n# experiments = {\"cifar10_fedavg\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedprox\": {\"run\": \"run_6\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedopt\": {\"run\": \"run_7\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_scaffold\": {\"run\": \"run_8\", \"tag\": \"val_acc_global_model\"}}\n\nadd_cross_site_val = True\n\n\ndef read_eventfile(filepath, tags=[\"val_acc_global_model\"]):\n    data = {}\n    for summary in tf.compat.v1.train.summary_iterator(filepath):\n        for v in summary.summary.value:\n            if v.tag in tags:\n                # print(v.tag, summary.step, v.simple_value)\n                if v.tag in data.keys():\n                    data[v.tag].append([summary.step, v.simple_value])\n                else:\n                    data[v.tag] = [[summary.step, v.simple_value]]\n    return data\n\n\ndef add_eventdata(data, config, filepath, tag=\"val_acc_global_model\"):\n    event_data = read_eventfile(filepath, tags=[tag])\n\n    assert len(event_data[tag]) > 0, f\"No data for key {tag}\"\n    # print(event_data)\n    for e in event_data[tag]:\n        # print(e)\n        data[\"Config\"].append(config)\n        data[\"Step\"].append(e[0])\n        data[\"Accuracy\"].append(e[1])\n    print(f\"added {len(event_data[tag])} entries for {tag}\")\n\n\ndef main():\n    data = {\"Config\": [], \"Step\": [], \"Accuracy\": []}\n\n    if add_cross_site_val:\n        xsite_keys = [\"SRV_FL_global_model.pt\", \"SRV_best_FL_global_model.pt\"]\n        xsite_data = {\"Config\": []}\n        for k in xsite_keys:\n            xsite_data.update({k: []})\n    else:\n        xsite_data = None\n        xsite_keys = None\n\n    # add event files\n    for config, exp in experiments.items():\n        eventfile = glob.glob(os.path.join(client_results_root, exp[\"run\"] + \"/**/events.*\"), recursive=True)\n        assert len(eventfile) == 1, \"No unique event file found!\"\n        eventfile = eventfile[0]\n        print(\"adding\", eventfile)\n        add_eventdata(data, config, eventfile, tag=exp[\"tag\"])\n\n        if add_cross_site_val:\n            xsite_file = glob.glob(\n                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_val_results.json\"), recursive=True\n            )\n            assert len(xsite_file) == 1, \"No unique x-site file found!\"\n            with open(xsite_file[0], \"r\") as f:\n                xsite_results = json.load(f)\n\n            xsite_data[\"Config\"].append(config)\n            for k in xsite_keys:\n                xsite_data[k].append(xsite_results[\"site-1\"][k][\"val_accuracy\"])\n\n    print(\"Training TB data:\")\n    print(pd.DataFrame(data))\n\n    if xsite_data:\n        print(\"Cross-site val data:\")\n        print(pd.DataFrame(xsite_data))\n\n    sns.lineplot(x=\"Step\", y=\"Accuracy\", hue=\"Config\", data=data)\n    plt.show()\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "examples/cifar10/figs/plot_tensorboard_events.py"}]}