We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
support greater_or_equal from ignite 0.4.4
**Is your feature request related to a problem? Please describe.**
In ignite 0.4.4, @vfdev-5 and I developed a new arg in CheckpointHandler `greater_or_equal`, which is a feature request from our interval research team.
https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L101
Need to add this support in MONAI as MONAI already updated the ignite dependency to 0.4.4.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `monai/handlers/checkpoint_saver.py`
Content:
```
1 # Copyright 2020 - 2021 MONAI Consortium
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at
5 #     http://www.apache.org/licenses/LICENSE-2.0
6 # Unless required by applicable law or agreed to in writing, software
7 # distributed under the License is distributed on an "AS IS" BASIS,
8 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9 # See the License for the specific language governing permissions and
10 # limitations under the License.
11 
12 import logging
13 import warnings
14 from typing import TYPE_CHECKING, Dict, Optional
15 
16 from monai.utils import exact_version, optional_import
17 
18 Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
19 Checkpoint, _ = optional_import("ignite.handlers", "0.4.4", exact_version, "Checkpoint")
20 BaseSaveHandler, _ = optional_import("ignite.handlers.checkpoint", "0.4.4", exact_version, "BaseSaveHandler")
21 
22 if TYPE_CHECKING:
23     from ignite.engine import Engine
24     from ignite.handlers import DiskSaver
25 else:
26     Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
27     DiskSaver, _ = optional_import("ignite.handlers", "0.4.4", exact_version, "DiskSaver")
28 
29 
30 class CheckpointSaver:
31     """
32     CheckpointSaver acts as an Ignite handler to save checkpoint data into files.
33     It supports to save according to metrics result, epoch number, iteration number
34     and last model or exception.
35 
36     Args:
37         save_dir: the target directory to save the checkpoints.
38         save_dict: source objects that save to the checkpoint. examples::
39 
40             {'network': net, 'optimizer': optimizer, 'lr_scheduler': lr_scheduler}
41 
42         name: identifier of logging.logger to use, if None, defaulting to ``engine.logger``.
43         file_prefix: prefix for the filenames to which objects will be saved.
44         save_final: whether to save checkpoint or session at final iteration or exception.
45             If checkpoints are to be saved when an exception is raised, put this handler before
46             `StatsHandler` in the handler list, because the logic with Ignite can only trigger
47             the first attached handler for `EXCEPTION_RAISED` event.
48         final_filename: set a fixed filename to save the final model if `save_final=True`.
49             If None, default to `checkpoint_final_iteration=N.pt`.
50         save_key_metric: whether to save checkpoint or session when the value of key_metric is
51             higher than all the previous values during training.keep 4 decimal places of metric,
52             checkpoint name is: {file_prefix}_key_metric=0.XXXX.pth.
53         key_metric_name: the name of key_metric in ignite metrics dictionary.
54             If None, use `engine.state.key_metric` instead.
55         key_metric_n_saved: save top N checkpoints or sessions, sorted by the value of key
56             metric in descending order.
57         key_metric_filename: set a fixed filename to set the best metric model, if not None,
58             `key_metric_n_saved` should be 1 and only keep the best metric model.
59         key_metric_save_state: whether to save the tracking list of key metric in the checkpoint file.
60             if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent
61             with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.
62             typically, it's used to resume training and compare current metric with previous N values.
63         epoch_level: save checkpoint during training for every N epochs or every N iterations.
64             `True` is epoch level, `False` is iteration level.
65         save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.
66         n_saved: save latest N checkpoints of epoch level or iteration level, 'None' is to save all.
67 
68     Note:
69         CheckpointHandler can be used during training, validation or evaluation.
70         example of saved files:
71 
72             - checkpoint_iteration=400.pt
73             - checkpoint_iteration=800.pt
74             - checkpoint_epoch=1.pt
75             - checkpoint_final_iteration=1000.pt
76             - checkpoint_key_metric=0.9387.pt
77 
78     """
79 
80     def __init__(
81         self,
82         save_dir: str,
83         save_dict: Dict,
84         name: Optional[str] = None,
85         file_prefix: str = "",
86         save_final: bool = False,
87         final_filename: Optional[str] = None,
88         save_key_metric: bool = False,
89         key_metric_name: Optional[str] = None,
90         key_metric_n_saved: int = 1,
91         key_metric_filename: Optional[str] = None,
92         key_metric_save_state: bool = False,
93         epoch_level: bool = True,
94         save_interval: int = 0,
95         n_saved: Optional[int] = None,
96     ) -> None:
97         if save_dir is None:
98             raise AssertionError("must provide directory to save the checkpoints.")
99         self.save_dir = save_dir
100         if not (save_dict is not None and len(save_dict) > 0):
101             raise AssertionError("must provide source objects to save.")
102         self.save_dict = save_dict
103         self.logger = logging.getLogger(name)
104         self.epoch_level = epoch_level
105         self.save_interval = save_interval
106         self._final_checkpoint = self._key_metric_checkpoint = self._interval_checkpoint = None
107         self._name = name
108 
109         class _DiskSaver(DiskSaver):
110             """
111             Enhance the DiskSaver to support fixed filename.
112 
113             """
114 
115             def __init__(self, dirname: str, filename: Optional[str] = None):
116                 super().__init__(dirname=dirname, require_empty=False)
117                 self.filename = filename
118 
119             def __call__(self, checkpoint: Dict, filename: str, metadata: Optional[Dict] = None) -> None:
120                 if self.filename is not None:
121                     filename = self.filename
122                 super().__call__(checkpoint=checkpoint, filename=filename, metadata=metadata)
123 
124             def remove(self, filename: str) -> None:
125                 if self.filename is not None:
126                     filename = self.filename
127                 super().remove(filename=filename)
128 
129         if save_final:
130 
131             def _final_func(engine: Engine):
132                 return engine.state.iteration
133 
134             self._final_checkpoint = Checkpoint(
135                 to_save=self.save_dict,
136                 save_handler=_DiskSaver(dirname=self.save_dir, filename=final_filename),
137                 filename_prefix=file_prefix,
138                 score_function=_final_func,
139                 score_name="final_iteration",
140             )
141 
142         if save_key_metric:
143 
144             def _score_func(engine: Engine):
145                 if isinstance(key_metric_name, str):
146                     metric_name = key_metric_name
147                 elif hasattr(engine.state, "key_metric_name") and isinstance(engine.state.key_metric_name, str):
148                     metric_name = engine.state.key_metric_name
149                 else:
150                     raise ValueError(
151                         f"Incompatible values: save_key_metric=True and key_metric_name={key_metric_name}."
152                     )
153                 return round(engine.state.metrics[metric_name], 4)
154 
155             if key_metric_filename is not None and key_metric_n_saved > 1:
156                 raise ValueError("if using fixed filename to save the best metric model, we should only save 1 model.")
157 
158             self._key_metric_checkpoint = Checkpoint(
159                 to_save=self.save_dict,
160                 save_handler=_DiskSaver(dirname=self.save_dir, filename=key_metric_filename),
161                 filename_prefix=file_prefix,
162                 score_function=_score_func,
163                 score_name="key_metric",
164                 n_saved=key_metric_n_saved,
165                 include_self=key_metric_save_state,
166             )
167 
168         if save_interval > 0:
169 
170             def _interval_func(engine: Engine):
171                 return engine.state.epoch if self.epoch_level else engine.state.iteration
172 
173             self._interval_checkpoint = Checkpoint(
174                 to_save=self.save_dict,
175                 save_handler=_DiskSaver(dirname=self.save_dir),
176                 filename_prefix=file_prefix,
177                 score_function=_interval_func,
178                 score_name="epoch" if self.epoch_level else "iteration",
179                 n_saved=n_saved,
180             )
181 
182     def load_state_dict(self, state_dict: Dict) -> None:
183         """
184         Utility to resume the internal state of key metric tracking list if configured to save
185         checkpoints based on the key metric value.
186         Note to set `key_metric_save_state=True` when saving the previous checkpoint.
187 
188         Example::
189 
190             CheckpointSaver(
191                 ...
192                 save_key_metric=True,
193                 key_metric_save_state=True,  # config to also save the state of this saver
194             ).attach(engine)
195             engine.run(...)
196 
197             # resumed training with a new CheckpointSaver
198             saver = CheckpointSaver(save_key_metric=True, ...)
199             # load the previous key metric tracking list into saver
200             CheckpointLoader("/test/model.pt"), {"checkpointer": saver}).attach(engine)
201 
202         """
203         if self._key_metric_checkpoint is not None:
204             self._key_metric_checkpoint.load_state_dict(state_dict)
205         else:
206             warnings.warn("no key metric checkpoint saver to resume the key metric tracking list.")
207 
208     def attach(self, engine: Engine) -> None:
209         """
210         Args:
211             engine: Ignite Engine, it can be a trainer, validator or evaluator.
212         """
213         if self._name is None:
214             self.logger = engine.logger
215         if self._final_checkpoint is not None:
216             engine.add_event_handler(Events.COMPLETED, self.completed)
217             engine.add_event_handler(Events.EXCEPTION_RAISED, self.exception_raised)
218         if self._key_metric_checkpoint is not None:
219             engine.add_event_handler(Events.EPOCH_COMPLETED, self.metrics_completed)
220         if self._interval_checkpoint is not None:
221             if self.epoch_level:
222                 engine.add_event_handler(Events.EPOCH_COMPLETED(every=self.save_interval), self.interval_completed)
223             else:
224                 engine.add_event_handler(Events.ITERATION_COMPLETED(every=self.save_interval), self.interval_completed)
225 
226     def _delete_previous_final_ckpt(self):
227         saved = self._final_checkpoint._saved
228         if len(saved) > 0:
229             item = saved.pop(0)
230             self._final_checkpoint.save_handler.remove(item.filename)
231             self.logger.info(f"Deleted previous saved final checkpoint: {item.filename}")
232 
233     def completed(self, engine: Engine) -> None:
234         """Callback for train or validation/evaluation completed Event.
235         Save final checkpoint if configure save_final is True.
236 
237         Args:
238             engine: Ignite Engine, it can be a trainer, validator or evaluator.
239         """
240         if not callable(self._final_checkpoint):
241             raise AssertionError("Error: _final_checkpoint function not specified.")
242         # delete previous saved final checkpoint if existing
243         self._delete_previous_final_ckpt()
244         self._final_checkpoint(engine)
245         if self.logger is None:
246             raise AssertionError
247         if not hasattr(self.logger, "info"):
248             raise AssertionError("Error, provided logger has not info attribute.")
249         self.logger.info(f"Train completed, saved final checkpoint: {self._final_checkpoint.last_checkpoint}")
250 
251     def exception_raised(self, engine: Engine, e: Exception) -> None:
252         """Callback for train or validation/evaluation exception raised Event.
253         Save current data as final checkpoint if configure save_final is True. This callback may be skipped
254         because the logic with Ignite can only trigger the first attached handler for `EXCEPTION_RAISED` event.
255 
256         Args:
257             engine: Ignite Engine, it can be a trainer, validator or evaluator.
258             e: the exception caught in Ignite during engine.run().
259         """
260         if not callable(self._final_checkpoint):
261             raise AssertionError("Error: _final_checkpoint function not specified.")
262         # delete previous saved final checkpoint if existing
263         self._delete_previous_final_ckpt()
264         self._final_checkpoint(engine)
265         if self.logger is None:
266             raise AssertionError
267         if not hasattr(self.logger, "info"):
268             raise AssertionError("Error, provided logger has not info attribute.")
269         self.logger.info(f"Exception_raised, saved exception checkpoint: {self._final_checkpoint.last_checkpoint}")
270         raise e
271 
272     def metrics_completed(self, engine: Engine) -> None:
273         """Callback to compare metrics and save models in train or validation when epoch completed.
274 
275         Args:
276             engine: Ignite Engine, it can be a trainer, validator or evaluator.
277         """
278         if not callable(self._key_metric_checkpoint):
279             raise AssertionError("Error: _key_metric_checkpoint function not specified.")
280         self._key_metric_checkpoint(engine)
281 
282     def interval_completed(self, engine: Engine) -> None:
283         """Callback for train epoch/iteration completed Event.
284         Save checkpoint if configure save_interval = N
285 
286         Args:
287             engine: Ignite Engine, it can be a trainer, validator or evaluator.
288         """
289         if not callable(self._interval_checkpoint):
290             raise AssertionError("Error: _interval_checkpoint function not specified.")
291         self._interval_checkpoint(engine)
292         if self.logger is None:
293             raise AssertionError
294         if not hasattr(self.logger, "info"):
295             raise AssertionError("Error, provided logger has not info attribute.")
296         if self.epoch_level:
297             self.logger.info(f"Saved checkpoint at epoch: {engine.state.epoch}")
298         else:
299             self.logger.info(f"Saved checkpoint at iteration: {engine.state.iteration}")
300
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py
--- a/monai/handlers/checkpoint_saver.py
+++ b/monai/handlers/checkpoint_saver.py
@@ -60,6 +60,8 @@
             if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent
             with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.
             typically, it's used to resume training and compare current metric with previous N values.
+        key_metric_greater_or_equal: if `True`, the latest equally scored model is stored. Otherwise,
+            save the the first equally scored model. default to `False`.
         epoch_level: save checkpoint during training for every N epochs or every N iterations.
             `True` is epoch level, `False` is iteration level.
         save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.
@@ -90,6 +92,7 @@
         key_metric_n_saved: int = 1,
         key_metric_filename: Optional[str] = None,
         key_metric_save_state: bool = False,
+        key_metric_greater_or_equal: bool = False,
         epoch_level: bool = True,
         save_interval: int = 0,
         n_saved: Optional[int] = None,
@@ -163,6 +166,7 @@
                 score_name="key_metric",
                 n_saved=key_metric_n_saved,
                 include_self=key_metric_save_state,
+                greater_or_equal=key_metric_greater_or_equal,
             )
 
         if save_interval > 0:

{"golden_diff": "diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py\n--- a/monai/handlers/checkpoint_saver.py\n+++ b/monai/handlers/checkpoint_saver.py\n@@ -60,6 +60,8 @@\n             if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent\n             with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.\n             typically, it's used to resume training and compare current metric with previous N values.\n+        key_metric_greater_or_equal: if `True`, the latest equally scored model is stored. Otherwise,\n+            save the the first equally scored model. default to `False`.\n         epoch_level: save checkpoint during training for every N epochs or every N iterations.\n             `True` is epoch level, `False` is iteration level.\n         save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.\n@@ -90,6 +92,7 @@\n         key_metric_n_saved: int = 1,\n         key_metric_filename: Optional[str] = None,\n         key_metric_save_state: bool = False,\n+        key_metric_greater_or_equal: bool = False,\n         epoch_level: bool = True,\n         save_interval: int = 0,\n         n_saved: Optional[int] = None,\n@@ -163,6 +166,7 @@\n                 score_name=\"key_metric\",\n                 n_saved=key_metric_n_saved,\n                 include_self=key_metric_save_state,\n+                greater_or_equal=key_metric_greater_or_equal,\n             )\n \n         if save_interval > 0:\n", "issue": "support greater_or_equal from ignite 0.4.4\n**Is your feature request related to a problem? Please describe.**\r\nIn ignite 0.4.4, @vfdev-5 and I developed a new arg in CheckpointHandler `greater_or_equal`, which is a feature request from our interval research team.\r\nhttps://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L101\r\nNeed to add this support in MONAI as MONAI already updated the ignite dependency to 0.4.4.\r\n\n", "before_files": [{"content": "# Copyright 2020 - 2021 MONAI Consortium\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#     http://www.apache.org/licenses/LICENSE-2.0\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport warnings\nfrom typing import TYPE_CHECKING, Dict, Optional\n\nfrom monai.utils import exact_version, optional_import\n\nEvents, _ = optional_import(\"ignite.engine\", \"0.4.4\", exact_version, \"Events\")\nCheckpoint, _ = optional_import(\"ignite.handlers\", \"0.4.4\", exact_version, \"Checkpoint\")\nBaseSaveHandler, _ = optional_import(\"ignite.handlers.checkpoint\", \"0.4.4\", exact_version, \"BaseSaveHandler\")\n\nif TYPE_CHECKING:\n    from ignite.engine import Engine\n    from ignite.handlers import DiskSaver\nelse:\n    Engine, _ = optional_import(\"ignite.engine\", \"0.4.4\", exact_version, \"Engine\")\n    DiskSaver, _ = optional_import(\"ignite.handlers\", \"0.4.4\", exact_version, \"DiskSaver\")\n\n\nclass CheckpointSaver:\n    \"\"\"\n    CheckpointSaver acts as an Ignite handler to save checkpoint data into files.\n    It supports to save according to metrics result, epoch number, iteration number\n    and last model or exception.\n\n    Args:\n        save_dir: the target directory to save the checkpoints.\n        save_dict: source objects that save to the checkpoint. examples::\n\n            {'network': net, 'optimizer': optimizer, 'lr_scheduler': lr_scheduler}\n\n        name: identifier of logging.logger to use, if None, defaulting to ``engine.logger``.\n        file_prefix: prefix for the filenames to which objects will be saved.\n        save_final: whether to save checkpoint or session at final iteration or exception.\n            If checkpoints are to be saved when an exception is raised, put this handler before\n            `StatsHandler` in the handler list, because the logic with Ignite can only trigger\n            the first attached handler for `EXCEPTION_RAISED` event.\n        final_filename: set a fixed filename to save the final model if `save_final=True`.\n            If None, default to `checkpoint_final_iteration=N.pt`.\n        save_key_metric: whether to save checkpoint or session when the value of key_metric is\n            higher than all the previous values during training.keep 4 decimal places of metric,\n            checkpoint name is: {file_prefix}_key_metric=0.XXXX.pth.\n        key_metric_name: the name of key_metric in ignite metrics dictionary.\n            If None, use `engine.state.key_metric` instead.\n        key_metric_n_saved: save top N checkpoints or sessions, sorted by the value of key\n            metric in descending order.\n        key_metric_filename: set a fixed filename to set the best metric model, if not None,\n            `key_metric_n_saved` should be 1 and only keep the best metric model.\n        key_metric_save_state: whether to save the tracking list of key metric in the checkpoint file.\n            if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent\n            with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.\n            typically, it's used to resume training and compare current metric with previous N values.\n        epoch_level: save checkpoint during training for every N epochs or every N iterations.\n            `True` is epoch level, `False` is iteration level.\n        save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.\n        n_saved: save latest N checkpoints of epoch level or iteration level, 'None' is to save all.\n\n    Note:\n        CheckpointHandler can be used during training, validation or evaluation.\n        example of saved files:\n\n            - checkpoint_iteration=400.pt\n            - checkpoint_iteration=800.pt\n            - checkpoint_epoch=1.pt\n            - checkpoint_final_iteration=1000.pt\n            - checkpoint_key_metric=0.9387.pt\n\n    \"\"\"\n\n    def __init__(\n        self,\n        save_dir: str,\n        save_dict: Dict,\n        name: Optional[str] = None,\n        file_prefix: str = \"\",\n        save_final: bool = False,\n        final_filename: Optional[str] = None,\n        save_key_metric: bool = False,\n        key_metric_name: Optional[str] = None,\n        key_metric_n_saved: int = 1,\n        key_metric_filename: Optional[str] = None,\n        key_metric_save_state: bool = False,\n        epoch_level: bool = True,\n        save_interval: int = 0,\n        n_saved: Optional[int] = None,\n    ) -> None:\n        if save_dir is None:\n            raise AssertionError(\"must provide directory to save the checkpoints.\")\n        self.save_dir = save_dir\n        if not (save_dict is not None and len(save_dict) > 0):\n            raise AssertionError(\"must provide source objects to save.\")\n        self.save_dict = save_dict\n        self.logger = logging.getLogger(name)\n        self.epoch_level = epoch_level\n        self.save_interval = save_interval\n        self._final_checkpoint = self._key_metric_checkpoint = self._interval_checkpoint = None\n        self._name = name\n\n        class _DiskSaver(DiskSaver):\n            \"\"\"\n            Enhance the DiskSaver to support fixed filename.\n\n            \"\"\"\n\n            def __init__(self, dirname: str, filename: Optional[str] = None):\n                super().__init__(dirname=dirname, require_empty=False)\n                self.filename = filename\n\n            def __call__(self, checkpoint: Dict, filename: str, metadata: Optional[Dict] = None) -> None:\n                if self.filename is not None:\n                    filename = self.filename\n                super().__call__(checkpoint=checkpoint, filename=filename, metadata=metadata)\n\n            def remove(self, filename: str) -> None:\n                if self.filename is not None:\n                    filename = self.filename\n                super().remove(filename=filename)\n\n        if save_final:\n\n            def _final_func(engine: Engine):\n                return engine.state.iteration\n\n            self._final_checkpoint = Checkpoint(\n                to_save=self.save_dict,\n                save_handler=_DiskSaver(dirname=self.save_dir, filename=final_filename),\n                filename_prefix=file_prefix,\n                score_function=_final_func,\n                score_name=\"final_iteration\",\n            )\n\n        if save_key_metric:\n\n            def _score_func(engine: Engine):\n                if isinstance(key_metric_name, str):\n                    metric_name = key_metric_name\n                elif hasattr(engine.state, \"key_metric_name\") and isinstance(engine.state.key_metric_name, str):\n                    metric_name = engine.state.key_metric_name\n                else:\n                    raise ValueError(\n                        f\"Incompatible values: save_key_metric=True and key_metric_name={key_metric_name}.\"\n                    )\n                return round(engine.state.metrics[metric_name], 4)\n\n            if key_metric_filename is not None and key_metric_n_saved > 1:\n                raise ValueError(\"if using fixed filename to save the best metric model, we should only save 1 model.\")\n\n            self._key_metric_checkpoint = Checkpoint(\n                to_save=self.save_dict,\n                save_handler=_DiskSaver(dirname=self.save_dir, filename=key_metric_filename),\n                filename_prefix=file_prefix,\n                score_function=_score_func,\n                score_name=\"key_metric\",\n                n_saved=key_metric_n_saved,\n                include_self=key_metric_save_state,\n            )\n\n        if save_interval > 0:\n\n            def _interval_func(engine: Engine):\n                return engine.state.epoch if self.epoch_level else engine.state.iteration\n\n            self._interval_checkpoint = Checkpoint(\n                to_save=self.save_dict,\n                save_handler=_DiskSaver(dirname=self.save_dir),\n                filename_prefix=file_prefix,\n                score_function=_interval_func,\n                score_name=\"epoch\" if self.epoch_level else \"iteration\",\n                n_saved=n_saved,\n            )\n\n    def load_state_dict(self, state_dict: Dict) -> None:\n        \"\"\"\n        Utility to resume the internal state of key metric tracking list if configured to save\n        checkpoints based on the key metric value.\n        Note to set `key_metric_save_state=True` when saving the previous checkpoint.\n\n        Example::\n\n            CheckpointSaver(\n                ...\n                save_key_metric=True,\n                key_metric_save_state=True,  # config to also save the state of this saver\n            ).attach(engine)\n            engine.run(...)\n\n            # resumed training with a new CheckpointSaver\n            saver = CheckpointSaver(save_key_metric=True, ...)\n            # load the previous key metric tracking list into saver\n            CheckpointLoader(\"/test/model.pt\"), {\"checkpointer\": saver}).attach(engine)\n\n        \"\"\"\n        if self._key_metric_checkpoint is not None:\n            self._key_metric_checkpoint.load_state_dict(state_dict)\n        else:\n            warnings.warn(\"no key metric checkpoint saver to resume the key metric tracking list.\")\n\n    def attach(self, engine: Engine) -> None:\n        \"\"\"\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if self._name is None:\n            self.logger = engine.logger\n        if self._final_checkpoint is not None:\n            engine.add_event_handler(Events.COMPLETED, self.completed)\n            engine.add_event_handler(Events.EXCEPTION_RAISED, self.exception_raised)\n        if self._key_metric_checkpoint is not None:\n            engine.add_event_handler(Events.EPOCH_COMPLETED, self.metrics_completed)\n        if self._interval_checkpoint is not None:\n            if self.epoch_level:\n                engine.add_event_handler(Events.EPOCH_COMPLETED(every=self.save_interval), self.interval_completed)\n            else:\n                engine.add_event_handler(Events.ITERATION_COMPLETED(every=self.save_interval), self.interval_completed)\n\n    def _delete_previous_final_ckpt(self):\n        saved = self._final_checkpoint._saved\n        if len(saved) > 0:\n            item = saved.pop(0)\n            self._final_checkpoint.save_handler.remove(item.filename)\n            self.logger.info(f\"Deleted previous saved final checkpoint: {item.filename}\")\n\n    def completed(self, engine: Engine) -> None:\n        \"\"\"Callback for train or validation/evaluation completed Event.\n        Save final checkpoint if configure save_final is True.\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if not callable(self._final_checkpoint):\n            raise AssertionError(\"Error: _final_checkpoint function not specified.\")\n        # delete previous saved final checkpoint if existing\n        self._delete_previous_final_ckpt()\n        self._final_checkpoint(engine)\n        if self.logger is None:\n            raise AssertionError\n        if not hasattr(self.logger, \"info\"):\n            raise AssertionError(\"Error, provided logger has not info attribute.\")\n        self.logger.info(f\"Train completed, saved final checkpoint: {self._final_checkpoint.last_checkpoint}\")\n\n    def exception_raised(self, engine: Engine, e: Exception) -> None:\n        \"\"\"Callback for train or validation/evaluation exception raised Event.\n        Save current data as final checkpoint if configure save_final is True. This callback may be skipped\n        because the logic with Ignite can only trigger the first attached handler for `EXCEPTION_RAISED` event.\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n            e: the exception caught in Ignite during engine.run().\n        \"\"\"\n        if not callable(self._final_checkpoint):\n            raise AssertionError(\"Error: _final_checkpoint function not specified.\")\n        # delete previous saved final checkpoint if existing\n        self._delete_previous_final_ckpt()\n        self._final_checkpoint(engine)\n        if self.logger is None:\n            raise AssertionError\n        if not hasattr(self.logger, \"info\"):\n            raise AssertionError(\"Error, provided logger has not info attribute.\")\n        self.logger.info(f\"Exception_raised, saved exception checkpoint: {self._final_checkpoint.last_checkpoint}\")\n        raise e\n\n    def metrics_completed(self, engine: Engine) -> None:\n        \"\"\"Callback to compare metrics and save models in train or validation when epoch completed.\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if not callable(self._key_metric_checkpoint):\n            raise AssertionError(\"Error: _key_metric_checkpoint function not specified.\")\n        self._key_metric_checkpoint(engine)\n\n    def interval_completed(self, engine: Engine) -> None:\n        \"\"\"Callback for train epoch/iteration completed Event.\n        Save checkpoint if configure save_interval = N\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if not callable(self._interval_checkpoint):\n            raise AssertionError(\"Error: _interval_checkpoint function not specified.\")\n        self._interval_checkpoint(engine)\n        if self.logger is None:\n            raise AssertionError\n        if not hasattr(self.logger, \"info\"):\n            raise AssertionError(\"Error, provided logger has not info attribute.\")\n        if self.epoch_level:\n            self.logger.info(f\"Saved checkpoint at epoch: {engine.state.epoch}\")\n        else:\n            self.logger.info(f\"Saved checkpoint at iteration: {engine.state.iteration}\")\n", "path": "monai/handlers/checkpoint_saver.py"}], "after_files": [{"content": "# Copyright 2020 - 2021 MONAI Consortium\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#     http://www.apache.org/licenses/LICENSE-2.0\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport warnings\nfrom typing import TYPE_CHECKING, Dict, Optional\n\nfrom monai.utils import exact_version, optional_import\n\nEvents, _ = optional_import(\"ignite.engine\", \"0.4.4\", exact_version, \"Events\")\nCheckpoint, _ = optional_import(\"ignite.handlers\", \"0.4.4\", exact_version, \"Checkpoint\")\nBaseSaveHandler, _ = optional_import(\"ignite.handlers.checkpoint\", \"0.4.4\", exact_version, \"BaseSaveHandler\")\n\nif TYPE_CHECKING:\n    from ignite.engine import Engine\n    from ignite.handlers import DiskSaver\nelse:\n    Engine, _ = optional_import(\"ignite.engine\", \"0.4.4\", exact_version, \"Engine\")\n    DiskSaver, _ = optional_import(\"ignite.handlers\", \"0.4.4\", exact_version, \"DiskSaver\")\n\n\nclass CheckpointSaver:\n    \"\"\"\n    CheckpointSaver acts as an Ignite handler to save checkpoint data into files.\n    It supports to save according to metrics result, epoch number, iteration number\n    and last model or exception.\n\n    Args:\n        save_dir: the target directory to save the checkpoints.\n        save_dict: source objects that save to the checkpoint. examples::\n\n            {'network': net, 'optimizer': optimizer, 'lr_scheduler': lr_scheduler}\n\n        name: identifier of logging.logger to use, if None, defaulting to ``engine.logger``.\n        file_prefix: prefix for the filenames to which objects will be saved.\n        save_final: whether to save checkpoint or session at final iteration or exception.\n            If checkpoints are to be saved when an exception is raised, put this handler before\n            `StatsHandler` in the handler list, because the logic with Ignite can only trigger\n            the first attached handler for `EXCEPTION_RAISED` event.\n        final_filename: set a fixed filename to save the final model if `save_final=True`.\n            If None, default to `checkpoint_final_iteration=N.pt`.\n        save_key_metric: whether to save checkpoint or session when the value of key_metric is\n            higher than all the previous values during training.keep 4 decimal places of metric,\n            checkpoint name is: {file_prefix}_key_metric=0.XXXX.pth.\n        key_metric_name: the name of key_metric in ignite metrics dictionary.\n            If None, use `engine.state.key_metric` instead.\n        key_metric_n_saved: save top N checkpoints or sessions, sorted by the value of key\n            metric in descending order.\n        key_metric_filename: set a fixed filename to set the best metric model, if not None,\n            `key_metric_n_saved` should be 1 and only keep the best metric model.\n        key_metric_save_state: whether to save the tracking list of key metric in the checkpoint file.\n            if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent\n            with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.\n            typically, it's used to resume training and compare current metric with previous N values.\n        key_metric_greater_or_equal: if `True`, the latest equally scored model is stored. Otherwise,\n            save the the first equally scored model. default to `False`.\n        epoch_level: save checkpoint during training for every N epochs or every N iterations.\n            `True` is epoch level, `False` is iteration level.\n        save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.\n        n_saved: save latest N checkpoints of epoch level or iteration level, 'None' is to save all.\n\n    Note:\n        CheckpointHandler can be used during training, validation or evaluation.\n        example of saved files:\n\n            - checkpoint_iteration=400.pt\n            - checkpoint_iteration=800.pt\n            - checkpoint_epoch=1.pt\n            - checkpoint_final_iteration=1000.pt\n            - checkpoint_key_metric=0.9387.pt\n\n    \"\"\"\n\n    def __init__(\n        self,\n        save_dir: str,\n        save_dict: Dict,\n        name: Optional[str] = None,\n        file_prefix: str = \"\",\n        save_final: bool = False,\n        final_filename: Optional[str] = None,\n        save_key_metric: bool = False,\n        key_metric_name: Optional[str] = None,\n        key_metric_n_saved: int = 1,\n        key_metric_filename: Optional[str] = None,\n        key_metric_save_state: bool = False,\n        key_metric_greater_or_equal: bool = False,\n        epoch_level: bool = True,\n        save_interval: int = 0,\n        n_saved: Optional[int] = None,\n    ) -> None:\n        if save_dir is None:\n            raise AssertionError(\"must provide directory to save the checkpoints.\")\n        self.save_dir = save_dir\n        if not (save_dict is not None and len(save_dict) > 0):\n            raise AssertionError(\"must provide source objects to save.\")\n        self.save_dict = save_dict\n        self.logger = logging.getLogger(name)\n        self.epoch_level = epoch_level\n        self.save_interval = save_interval\n        self._final_checkpoint = self._key_metric_checkpoint = self._interval_checkpoint = None\n        self._name = name\n\n        class _DiskSaver(DiskSaver):\n            \"\"\"\n            Enhance the DiskSaver to support fixed filename.\n\n            \"\"\"\n\n            def __init__(self, dirname: str, filename: Optional[str] = None):\n                super().__init__(dirname=dirname, require_empty=False)\n                self.filename = filename\n\n            def __call__(self, checkpoint: Dict, filename: str, metadata: Optional[Dict] = None) -> None:\n                if self.filename is not None:\n                    filename = self.filename\n                super().__call__(checkpoint=checkpoint, filename=filename, metadata=metadata)\n\n            def remove(self, filename: str) -> None:\n                if self.filename is not None:\n                    filename = self.filename\n                super().remove(filename=filename)\n\n        if save_final:\n\n            def _final_func(engine: Engine):\n                return engine.state.iteration\n\n            self._final_checkpoint = Checkpoint(\n                to_save=self.save_dict,\n                save_handler=_DiskSaver(dirname=self.save_dir, filename=final_filename),\n                filename_prefix=file_prefix,\n                score_function=_final_func,\n                score_name=\"final_iteration\",\n            )\n\n        if save_key_metric:\n\n            def _score_func(engine: Engine):\n                if isinstance(key_metric_name, str):\n                    metric_name = key_metric_name\n                elif hasattr(engine.state, \"key_metric_name\") and isinstance(engine.state.key_metric_name, str):\n                    metric_name = engine.state.key_metric_name\n                else:\n                    raise ValueError(\n                        f\"Incompatible values: save_key_metric=True and key_metric_name={key_metric_name}.\"\n                    )\n                return round(engine.state.metrics[metric_name], 4)\n\n            if key_metric_filename is not None and key_metric_n_saved > 1:\n                raise ValueError(\"if using fixed filename to save the best metric model, we should only save 1 model.\")\n\n            self._key_metric_checkpoint = Checkpoint(\n                to_save=self.save_dict,\n                save_handler=_DiskSaver(dirname=self.save_dir, filename=key_metric_filename),\n                filename_prefix=file_prefix,\n                score_function=_score_func,\n                score_name=\"key_metric\",\n                n_saved=key_metric_n_saved,\n                include_self=key_metric_save_state,\n                greater_or_equal=key_metric_greater_or_equal,\n            )\n\n        if save_interval > 0:\n\n            def _interval_func(engine: Engine):\n                return engine.state.epoch if self.epoch_level else engine.state.iteration\n\n            self._interval_checkpoint = Checkpoint(\n                to_save=self.save_dict,\n                save_handler=_DiskSaver(dirname=self.save_dir),\n                filename_prefix=file_prefix,\n                score_function=_interval_func,\n                score_name=\"epoch\" if self.epoch_level else \"iteration\",\n                n_saved=n_saved,\n            )\n\n    def load_state_dict(self, state_dict: Dict) -> None:\n        \"\"\"\n        Utility to resume the internal state of key metric tracking list if configured to save\n        checkpoints based on the key metric value.\n        Note to set `key_metric_save_state=True` when saving the previous checkpoint.\n\n        Example::\n\n            CheckpointSaver(\n                ...\n                save_key_metric=True,\n                key_metric_save_state=True,  # config to also save the state of this saver\n            ).attach(engine)\n            engine.run(...)\n\n            # resumed training with a new CheckpointSaver\n            saver = CheckpointSaver(save_key_metric=True, ...)\n            # load the previous key metric tracking list into saver\n            CheckpointLoader(\"/test/model.pt\"), {\"checkpointer\": saver}).attach(engine)\n\n        \"\"\"\n        if self._key_metric_checkpoint is not None:\n            self._key_metric_checkpoint.load_state_dict(state_dict)\n        else:\n            warnings.warn(\"no key metric checkpoint saver to resume the key metric tracking list.\")\n\n    def attach(self, engine: Engine) -> None:\n        \"\"\"\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if self._name is None:\n            self.logger = engine.logger\n        if self._final_checkpoint is not None:\n            engine.add_event_handler(Events.COMPLETED, self.completed)\n            engine.add_event_handler(Events.EXCEPTION_RAISED, self.exception_raised)\n        if self._key_metric_checkpoint is not None:\n            engine.add_event_handler(Events.EPOCH_COMPLETED, self.metrics_completed)\n        if self._interval_checkpoint is not None:\n            if self.epoch_level:\n                engine.add_event_handler(Events.EPOCH_COMPLETED(every=self.save_interval), self.interval_completed)\n            else:\n                engine.add_event_handler(Events.ITERATION_COMPLETED(every=self.save_interval), self.interval_completed)\n\n    def _delete_previous_final_ckpt(self):\n        saved = self._final_checkpoint._saved\n        if len(saved) > 0:\n            item = saved.pop(0)\n            self._final_checkpoint.save_handler.remove(item.filename)\n            self.logger.info(f\"Deleted previous saved final checkpoint: {item.filename}\")\n\n    def completed(self, engine: Engine) -> None:\n        \"\"\"Callback for train or validation/evaluation completed Event.\n        Save final checkpoint if configure save_final is True.\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if not callable(self._final_checkpoint):\n            raise AssertionError(\"Error: _final_checkpoint function not specified.\")\n        # delete previous saved final checkpoint if existing\n        self._delete_previous_final_ckpt()\n        self._final_checkpoint(engine)\n        if self.logger is None:\n            raise AssertionError\n        if not hasattr(self.logger, \"info\"):\n            raise AssertionError(\"Error, provided logger has not info attribute.\")\n        self.logger.info(f\"Train completed, saved final checkpoint: {self._final_checkpoint.last_checkpoint}\")\n\n    def exception_raised(self, engine: Engine, e: Exception) -> None:\n        \"\"\"Callback for train or validation/evaluation exception raised Event.\n        Save current data as final checkpoint if configure save_final is True. This callback may be skipped\n        because the logic with Ignite can only trigger the first attached handler for `EXCEPTION_RAISED` event.\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n            e: the exception caught in Ignite during engine.run().\n        \"\"\"\n        if not callable(self._final_checkpoint):\n            raise AssertionError(\"Error: _final_checkpoint function not specified.\")\n        # delete previous saved final checkpoint if existing\n        self._delete_previous_final_ckpt()\n        self._final_checkpoint(engine)\n        if self.logger is None:\n            raise AssertionError\n        if not hasattr(self.logger, \"info\"):\n            raise AssertionError(\"Error, provided logger has not info attribute.\")\n        self.logger.info(f\"Exception_raised, saved exception checkpoint: {self._final_checkpoint.last_checkpoint}\")\n        raise e\n\n    def metrics_completed(self, engine: Engine) -> None:\n        \"\"\"Callback to compare metrics and save models in train or validation when epoch completed.\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if not callable(self._key_metric_checkpoint):\n            raise AssertionError(\"Error: _key_metric_checkpoint function not specified.\")\n        self._key_metric_checkpoint(engine)\n\n    def interval_completed(self, engine: Engine) -> None:\n        \"\"\"Callback for train epoch/iteration completed Event.\n        Save checkpoint if configure save_interval = N\n\n        Args:\n            engine: Ignite Engine, it can be a trainer, validator or evaluator.\n        \"\"\"\n        if not callable(self._interval_checkpoint):\n            raise AssertionError(\"Error: _interval_checkpoint function not specified.\")\n        self._interval_checkpoint(engine)\n        if self.logger is None:\n            raise AssertionError\n        if not hasattr(self.logger, \"info\"):\n            raise AssertionError(\"Error, provided logger has not info attribute.\")\n        if self.epoch_level:\n            self.logger.info(f\"Saved checkpoint at epoch: {engine.state.epoch}\")\n        else:\n            self.logger.info(f\"Saved checkpoint at iteration: {engine.state.iteration}\")\n", "path": "monai/handlers/checkpoint_saver.py"}]}