We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Propose to save new model before deleting previous ones in ModelCheckpointing
## 🚀 Feature
<!-- A clear and concise description of the feature proposal -->
In an edge case, the trainer deleted previous model and then was killed because of system error before successfully saving new model. Thus all the models were lost.
I understand specifying save_top_k > 1 helps, and saving before deleting leads to larger disk consumption. But it might be good to provide an option for this?
### Motivation

<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->

### Pitch

<!-- A clear and concise description of what you want to happen. -->
in the worst case, you have two but never none...

### Alternatives

<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->

### Additional context

<!-- Add any other context or screenshots about the feature request here. -->


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/callbacks/model_checkpoint.py`
Content:
```
1 """
2 Model Checkpointing
3 ===================
4 
5 Automatically save model checkpoints during training.
6 
7 """
8 
9 import os
10 import re
11 
12 import numpy as np
13 
14 from pytorch_lightning import _logger as log
15 from pytorch_lightning.callbacks.base import Callback
16 from pytorch_lightning.utilities import rank_zero_warn
17 
18 
19 class ModelCheckpoint(Callback):
20     r"""
21     Save the model after every epoch.
22 
23     Args:
24         filepath: path to save the model file.
25             Can contain named formatting options to be auto-filled.
26 
27             Example::
28 
29                 # custom path
30                 # saves a file like: my/path/epoch_0.ckpt
31                 >>> checkpoint_callback = ModelCheckpoint('my/path/')
32 
33                 # save any arbitrary metrics like `val_loss`, etc. in name
34                 # saves a file like: my/path/epoch=2-val_loss=0.2_other_metric=0.3.ckpt
35                 >>> checkpoint_callback = ModelCheckpoint(
36                 ...     filepath='my/path/{epoch}-{val_loss:.2f}-{other_metric:.2f}'
37                 ... )
38 
39         monitor: quantity to monitor.
40         verbose: verbosity mode. Default: ``False``.
41         save_top_k: if `save_top_k == k`,
42             the best k models according to
43             the quantity monitored will be saved.
44             if ``save_top_k == 0``, no models are saved.
45             if ``save_top_k == -1``, all models are saved.
46             Please note that the monitors are checked every `period` epochs.
47             if ``save_top_k >= 2`` and the callback is called multiple
48             times inside an epoch, the name of the saved file will be
49             appended with a version count starting with `v0`.
50         mode: one of {auto, min, max}.
51             If ``save_top_k != 0``, the decision
52             to overwrite the current save file is made
53             based on either the maximization or the
54             minimization of the monitored quantity. For `val_acc`,
55             this should be `max`, for `val_loss` this should
56             be `min`, etc. In `auto` mode, the direction is
57             automatically inferred from the name of the monitored quantity.
58         save_weights_only: if ``True``, then only the model's weights will be
59             saved (``model.save_weights(filepath)``), else the full model
60             is saved (``model.save(filepath)``).
61         period: Interval (number of epochs) between checkpoints.
62 
63     Example::
64 
65         >>> from pytorch_lightning import Trainer
66         >>> from pytorch_lightning.callbacks import ModelCheckpoint
67 
68         # saves checkpoints to 'my/path/' whenever 'val_loss' has a new min
69         >>> checkpoint_callback = ModelCheckpoint(filepath='my/path/')
70         >>> trainer = Trainer(checkpoint_callback=checkpoint_callback)
71 
72         # save epoch and val_loss in name
73         # saves a file like: my/path/sample-mnist_epoch=02_val_loss=0.32.ckpt
74         >>> checkpoint_callback = ModelCheckpoint(
75         ...     filepath='my/path/sample-mnist_{epoch:02d}-{val_loss:.2f}'
76         ... )
77 
78     """
79 
80     def __init__(self, filepath: str, monitor: str = 'val_loss', verbose: bool = False,
81                  save_top_k: int = 1, save_weights_only: bool = False,
82                  mode: str = 'auto', period: int = 1, prefix: str = ''):
83         super().__init__()
84         if save_top_k > 0 and os.path.isdir(filepath) and len(os.listdir(filepath)) > 0:
85             rank_zero_warn(
86                 f"Checkpoint directory {filepath} exists and is not empty with save_top_k != 0."
87                 "All files in this directory will be deleted when a checkpoint is saved!"
88             )
89 
90         self.monitor = monitor
91         self.verbose = verbose
92         if os.path.isdir(filepath):
93             self.dirpath, self.filename = filepath, '{epoch}'
94         else:
95             self.dirpath, self.filename = os.path.split(filepath)
96 
97         os.makedirs(self.dirpath, exist_ok=True)
98         self.save_top_k = save_top_k
99         self.save_weights_only = save_weights_only
100         self.period = period
101         self.epoch_last_check = None
102         self.prefix = prefix
103         self.best_k_models = {}
104         # {filename: monitor}
105         self.kth_best_model = ''
106         self.best = 0
107         self.save_function = None
108 
109         mode_dict = {
110             'min': (np.less, np.Inf, 'min'),
111             'max': (np.greater, -np.Inf, 'max'),
112             'auto': (np.greater, -np.Inf, 'max') if 'acc' in self.monitor or self.monitor.startswith('fmeasure')
113             else (np.less, np.Inf, 'min'),
114         }
115 
116         if mode not in mode_dict:
117             rank_zero_warn(f'ModelCheckpoint mode {mode} is unknown, fallback to auto mode.', RuntimeWarning)
118             mode = 'auto'
119 
120         self.monitor_op, self.kth_value, self.mode = mode_dict[mode]
121 
122     def _del_model(self, filepath):
123         os.remove(filepath)
124 
125     def _save_model(self, filepath):
126         # make paths
127         os.makedirs(os.path.dirname(filepath), exist_ok=True)
128 
129         # delegate the saving to the model
130         if self.save_function is not None:
131             self.save_function(filepath)
132         else:
133             raise ValueError(".save_function() not set")
134 
135     def check_monitor_top_k(self, current):
136         less_than_k_models = len(self.best_k_models) < self.save_top_k
137         if less_than_k_models:
138             return True
139         return self.monitor_op(current, self.best_k_models[self.kth_best_model])
140 
141     def format_checkpoint_name(self, epoch, metrics, ver=None):
142         """Generate a filename according to the defined template.
143 
144         Example::
145 
146             >>> tmpdir = os.path.dirname(__file__)
147             >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch}'))
148             >>> os.path.basename(ckpt.format_checkpoint_name(0, {}))
149             'epoch=0.ckpt'
150             >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch:03d}'))
151             >>> os.path.basename(ckpt.format_checkpoint_name(5, {}))
152             'epoch=005.ckpt'
153             >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch}-{val_loss:.2f}'))
154             >>> os.path.basename(ckpt.format_checkpoint_name(2, dict(val_loss=0.123456)))
155             'epoch=2-val_loss=0.12.ckpt'
156             >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{missing:d}'))
157             >>> os.path.basename(ckpt.format_checkpoint_name(0, {}))
158             'missing=0.ckpt'
159         """
160         # check if user passed in keys to the string
161         groups = re.findall(r'(\{.*?)[:\}]', self.filename)
162 
163         if len(groups) == 0:
164             # default name
165             filename = f'{self.prefix}_ckpt_epoch_{epoch}'
166         else:
167             metrics['epoch'] = epoch
168             filename = self.filename
169             for tmp in groups:
170                 name = tmp[1:]
171                 filename = filename.replace(tmp, name + '={' + name)
172                 if name not in metrics:
173                     metrics[name] = 0
174             filename = filename.format(**metrics)
175         str_ver = f'_v{ver}' if ver is not None else ''
176         filepath = os.path.join(self.dirpath, self.prefix + filename + str_ver + '.ckpt')
177         return filepath
178 
179     def on_validation_end(self, trainer, pl_module):
180         # only run on main process
181         if trainer.proc_rank != 0:
182             return
183 
184         metrics = trainer.callback_metrics
185         epoch = trainer.current_epoch
186         if self.save_top_k == 0:
187             # no models are saved
188             return
189         if self.epoch_last_check is not None and (epoch - self.epoch_last_check) < self.period:
190             # skipping in this term
191             return
192 
193         self.epoch_last_check = epoch
194 
195         filepath = self.format_checkpoint_name(epoch, metrics)
196         version_cnt = 0
197         while os.path.isfile(filepath):
198             filepath = self.format_checkpoint_name(epoch, metrics, ver=version_cnt)
199             # this epoch called before
200             version_cnt += 1
201 
202         if self.save_top_k != -1:
203             current = metrics.get(self.monitor)
204 
205             if current is None:
206                 rank_zero_warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)
207             elif self.check_monitor_top_k(current):
208                 self._do_check_save(filepath, current, epoch)
209             elif self.verbose > 0:
210                 log.info(f'\nEpoch {epoch:05d}: {self.monitor}  was not in top {self.save_top_k}')
211 
212         else:
213             if self.verbose > 0:
214                 log.info(f'\nEpoch {epoch:05d}: saving model to {filepath}')
215             self._save_model(filepath)
216 
217     def _do_check_save(self, filepath, current, epoch):
218         # remove kth
219         if len(self.best_k_models) == self.save_top_k and self.save_top_k > 0:
220             delpath = self.kth_best_model
221             self.best_k_models.pop(self.kth_best_model)
222             self._del_model(delpath)
223 
224         self.best_k_models[filepath] = current
225         if len(self.best_k_models) == self.save_top_k:
226             # monitor dict has reached k elements
227             _op = max if self.mode == 'min' else min
228             self.kth_best_model = _op(self.best_k_models,
229                                       key=self.best_k_models.get)
230             self.kth_value = self.best_k_models[self.kth_best_model]
231 
232         _op = min if self.mode == 'min' else max
233         self.best = _op(self.best_k_models.values())
234 
235         if self.verbose > 0:
236             log.info(
237                 f'\nEpoch {epoch:05d}: {self.monitor} reached'
238                 f' {current:0.5f} (best {self.best:0.5f}), saving model to'
239                 f' {filepath} as top {self.save_top_k}')
240         self._save_model(filepath)
241
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py
--- a/pytorch_lightning/callbacks/model_checkpoint.py
+++ b/pytorch_lightning/callbacks/model_checkpoint.py
@@ -216,10 +216,12 @@
 
     def _do_check_save(self, filepath, current, epoch):
         # remove kth
+
+        del_list = []
         if len(self.best_k_models) == self.save_top_k and self.save_top_k > 0:
             delpath = self.kth_best_model
             self.best_k_models.pop(self.kth_best_model)
-            self._del_model(delpath)
+            del_list.append(delpath)
 
         self.best_k_models[filepath] = current
         if len(self.best_k_models) == self.save_top_k:
@@ -238,3 +240,7 @@
                 f' {current:0.5f} (best {self.best:0.5f}), saving model to'
                 f' {filepath} as top {self.save_top_k}')
         self._save_model(filepath)
+
+        for cur_path in del_list:
+            if cur_path != filepath:
+                self._del_model(cur_path)

{"golden_diff": "diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py\n--- a/pytorch_lightning/callbacks/model_checkpoint.py\n+++ b/pytorch_lightning/callbacks/model_checkpoint.py\n@@ -216,10 +216,12 @@\n \n     def _do_check_save(self, filepath, current, epoch):\n         # remove kth\n+\n+        del_list = []\n         if len(self.best_k_models) == self.save_top_k and self.save_top_k > 0:\n             delpath = self.kth_best_model\n             self.best_k_models.pop(self.kth_best_model)\n-            self._del_model(delpath)\n+            del_list.append(delpath)\n \n         self.best_k_models[filepath] = current\n         if len(self.best_k_models) == self.save_top_k:\n@@ -238,3 +240,7 @@\n                 f' {current:0.5f} (best {self.best:0.5f}), saving model to'\n                 f' {filepath} as top {self.save_top_k}')\n         self._save_model(filepath)\n+\n+        for cur_path in del_list:\n+            if cur_path != filepath:\n+                self._del_model(cur_path)\n", "issue": "Propose to save new model before deleting previous ones in ModelCheckpointing\n## \ud83d\ude80 Feature\r\n<!-- A clear and concise description of the feature proposal -->\r\nIn an edge case, the trainer deleted previous model and then was killed because of system error before successfully saving new model. Thus all the models were lost.\r\nI understand specifying save_top_k > 1 helps, and saving before deleting leads to larger disk consumption. But it might be good to provide an option for this?\r\n### Motivation\r\n\r\n<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->\r\n\r\n### Pitch\r\n\r\n<!-- A clear and concise description of what you want to happen. -->\r\nin the worst case, you have two but never none...\r\n\r\n### Alternatives\r\n\r\n<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->\r\n\r\n### Additional context\r\n\r\n<!-- Add any other context or screenshots about the feature request here. -->\r\n\n", "before_files": [{"content": "\"\"\"\nModel Checkpointing\n===================\n\nAutomatically save model checkpoints during training.\n\n\"\"\"\n\nimport os\nimport re\n\nimport numpy as np\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.callbacks.base import Callback\nfrom pytorch_lightning.utilities import rank_zero_warn\n\n\nclass ModelCheckpoint(Callback):\n    r\"\"\"\n    Save the model after every epoch.\n\n    Args:\n        filepath: path to save the model file.\n            Can contain named formatting options to be auto-filled.\n\n            Example::\n\n                # custom path\n                # saves a file like: my/path/epoch_0.ckpt\n                >>> checkpoint_callback = ModelCheckpoint('my/path/')\n\n                # save any arbitrary metrics like `val_loss`, etc. in name\n                # saves a file like: my/path/epoch=2-val_loss=0.2_other_metric=0.3.ckpt\n                >>> checkpoint_callback = ModelCheckpoint(\n                ...     filepath='my/path/{epoch}-{val_loss:.2f}-{other_metric:.2f}'\n                ... )\n\n        monitor: quantity to monitor.\n        verbose: verbosity mode. Default: ``False``.\n        save_top_k: if `save_top_k == k`,\n            the best k models according to\n            the quantity monitored will be saved.\n            if ``save_top_k == 0``, no models are saved.\n            if ``save_top_k == -1``, all models are saved.\n            Please note that the monitors are checked every `period` epochs.\n            if ``save_top_k >= 2`` and the callback is called multiple\n            times inside an epoch, the name of the saved file will be\n            appended with a version count starting with `v0`.\n        mode: one of {auto, min, max}.\n            If ``save_top_k != 0``, the decision\n            to overwrite the current save file is made\n            based on either the maximization or the\n            minimization of the monitored quantity. For `val_acc`,\n            this should be `max`, for `val_loss` this should\n            be `min`, etc. In `auto` mode, the direction is\n            automatically inferred from the name of the monitored quantity.\n        save_weights_only: if ``True``, then only the model's weights will be\n            saved (``model.save_weights(filepath)``), else the full model\n            is saved (``model.save(filepath)``).\n        period: Interval (number of epochs) between checkpoints.\n\n    Example::\n\n        >>> from pytorch_lightning import Trainer\n        >>> from pytorch_lightning.callbacks import ModelCheckpoint\n\n        # saves checkpoints to 'my/path/' whenever 'val_loss' has a new min\n        >>> checkpoint_callback = ModelCheckpoint(filepath='my/path/')\n        >>> trainer = Trainer(checkpoint_callback=checkpoint_callback)\n\n        # save epoch and val_loss in name\n        # saves a file like: my/path/sample-mnist_epoch=02_val_loss=0.32.ckpt\n        >>> checkpoint_callback = ModelCheckpoint(\n        ...     filepath='my/path/sample-mnist_{epoch:02d}-{val_loss:.2f}'\n        ... )\n\n    \"\"\"\n\n    def __init__(self, filepath: str, monitor: str = 'val_loss', verbose: bool = False,\n                 save_top_k: int = 1, save_weights_only: bool = False,\n                 mode: str = 'auto', period: int = 1, prefix: str = ''):\n        super().__init__()\n        if save_top_k > 0 and os.path.isdir(filepath) and len(os.listdir(filepath)) > 0:\n            rank_zero_warn(\n                f\"Checkpoint directory {filepath} exists and is not empty with save_top_k != 0.\"\n                \"All files in this directory will be deleted when a checkpoint is saved!\"\n            )\n\n        self.monitor = monitor\n        self.verbose = verbose\n        if os.path.isdir(filepath):\n            self.dirpath, self.filename = filepath, '{epoch}'\n        else:\n            self.dirpath, self.filename = os.path.split(filepath)\n\n        os.makedirs(self.dirpath, exist_ok=True)\n        self.save_top_k = save_top_k\n        self.save_weights_only = save_weights_only\n        self.period = period\n        self.epoch_last_check = None\n        self.prefix = prefix\n        self.best_k_models = {}\n        # {filename: monitor}\n        self.kth_best_model = ''\n        self.best = 0\n        self.save_function = None\n\n        mode_dict = {\n            'min': (np.less, np.Inf, 'min'),\n            'max': (np.greater, -np.Inf, 'max'),\n            'auto': (np.greater, -np.Inf, 'max') if 'acc' in self.monitor or self.monitor.startswith('fmeasure')\n            else (np.less, np.Inf, 'min'),\n        }\n\n        if mode not in mode_dict:\n            rank_zero_warn(f'ModelCheckpoint mode {mode} is unknown, fallback to auto mode.', RuntimeWarning)\n            mode = 'auto'\n\n        self.monitor_op, self.kth_value, self.mode = mode_dict[mode]\n\n    def _del_model(self, filepath):\n        os.remove(filepath)\n\n    def _save_model(self, filepath):\n        # make paths\n        os.makedirs(os.path.dirname(filepath), exist_ok=True)\n\n        # delegate the saving to the model\n        if self.save_function is not None:\n            self.save_function(filepath)\n        else:\n            raise ValueError(\".save_function() not set\")\n\n    def check_monitor_top_k(self, current):\n        less_than_k_models = len(self.best_k_models) < self.save_top_k\n        if less_than_k_models:\n            return True\n        return self.monitor_op(current, self.best_k_models[self.kth_best_model])\n\n    def format_checkpoint_name(self, epoch, metrics, ver=None):\n        \"\"\"Generate a filename according to the defined template.\n\n        Example::\n\n            >>> tmpdir = os.path.dirname(__file__)\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(0, {}))\n            'epoch=0.ckpt'\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch:03d}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(5, {}))\n            'epoch=005.ckpt'\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch}-{val_loss:.2f}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(2, dict(val_loss=0.123456)))\n            'epoch=2-val_loss=0.12.ckpt'\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{missing:d}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(0, {}))\n            'missing=0.ckpt'\n        \"\"\"\n        # check if user passed in keys to the string\n        groups = re.findall(r'(\\{.*?)[:\\}]', self.filename)\n\n        if len(groups) == 0:\n            # default name\n            filename = f'{self.prefix}_ckpt_epoch_{epoch}'\n        else:\n            metrics['epoch'] = epoch\n            filename = self.filename\n            for tmp in groups:\n                name = tmp[1:]\n                filename = filename.replace(tmp, name + '={' + name)\n                if name not in metrics:\n                    metrics[name] = 0\n            filename = filename.format(**metrics)\n        str_ver = f'_v{ver}' if ver is not None else ''\n        filepath = os.path.join(self.dirpath, self.prefix + filename + str_ver + '.ckpt')\n        return filepath\n\n    def on_validation_end(self, trainer, pl_module):\n        # only run on main process\n        if trainer.proc_rank != 0:\n            return\n\n        metrics = trainer.callback_metrics\n        epoch = trainer.current_epoch\n        if self.save_top_k == 0:\n            # no models are saved\n            return\n        if self.epoch_last_check is not None and (epoch - self.epoch_last_check) < self.period:\n            # skipping in this term\n            return\n\n        self.epoch_last_check = epoch\n\n        filepath = self.format_checkpoint_name(epoch, metrics)\n        version_cnt = 0\n        while os.path.isfile(filepath):\n            filepath = self.format_checkpoint_name(epoch, metrics, ver=version_cnt)\n            # this epoch called before\n            version_cnt += 1\n\n        if self.save_top_k != -1:\n            current = metrics.get(self.monitor)\n\n            if current is None:\n                rank_zero_warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)\n            elif self.check_monitor_top_k(current):\n                self._do_check_save(filepath, current, epoch)\n            elif self.verbose > 0:\n                log.info(f'\\nEpoch {epoch:05d}: {self.monitor}  was not in top {self.save_top_k}')\n\n        else:\n            if self.verbose > 0:\n                log.info(f'\\nEpoch {epoch:05d}: saving model to {filepath}')\n            self._save_model(filepath)\n\n    def _do_check_save(self, filepath, current, epoch):\n        # remove kth\n        if len(self.best_k_models) == self.save_top_k and self.save_top_k > 0:\n            delpath = self.kth_best_model\n            self.best_k_models.pop(self.kth_best_model)\n            self._del_model(delpath)\n\n        self.best_k_models[filepath] = current\n        if len(self.best_k_models) == self.save_top_k:\n            # monitor dict has reached k elements\n            _op = max if self.mode == 'min' else min\n            self.kth_best_model = _op(self.best_k_models,\n                                      key=self.best_k_models.get)\n            self.kth_value = self.best_k_models[self.kth_best_model]\n\n        _op = min if self.mode == 'min' else max\n        self.best = _op(self.best_k_models.values())\n\n        if self.verbose > 0:\n            log.info(\n                f'\\nEpoch {epoch:05d}: {self.monitor} reached'\n                f' {current:0.5f} (best {self.best:0.5f}), saving model to'\n                f' {filepath} as top {self.save_top_k}')\n        self._save_model(filepath)\n", "path": "pytorch_lightning/callbacks/model_checkpoint.py"}], "after_files": [{"content": "\"\"\"\nModel Checkpointing\n===================\n\nAutomatically save model checkpoints during training.\n\n\"\"\"\n\nimport os\nimport re\n\nimport numpy as np\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.callbacks.base import Callback\nfrom pytorch_lightning.utilities import rank_zero_warn\n\n\nclass ModelCheckpoint(Callback):\n    r\"\"\"\n    Save the model after every epoch.\n\n    Args:\n        filepath: path to save the model file.\n            Can contain named formatting options to be auto-filled.\n\n            Example::\n\n                # custom path\n                # saves a file like: my/path/epoch_0.ckpt\n                >>> checkpoint_callback = ModelCheckpoint('my/path/')\n\n                # save any arbitrary metrics like `val_loss`, etc. in name\n                # saves a file like: my/path/epoch=2-val_loss=0.2_other_metric=0.3.ckpt\n                >>> checkpoint_callback = ModelCheckpoint(\n                ...     filepath='my/path/{epoch}-{val_loss:.2f}-{other_metric:.2f}'\n                ... )\n\n        monitor: quantity to monitor.\n        verbose: verbosity mode. Default: ``False``.\n        save_top_k: if `save_top_k == k`,\n            the best k models according to\n            the quantity monitored will be saved.\n            if ``save_top_k == 0``, no models are saved.\n            if ``save_top_k == -1``, all models are saved.\n            Please note that the monitors are checked every `period` epochs.\n            if ``save_top_k >= 2`` and the callback is called multiple\n            times inside an epoch, the name of the saved file will be\n            appended with a version count starting with `v0`.\n        mode: one of {auto, min, max}.\n            If ``save_top_k != 0``, the decision\n            to overwrite the current save file is made\n            based on either the maximization or the\n            minimization of the monitored quantity. For `val_acc`,\n            this should be `max`, for `val_loss` this should\n            be `min`, etc. In `auto` mode, the direction is\n            automatically inferred from the name of the monitored quantity.\n        save_weights_only: if ``True``, then only the model's weights will be\n            saved (``model.save_weights(filepath)``), else the full model\n            is saved (``model.save(filepath)``).\n        period: Interval (number of epochs) between checkpoints.\n\n    Example::\n\n        >>> from pytorch_lightning import Trainer\n        >>> from pytorch_lightning.callbacks import ModelCheckpoint\n\n        # saves checkpoints to 'my/path/' whenever 'val_loss' has a new min\n        >>> checkpoint_callback = ModelCheckpoint(filepath='my/path/')\n        >>> trainer = Trainer(checkpoint_callback=checkpoint_callback)\n\n        # save epoch and val_loss in name\n        # saves a file like: my/path/sample-mnist_epoch=02_val_loss=0.32.ckpt\n        >>> checkpoint_callback = ModelCheckpoint(\n        ...     filepath='my/path/sample-mnist_{epoch:02d}-{val_loss:.2f}'\n        ... )\n\n    \"\"\"\n\n    def __init__(self, filepath: str, monitor: str = 'val_loss', verbose: bool = False,\n                 save_top_k: int = 1, save_weights_only: bool = False,\n                 mode: str = 'auto', period: int = 1, prefix: str = ''):\n        super().__init__()\n        if save_top_k > 0 and os.path.isdir(filepath) and len(os.listdir(filepath)) > 0:\n            rank_zero_warn(\n                f\"Checkpoint directory {filepath} exists and is not empty with save_top_k != 0.\"\n                \"All files in this directory will be deleted when a checkpoint is saved!\"\n            )\n\n        self.monitor = monitor\n        self.verbose = verbose\n        if os.path.isdir(filepath):\n            self.dirpath, self.filename = filepath, '{epoch}'\n        else:\n            self.dirpath, self.filename = os.path.split(filepath)\n\n        os.makedirs(self.dirpath, exist_ok=True)\n        self.save_top_k = save_top_k\n        self.save_weights_only = save_weights_only\n        self.period = period\n        self.epoch_last_check = None\n        self.prefix = prefix\n        self.best_k_models = {}\n        # {filename: monitor}\n        self.kth_best_model = ''\n        self.best = 0\n        self.save_function = None\n\n        mode_dict = {\n            'min': (np.less, np.Inf, 'min'),\n            'max': (np.greater, -np.Inf, 'max'),\n            'auto': (np.greater, -np.Inf, 'max') if 'acc' in self.monitor or self.monitor.startswith('fmeasure')\n            else (np.less, np.Inf, 'min'),\n        }\n\n        if mode not in mode_dict:\n            rank_zero_warn(f'ModelCheckpoint mode {mode} is unknown, fallback to auto mode.', RuntimeWarning)\n            mode = 'auto'\n\n        self.monitor_op, self.kth_value, self.mode = mode_dict[mode]\n\n    def _del_model(self, filepath):\n        os.remove(filepath)\n\n    def _save_model(self, filepath):\n        # make paths\n        os.makedirs(os.path.dirname(filepath), exist_ok=True)\n\n        # delegate the saving to the model\n        if self.save_function is not None:\n            self.save_function(filepath)\n        else:\n            raise ValueError(\".save_function() not set\")\n\n    def check_monitor_top_k(self, current):\n        less_than_k_models = len(self.best_k_models) < self.save_top_k\n        if less_than_k_models:\n            return True\n        return self.monitor_op(current, self.best_k_models[self.kth_best_model])\n\n    def format_checkpoint_name(self, epoch, metrics, ver=None):\n        \"\"\"Generate a filename according to the defined template.\n\n        Example::\n\n            >>> tmpdir = os.path.dirname(__file__)\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(0, {}))\n            'epoch=0.ckpt'\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch:03d}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(5, {}))\n            'epoch=005.ckpt'\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{epoch}-{val_loss:.2f}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(2, dict(val_loss=0.123456)))\n            'epoch=2-val_loss=0.12.ckpt'\n            >>> ckpt = ModelCheckpoint(os.path.join(tmpdir, '{missing:d}'))\n            >>> os.path.basename(ckpt.format_checkpoint_name(0, {}))\n            'missing=0.ckpt'\n        \"\"\"\n        # check if user passed in keys to the string\n        groups = re.findall(r'(\\{.*?)[:\\}]', self.filename)\n\n        if len(groups) == 0:\n            # default name\n            filename = f'{self.prefix}_ckpt_epoch_{epoch}'\n        else:\n            metrics['epoch'] = epoch\n            filename = self.filename\n            for tmp in groups:\n                name = tmp[1:]\n                filename = filename.replace(tmp, name + '={' + name)\n                if name not in metrics:\n                    metrics[name] = 0\n            filename = filename.format(**metrics)\n        str_ver = f'_v{ver}' if ver is not None else ''\n        filepath = os.path.join(self.dirpath, self.prefix + filename + str_ver + '.ckpt')\n        return filepath\n\n    def on_validation_end(self, trainer, pl_module):\n        # only run on main process\n        if trainer.proc_rank != 0:\n            return\n\n        metrics = trainer.callback_metrics\n        epoch = trainer.current_epoch\n        if self.save_top_k == 0:\n            # no models are saved\n            return\n        if self.epoch_last_check is not None and (epoch - self.epoch_last_check) < self.period:\n            # skipping in this term\n            return\n\n        self.epoch_last_check = epoch\n\n        filepath = self.format_checkpoint_name(epoch, metrics)\n        version_cnt = 0\n        while os.path.isfile(filepath):\n            filepath = self.format_checkpoint_name(epoch, metrics, ver=version_cnt)\n            # this epoch called before\n            version_cnt += 1\n\n        if self.save_top_k != -1:\n            current = metrics.get(self.monitor)\n\n            if current is None:\n                rank_zero_warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)\n            elif self.check_monitor_top_k(current):\n                self._do_check_save(filepath, current, epoch)\n            elif self.verbose > 0:\n                log.info(f'\\nEpoch {epoch:05d}: {self.monitor}  was not in top {self.save_top_k}')\n\n        else:\n            if self.verbose > 0:\n                log.info(f'\\nEpoch {epoch:05d}: saving model to {filepath}')\n            self._save_model(filepath)\n\n    def _do_check_save(self, filepath, current, epoch):\n        # remove kth\n\n        del_list = []\n        if len(self.best_k_models) == self.save_top_k and self.save_top_k > 0:\n            delpath = self.kth_best_model\n            self.best_k_models.pop(self.kth_best_model)\n            del_list.append(delpath)\n\n        self.best_k_models[filepath] = current\n        if len(self.best_k_models) == self.save_top_k:\n            # monitor dict has reached k elements\n            _op = max if self.mode == 'min' else min\n            self.kth_best_model = _op(self.best_k_models,\n                                      key=self.best_k_models.get)\n            self.kth_value = self.best_k_models[self.kth_best_model]\n\n        _op = min if self.mode == 'min' else max\n        self.best = _op(self.best_k_models.values())\n\n        if self.verbose > 0:\n            log.info(\n                f'\\nEpoch {epoch:05d}: {self.monitor} reached'\n                f' {current:0.5f} (best {self.best:0.5f}), saving model to'\n                f' {filepath} as top {self.save_top_k}')\n        self._save_model(filepath)\n\n        for cur_path in del_list:\n            if cur_path != filepath:\n                self._del_model(cur_path)\n", "path": "pytorch_lightning/callbacks/model_checkpoint.py"}]}