We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Pascal training bug with CM
## 🐛 Bug description

```
 File "./code/scripts/training.py", line 252, in log_cm
cm = cm_metric.compute().numpy()
TypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.
2020-07-14 03:01:54,416 ignite.distributed.launcher.Parallel INFO: Finalized processing group with backend: 'nccl'
2020-07-14 03:01:54,417|training|ERROR|
```

## Environment

 - PyTorch Version (e.g., 1.4):
 - Ignite Version (e.g., 0.3.0):
 - OS (e.g., Linux):
 - How you installed Ignite (`conda`, `pip`, source):
 - Python version:
 - Any other relevant information:


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/references/segmentation/pascal_voc2012/code/scripts/training.py`
Content:
```
1 # This a training script launched with py_config_runner
2 # It should obligatory contain `run(config, **kwargs)` method
3 
4 from pathlib import Path
5 from collections.abc import Mapping
6 
7 import torch
8 
9 from apex import amp
10 
11 import ignite
12 import ignite.distributed as idist
13 from ignite.contrib.engines import common
14 from ignite.engine import Engine, Events, create_supervised_evaluator
15 from ignite.handlers import DiskSaver
16 from ignite.metrics import ConfusionMatrix, IoU, mIoU
17 from ignite.utils import setup_logger
18 
19 from py_config_runner.utils import set_seed
20 from py_config_runner.config_utils import get_params, TRAINVAL_CONFIG, assert_config
21 
22 import sys
23 
24 # Adds "code" folder to python path
25 sys.path.insert(0, Path(__file__).parent.parent.as_posix())
26 
27 from utils.handlers import predictions_gt_images_handler
28 from utils import exp_tracking
29 from dataflow.datasets import VOCSegmentationOpencv
30 
31 
32 def initialize(config):
33 
34     model = config.model.to(config.device)
35     optimizer = config.optimizer
36     # Setup Nvidia/Apex AMP
37     model, optimizer = amp.initialize(model, optimizer, opt_level=getattr(config, "fp16_opt_level", "O2"), num_losses=1)
38 
39     # Adapt model to dist conf
40     model = idist.auto_model(model)
41 
42     criterion = config.criterion.to(config.device)
43 
44     return model, optimizer, criterion
45 
46 
47 def get_save_handler(config):
48     if exp_tracking.has_trains:
49         from ignite.contrib.handlers.trains_logger import TrainsSaver
50 
51         return TrainsSaver(dirname=config.output_path.as_posix())
52 
53     return DiskSaver(config.output_path.as_posix())
54 
55 
56 def create_trainer(model, optimizer, criterion, train_sampler, config, logger):
57     prepare_batch = config.prepare_batch
58     device = config.device
59 
60     # Setup trainer
61     accumulation_steps = getattr(config, "accumulation_steps", 1)
62     model_output_transform = getattr(config, "model_output_transform", lambda x: x)
63 
64     def train_update_function(engine, batch):
65 
66         model.train()
67 
68         x, y = prepare_batch(batch, device=device, non_blocking=True)
69         y_pred = model(x)
70         y_pred = model_output_transform(y_pred)
71         loss = criterion(y_pred, y)
72 
73         if isinstance(loss, Mapping):
74             assert "supervised batch loss" in loss
75             loss_dict = loss
76             output = {k: v.item() for k, v in loss_dict.items()}
77             loss = loss_dict["supervised batch loss"] / accumulation_steps
78         else:
79             output = {"supervised batch loss": loss.item()}
80 
81         with amp.scale_loss(loss, optimizer, loss_id=0) as scaled_loss:
82             scaled_loss.backward()
83 
84         if engine.state.iteration % accumulation_steps == 0:
85             optimizer.step()
86             optimizer.zero_grad()
87 
88         return output
89 
90     output_names = getattr(config, "output_names", ["supervised batch loss",])
91     lr_scheduler = config.lr_scheduler
92 
93     trainer = Engine(train_update_function)
94     trainer.logger = logger
95 
96     to_save = {"model": model, "optimizer": optimizer, "lr_scheduler": lr_scheduler, "trainer": trainer, "amp": amp}
97 
98     save_every_iters = getattr(config, "save_every_iters", 1000)
99 
100     common.setup_common_training_handlers(
101         trainer,
102         train_sampler,
103         to_save=to_save,
104         save_every_iters=save_every_iters,
105         save_handler=get_save_handler(config),
106         lr_scheduler=lr_scheduler,
107         with_gpu_stats=exp_tracking.has_mlflow,
108         output_names=output_names,
109         with_pbars=False,
110     )
111 
112     if idist.get_rank() == 0:
113         common.ProgressBar(persist=False).attach(trainer, metric_names="all")
114 
115     return trainer
116 
117 
118 def create_evaluators(model, metrics, config):
119     model_output_transform = getattr(config, "model_output_transform", lambda x: x)
120 
121     evaluator_args = dict(
122         model=model,
123         metrics=metrics,
124         device=config.device,
125         non_blocking=True,
126         prepare_batch=config.prepare_batch,
127         output_transform=lambda x, y, y_pred: (model_output_transform(y_pred), y,),
128     )
129     train_evaluator = create_supervised_evaluator(**evaluator_args)
130     evaluator = create_supervised_evaluator(**evaluator_args)
131 
132     if idist.get_rank() == 0:
133         common.ProgressBar(desc="Evaluation (train)", persist=False).attach(train_evaluator)
134         common.ProgressBar(desc="Evaluation (val)", persist=False).attach(evaluator)
135 
136     return evaluator, train_evaluator
137 
138 
139 def log_metrics(logger, epoch, elapsed, tag, metrics):
140     logger.info(
141         "\nEpoch {} - Evaluation time (seconds): {} - {} metrics:\n {}".format(
142             epoch, int(elapsed), tag, "\n".join(["\t{}: {}".format(k, v) for k, v in metrics.items()])
143         )
144     )
145 
146 
147 def log_basic_info(logger, config):
148 
149     msg = "\n- PyTorch version: {}".format(torch.__version__)
150     msg += "\n- Ignite version: {}".format(ignite.__version__)
151     msg += "\n- Cuda device name: {}".format(torch.cuda.get_device_name(idist.get_local_rank()))
152 
153     logger.info(msg)
154 
155     if idist.get_world_size() > 1:
156         msg = "\nDistributed setting:"
157         msg += "\tbackend: {}".format(idist.backend())
158         msg += "\trank: {}".format(idist.get_rank())
159         msg += "\tworld size: {}".format(idist.get_world_size())
160         logger.info(msg)
161 
162 
163 def training(local_rank, config, logger=None):
164 
165     if not getattr(config, "use_fp16", True):
166         raise RuntimeError("This training script uses by default fp16 AMP")
167 
168     torch.backends.cudnn.benchmark = True
169 
170     set_seed(config.seed + local_rank)
171 
172     train_loader, val_loader, train_eval_loader = config.train_loader, config.val_loader, config.train_eval_loader
173 
174     # Setup model, optimizer, criterion
175     model, optimizer, criterion = initialize(config)
176 
177     # Setup trainer for this specific task
178     trainer = create_trainer(model, optimizer, criterion, train_loader.sampler, config, logger)
179 
180     # Setup evaluators
181     num_classes = config.num_classes
182     cm_metric = ConfusionMatrix(num_classes=num_classes)
183 
184     val_metrics = {
185         "IoU": IoU(cm_metric),
186         "mIoU_bg": mIoU(cm_metric),
187     }
188 
189     if hasattr(config, "val_metrics") and isinstance(config.val_metrics, dict):
190         val_metrics.update(config.val_metrics)
191 
192     evaluator, train_evaluator = create_evaluators(model, val_metrics, config)
193 
194     @trainer.on(Events.EPOCH_COMPLETED(every=getattr(config, "val_interval", 1)) | Events.COMPLETED)
195     def run_validation():
196         epoch = trainer.state.epoch
197         state = train_evaluator.run(train_eval_loader)
198         log_metrics(logger, epoch, state.times["COMPLETED"], "Train", state.metrics)
199         state = evaluator.run(val_loader)
200         log_metrics(logger, epoch, state.times["COMPLETED"], "Test", state.metrics)
201 
202     if getattr(config, "start_by_validation", False):
203         trainer.add_event_handler(Events.STARTED, run_validation)
204 
205     score_metric_name = "mIoU_bg"
206 
207     if hasattr(config, "es_patience"):
208         common.add_early_stopping_by_val_score(config.es_patience, evaluator, trainer, metric_name=score_metric_name)
209 
210     # Store 3 best models by validation accuracy:
211     common.gen_save_best_models_by_val_score(
212         save_handler=get_save_handler(config),
213         evaluator=evaluator,
214         models=model,
215         metric_name=score_metric_name,
216         n_saved=3,
217         trainer=trainer,
218         tag="val",
219     )
220 
221     if idist.get_rank() == 0:
222 
223         tb_logger = common.setup_tb_logging(
224             config.output_path.as_posix(),
225             trainer,
226             optimizer,
227             evaluators={"training": train_evaluator, "validation": evaluator},
228         )
229 
230         if not exp_tracking.has_trains:
231             exp_tracking_logger = exp_tracking.setup_logging(
232                 trainer, optimizer, evaluators={"training": train_evaluator, "validation": evaluator}
233             )
234 
235         # Log val predictions:
236         tb_logger.attach(
237             evaluator,
238             log_handler=predictions_gt_images_handler(
239                 img_denormalize_fn=config.img_denormalize, n_images=15, another_engine=trainer, prefix_tag="validation"
240             ),
241             event_name=Events.ITERATION_COMPLETED(once=len(val_loader) // 2),
242         )
243 
244         # Log confusion matrix to Trains:
245         if exp_tracking.has_trains:
246             from trains import Task
247 
248             trains_logger = Task.current_task().get_logger()
249 
250             @trainer.on(Events.COMPLETED)
251             def log_cm():
252                 cm = cm_metric.compute().numpy()
253                 cm = cm / (cm.sum(axis=1)[:, None] + 1e-15)
254                 trains_logger.report_confusion_matrix(
255                     title="Final Confusion Matrix",
256                     series="cm-preds-gt",
257                     matrix=cm,
258                     iteration=trainer.state.iteration,
259                     xlabels=VOCSegmentationOpencv.target_names,
260                     ylabels=VOCSegmentationOpencv.target_names,
261                 )
262 
263     trainer.run(train_loader, max_epochs=config.num_epochs)
264 
265     if idist.get_rank() == 0:
266         tb_logger.close()
267         if not exp_tracking.has_trains:
268             exp_tracking_logger.close()
269 
270 
271 def run(config, **kwargs):
272     """This is the main method to run the training. As this training script is launched with `py_config_runner`
273     it should obligatory contain `run(config, **kwargs)` method.
274 
275     """
276 
277     assert torch.cuda.is_available(), torch.cuda.is_available()
278     assert torch.backends.cudnn.enabled, "Nvidia/Amp requires cudnn backend to be enabled."
279 
280     with idist.Parallel(backend="nccl") as parallel:
281 
282         logger = setup_logger(name="Pascal-VOC12 Training", distributed_rank=idist.get_rank())
283 
284         assert_config(config, TRAINVAL_CONFIG)
285         # The following attributes are automatically added by py_config_runner
286         assert hasattr(config, "config_filepath") and isinstance(config.config_filepath, Path)
287         assert hasattr(config, "script_filepath") and isinstance(config.script_filepath, Path)
288 
289         if idist.get_rank() == 0 and exp_tracking.has_trains:
290             from trains import Task
291 
292             task = Task.init("Pascal-VOC12 Training", config.config_filepath.stem)
293             task.connect_configuration(config.config_filepath.as_posix())
294 
295         log_basic_info(logger, config)
296 
297         config.output_path = Path(exp_tracking.get_output_path())
298         # dump python files to reproduce the run
299         exp_tracking.log_artifact(config.config_filepath.as_posix())
300         exp_tracking.log_artifact(config.script_filepath.as_posix())
301         exp_tracking.log_params(get_params(config, TRAINVAL_CONFIG))
302 
303         try:
304             parallel.run(training, config, logger=logger)
305         except KeyboardInterrupt:
306             logger.info("Catched KeyboardInterrupt -> exit")
307         except Exception as e:  # noqa
308             logger.exception("")
309             raise e
310
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/references/segmentation/pascal_voc2012/code/scripts/training.py b/examples/references/segmentation/pascal_voc2012/code/scripts/training.py
--- a/examples/references/segmentation/pascal_voc2012/code/scripts/training.py
+++ b/examples/references/segmentation/pascal_voc2012/code/scripts/training.py
@@ -179,7 +179,7 @@
 
     # Setup evaluators
     num_classes = config.num_classes
-    cm_metric = ConfusionMatrix(num_classes=num_classes)
+    cm_metric = ConfusionMatrix(num_classes=num_classes, average="recall")
 
     val_metrics = {
         "IoU": IoU(cm_metric),
@@ -241,16 +241,17 @@
             event_name=Events.ITERATION_COMPLETED(once=len(val_loader) // 2),
         )
 
-        # Log confusion matrix to Trains:
-        if exp_tracking.has_trains:
-            from trains import Task
+    # Log confusion matrix to Trains:
+    if exp_tracking.has_trains:
+
+        @trainer.on(Events.COMPLETED)
+        def compute_and_log_cm():
+            cm = cm_metric.compute().cpu().numpy()
 
-            trains_logger = Task.current_task().get_logger()
+            if idist.get_rank() == 0:
+                from trains import Task
 
-            @trainer.on(Events.COMPLETED)
-            def log_cm():
-                cm = cm_metric.compute().numpy()
-                cm = cm / (cm.sum(axis=1)[:, None] + 1e-15)
+                trains_logger = Task.current_task().get_logger()
                 trains_logger.report_confusion_matrix(
                     title="Final Confusion Matrix",
                     series="cm-preds-gt",

{"golden_diff": "diff --git a/examples/references/segmentation/pascal_voc2012/code/scripts/training.py b/examples/references/segmentation/pascal_voc2012/code/scripts/training.py\n--- a/examples/references/segmentation/pascal_voc2012/code/scripts/training.py\n+++ b/examples/references/segmentation/pascal_voc2012/code/scripts/training.py\n@@ -179,7 +179,7 @@\n \n     # Setup evaluators\n     num_classes = config.num_classes\n-    cm_metric = ConfusionMatrix(num_classes=num_classes)\n+    cm_metric = ConfusionMatrix(num_classes=num_classes, average=\"recall\")\n \n     val_metrics = {\n         \"IoU\": IoU(cm_metric),\n@@ -241,16 +241,17 @@\n             event_name=Events.ITERATION_COMPLETED(once=len(val_loader) // 2),\n         )\n \n-        # Log confusion matrix to Trains:\n-        if exp_tracking.has_trains:\n-            from trains import Task\n+    # Log confusion matrix to Trains:\n+    if exp_tracking.has_trains:\n+\n+        @trainer.on(Events.COMPLETED)\n+        def compute_and_log_cm():\n+            cm = cm_metric.compute().cpu().numpy()\n \n-            trains_logger = Task.current_task().get_logger()\n+            if idist.get_rank() == 0:\n+                from trains import Task\n \n-            @trainer.on(Events.COMPLETED)\n-            def log_cm():\n-                cm = cm_metric.compute().numpy()\n-                cm = cm / (cm.sum(axis=1)[:, None] + 1e-15)\n+                trains_logger = Task.current_task().get_logger()\n                 trains_logger.report_confusion_matrix(\n                     title=\"Final Confusion Matrix\",\n                     series=\"cm-preds-gt\",\n", "issue": "Pascal training bug with CM\n## \ud83d\udc1b Bug description\r\n\r\n```\r\n File \"./code/scripts/training.py\", line 252, in log_cm\r\ncm = cm_metric.compute().numpy()\r\nTypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.\r\n2020-07-14 03:01:54,416 ignite.distributed.launcher.Parallel INFO: Finalized processing group with backend: 'nccl'\r\n2020-07-14 03:01:54,417|training|ERROR|\r\n```\r\n\r\n## Environment\r\n\r\n - PyTorch Version (e.g., 1.4):\r\n - Ignite Version (e.g., 0.3.0):\r\n - OS (e.g., Linux):\r\n - How you installed Ignite (`conda`, `pip`, source):\r\n - Python version:\r\n - Any other relevant information:\r\n\n", "before_files": [{"content": "# This a training script launched with py_config_runner\n# It should obligatory contain `run(config, **kwargs)` method\n\nfrom pathlib import Path\nfrom collections.abc import Mapping\n\nimport torch\n\nfrom apex import amp\n\nimport ignite\nimport ignite.distributed as idist\nfrom ignite.contrib.engines import common\nfrom ignite.engine import Engine, Events, create_supervised_evaluator\nfrom ignite.handlers import DiskSaver\nfrom ignite.metrics import ConfusionMatrix, IoU, mIoU\nfrom ignite.utils import setup_logger\n\nfrom py_config_runner.utils import set_seed\nfrom py_config_runner.config_utils import get_params, TRAINVAL_CONFIG, assert_config\n\nimport sys\n\n# Adds \"code\" folder to python path\nsys.path.insert(0, Path(__file__).parent.parent.as_posix())\n\nfrom utils.handlers import predictions_gt_images_handler\nfrom utils import exp_tracking\nfrom dataflow.datasets import VOCSegmentationOpencv\n\n\ndef initialize(config):\n\n    model = config.model.to(config.device)\n    optimizer = config.optimizer\n    # Setup Nvidia/Apex AMP\n    model, optimizer = amp.initialize(model, optimizer, opt_level=getattr(config, \"fp16_opt_level\", \"O2\"), num_losses=1)\n\n    # Adapt model to dist conf\n    model = idist.auto_model(model)\n\n    criterion = config.criterion.to(config.device)\n\n    return model, optimizer, criterion\n\n\ndef get_save_handler(config):\n    if exp_tracking.has_trains:\n        from ignite.contrib.handlers.trains_logger import TrainsSaver\n\n        return TrainsSaver(dirname=config.output_path.as_posix())\n\n    return DiskSaver(config.output_path.as_posix())\n\n\ndef create_trainer(model, optimizer, criterion, train_sampler, config, logger):\n    prepare_batch = config.prepare_batch\n    device = config.device\n\n    # Setup trainer\n    accumulation_steps = getattr(config, \"accumulation_steps\", 1)\n    model_output_transform = getattr(config, \"model_output_transform\", lambda x: x)\n\n    def train_update_function(engine, batch):\n\n        model.train()\n\n        x, y = prepare_batch(batch, device=device, non_blocking=True)\n        y_pred = model(x)\n        y_pred = model_output_transform(y_pred)\n        loss = criterion(y_pred, y)\n\n        if isinstance(loss, Mapping):\n            assert \"supervised batch loss\" in loss\n            loss_dict = loss\n            output = {k: v.item() for k, v in loss_dict.items()}\n            loss = loss_dict[\"supervised batch loss\"] / accumulation_steps\n        else:\n            output = {\"supervised batch loss\": loss.item()}\n\n        with amp.scale_loss(loss, optimizer, loss_id=0) as scaled_loss:\n            scaled_loss.backward()\n\n        if engine.state.iteration % accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n\n        return output\n\n    output_names = getattr(config, \"output_names\", [\"supervised batch loss\",])\n    lr_scheduler = config.lr_scheduler\n\n    trainer = Engine(train_update_function)\n    trainer.logger = logger\n\n    to_save = {\"model\": model, \"optimizer\": optimizer, \"lr_scheduler\": lr_scheduler, \"trainer\": trainer, \"amp\": amp}\n\n    save_every_iters = getattr(config, \"save_every_iters\", 1000)\n\n    common.setup_common_training_handlers(\n        trainer,\n        train_sampler,\n        to_save=to_save,\n        save_every_iters=save_every_iters,\n        save_handler=get_save_handler(config),\n        lr_scheduler=lr_scheduler,\n        with_gpu_stats=exp_tracking.has_mlflow,\n        output_names=output_names,\n        with_pbars=False,\n    )\n\n    if idist.get_rank() == 0:\n        common.ProgressBar(persist=False).attach(trainer, metric_names=\"all\")\n\n    return trainer\n\n\ndef create_evaluators(model, metrics, config):\n    model_output_transform = getattr(config, \"model_output_transform\", lambda x: x)\n\n    evaluator_args = dict(\n        model=model,\n        metrics=metrics,\n        device=config.device,\n        non_blocking=True,\n        prepare_batch=config.prepare_batch,\n        output_transform=lambda x, y, y_pred: (model_output_transform(y_pred), y,),\n    )\n    train_evaluator = create_supervised_evaluator(**evaluator_args)\n    evaluator = create_supervised_evaluator(**evaluator_args)\n\n    if idist.get_rank() == 0:\n        common.ProgressBar(desc=\"Evaluation (train)\", persist=False).attach(train_evaluator)\n        common.ProgressBar(desc=\"Evaluation (val)\", persist=False).attach(evaluator)\n\n    return evaluator, train_evaluator\n\n\ndef log_metrics(logger, epoch, elapsed, tag, metrics):\n    logger.info(\n        \"\\nEpoch {} - Evaluation time (seconds): {} - {} metrics:\\n {}\".format(\n            epoch, int(elapsed), tag, \"\\n\".join([\"\\t{}: {}\".format(k, v) for k, v in metrics.items()])\n        )\n    )\n\n\ndef log_basic_info(logger, config):\n\n    msg = \"\\n- PyTorch version: {}\".format(torch.__version__)\n    msg += \"\\n- Ignite version: {}\".format(ignite.__version__)\n    msg += \"\\n- Cuda device name: {}\".format(torch.cuda.get_device_name(idist.get_local_rank()))\n\n    logger.info(msg)\n\n    if idist.get_world_size() > 1:\n        msg = \"\\nDistributed setting:\"\n        msg += \"\\tbackend: {}\".format(idist.backend())\n        msg += \"\\trank: {}\".format(idist.get_rank())\n        msg += \"\\tworld size: {}\".format(idist.get_world_size())\n        logger.info(msg)\n\n\ndef training(local_rank, config, logger=None):\n\n    if not getattr(config, \"use_fp16\", True):\n        raise RuntimeError(\"This training script uses by default fp16 AMP\")\n\n    torch.backends.cudnn.benchmark = True\n\n    set_seed(config.seed + local_rank)\n\n    train_loader, val_loader, train_eval_loader = config.train_loader, config.val_loader, config.train_eval_loader\n\n    # Setup model, optimizer, criterion\n    model, optimizer, criterion = initialize(config)\n\n    # Setup trainer for this specific task\n    trainer = create_trainer(model, optimizer, criterion, train_loader.sampler, config, logger)\n\n    # Setup evaluators\n    num_classes = config.num_classes\n    cm_metric = ConfusionMatrix(num_classes=num_classes)\n\n    val_metrics = {\n        \"IoU\": IoU(cm_metric),\n        \"mIoU_bg\": mIoU(cm_metric),\n    }\n\n    if hasattr(config, \"val_metrics\") and isinstance(config.val_metrics, dict):\n        val_metrics.update(config.val_metrics)\n\n    evaluator, train_evaluator = create_evaluators(model, val_metrics, config)\n\n    @trainer.on(Events.EPOCH_COMPLETED(every=getattr(config, \"val_interval\", 1)) | Events.COMPLETED)\n    def run_validation():\n        epoch = trainer.state.epoch\n        state = train_evaluator.run(train_eval_loader)\n        log_metrics(logger, epoch, state.times[\"COMPLETED\"], \"Train\", state.metrics)\n        state = evaluator.run(val_loader)\n        log_metrics(logger, epoch, state.times[\"COMPLETED\"], \"Test\", state.metrics)\n\n    if getattr(config, \"start_by_validation\", False):\n        trainer.add_event_handler(Events.STARTED, run_validation)\n\n    score_metric_name = \"mIoU_bg\"\n\n    if hasattr(config, \"es_patience\"):\n        common.add_early_stopping_by_val_score(config.es_patience, evaluator, trainer, metric_name=score_metric_name)\n\n    # Store 3 best models by validation accuracy:\n    common.gen_save_best_models_by_val_score(\n        save_handler=get_save_handler(config),\n        evaluator=evaluator,\n        models=model,\n        metric_name=score_metric_name,\n        n_saved=3,\n        trainer=trainer,\n        tag=\"val\",\n    )\n\n    if idist.get_rank() == 0:\n\n        tb_logger = common.setup_tb_logging(\n            config.output_path.as_posix(),\n            trainer,\n            optimizer,\n            evaluators={\"training\": train_evaluator, \"validation\": evaluator},\n        )\n\n        if not exp_tracking.has_trains:\n            exp_tracking_logger = exp_tracking.setup_logging(\n                trainer, optimizer, evaluators={\"training\": train_evaluator, \"validation\": evaluator}\n            )\n\n        # Log val predictions:\n        tb_logger.attach(\n            evaluator,\n            log_handler=predictions_gt_images_handler(\n                img_denormalize_fn=config.img_denormalize, n_images=15, another_engine=trainer, prefix_tag=\"validation\"\n            ),\n            event_name=Events.ITERATION_COMPLETED(once=len(val_loader) // 2),\n        )\n\n        # Log confusion matrix to Trains:\n        if exp_tracking.has_trains:\n            from trains import Task\n\n            trains_logger = Task.current_task().get_logger()\n\n            @trainer.on(Events.COMPLETED)\n            def log_cm():\n                cm = cm_metric.compute().numpy()\n                cm = cm / (cm.sum(axis=1)[:, None] + 1e-15)\n                trains_logger.report_confusion_matrix(\n                    title=\"Final Confusion Matrix\",\n                    series=\"cm-preds-gt\",\n                    matrix=cm,\n                    iteration=trainer.state.iteration,\n                    xlabels=VOCSegmentationOpencv.target_names,\n                    ylabels=VOCSegmentationOpencv.target_names,\n                )\n\n    trainer.run(train_loader, max_epochs=config.num_epochs)\n\n    if idist.get_rank() == 0:\n        tb_logger.close()\n        if not exp_tracking.has_trains:\n            exp_tracking_logger.close()\n\n\ndef run(config, **kwargs):\n    \"\"\"This is the main method to run the training. As this training script is launched with `py_config_runner`\n    it should obligatory contain `run(config, **kwargs)` method.\n\n    \"\"\"\n\n    assert torch.cuda.is_available(), torch.cuda.is_available()\n    assert torch.backends.cudnn.enabled, \"Nvidia/Amp requires cudnn backend to be enabled.\"\n\n    with idist.Parallel(backend=\"nccl\") as parallel:\n\n        logger = setup_logger(name=\"Pascal-VOC12 Training\", distributed_rank=idist.get_rank())\n\n        assert_config(config, TRAINVAL_CONFIG)\n        # The following attributes are automatically added by py_config_runner\n        assert hasattr(config, \"config_filepath\") and isinstance(config.config_filepath, Path)\n        assert hasattr(config, \"script_filepath\") and isinstance(config.script_filepath, Path)\n\n        if idist.get_rank() == 0 and exp_tracking.has_trains:\n            from trains import Task\n\n            task = Task.init(\"Pascal-VOC12 Training\", config.config_filepath.stem)\n            task.connect_configuration(config.config_filepath.as_posix())\n\n        log_basic_info(logger, config)\n\n        config.output_path = Path(exp_tracking.get_output_path())\n        # dump python files to reproduce the run\n        exp_tracking.log_artifact(config.config_filepath.as_posix())\n        exp_tracking.log_artifact(config.script_filepath.as_posix())\n        exp_tracking.log_params(get_params(config, TRAINVAL_CONFIG))\n\n        try:\n            parallel.run(training, config, logger=logger)\n        except KeyboardInterrupt:\n            logger.info(\"Catched KeyboardInterrupt -> exit\")\n        except Exception as e:  # noqa\n            logger.exception(\"\")\n            raise e\n", "path": "examples/references/segmentation/pascal_voc2012/code/scripts/training.py"}], "after_files": [{"content": "# This a training script launched with py_config_runner\n# It should obligatory contain `run(config, **kwargs)` method\n\nfrom pathlib import Path\nfrom collections.abc import Mapping\n\nimport torch\n\nfrom apex import amp\n\nimport ignite\nimport ignite.distributed as idist\nfrom ignite.contrib.engines import common\nfrom ignite.engine import Engine, Events, create_supervised_evaluator\nfrom ignite.handlers import DiskSaver\nfrom ignite.metrics import ConfusionMatrix, IoU, mIoU\nfrom ignite.utils import setup_logger\n\nfrom py_config_runner.utils import set_seed\nfrom py_config_runner.config_utils import get_params, TRAINVAL_CONFIG, assert_config\n\nimport sys\n\n# Adds \"code\" folder to python path\nsys.path.insert(0, Path(__file__).parent.parent.as_posix())\n\nfrom utils.handlers import predictions_gt_images_handler\nfrom utils import exp_tracking\nfrom dataflow.datasets import VOCSegmentationOpencv\n\n\ndef initialize(config):\n\n    model = config.model.to(config.device)\n    optimizer = config.optimizer\n    # Setup Nvidia/Apex AMP\n    model, optimizer = amp.initialize(model, optimizer, opt_level=getattr(config, \"fp16_opt_level\", \"O2\"), num_losses=1)\n\n    # Adapt model to dist conf\n    model = idist.auto_model(model)\n\n    criterion = config.criterion.to(config.device)\n\n    return model, optimizer, criterion\n\n\ndef get_save_handler(config):\n    if exp_tracking.has_trains:\n        from ignite.contrib.handlers.trains_logger import TrainsSaver\n\n        return TrainsSaver(dirname=config.output_path.as_posix())\n\n    return DiskSaver(config.output_path.as_posix())\n\n\ndef create_trainer(model, optimizer, criterion, train_sampler, config, logger):\n    prepare_batch = config.prepare_batch\n    device = config.device\n\n    # Setup trainer\n    accumulation_steps = getattr(config, \"accumulation_steps\", 1)\n    model_output_transform = getattr(config, \"model_output_transform\", lambda x: x)\n\n    def train_update_function(engine, batch):\n\n        model.train()\n\n        x, y = prepare_batch(batch, device=device, non_blocking=True)\n        y_pred = model(x)\n        y_pred = model_output_transform(y_pred)\n        loss = criterion(y_pred, y)\n\n        if isinstance(loss, Mapping):\n            assert \"supervised batch loss\" in loss\n            loss_dict = loss\n            output = {k: v.item() for k, v in loss_dict.items()}\n            loss = loss_dict[\"supervised batch loss\"] / accumulation_steps\n        else:\n            output = {\"supervised batch loss\": loss.item()}\n\n        with amp.scale_loss(loss, optimizer, loss_id=0) as scaled_loss:\n            scaled_loss.backward()\n\n        if engine.state.iteration % accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n\n        return output\n\n    output_names = getattr(config, \"output_names\", [\"supervised batch loss\",])\n    lr_scheduler = config.lr_scheduler\n\n    trainer = Engine(train_update_function)\n    trainer.logger = logger\n\n    to_save = {\"model\": model, \"optimizer\": optimizer, \"lr_scheduler\": lr_scheduler, \"trainer\": trainer, \"amp\": amp}\n\n    save_every_iters = getattr(config, \"save_every_iters\", 1000)\n\n    common.setup_common_training_handlers(\n        trainer,\n        train_sampler,\n        to_save=to_save,\n        save_every_iters=save_every_iters,\n        save_handler=get_save_handler(config),\n        lr_scheduler=lr_scheduler,\n        with_gpu_stats=exp_tracking.has_mlflow,\n        output_names=output_names,\n        with_pbars=False,\n    )\n\n    if idist.get_rank() == 0:\n        common.ProgressBar(persist=False).attach(trainer, metric_names=\"all\")\n\n    return trainer\n\n\ndef create_evaluators(model, metrics, config):\n    model_output_transform = getattr(config, \"model_output_transform\", lambda x: x)\n\n    evaluator_args = dict(\n        model=model,\n        metrics=metrics,\n        device=config.device,\n        non_blocking=True,\n        prepare_batch=config.prepare_batch,\n        output_transform=lambda x, y, y_pred: (model_output_transform(y_pred), y,),\n    )\n    train_evaluator = create_supervised_evaluator(**evaluator_args)\n    evaluator = create_supervised_evaluator(**evaluator_args)\n\n    if idist.get_rank() == 0:\n        common.ProgressBar(desc=\"Evaluation (train)\", persist=False).attach(train_evaluator)\n        common.ProgressBar(desc=\"Evaluation (val)\", persist=False).attach(evaluator)\n\n    return evaluator, train_evaluator\n\n\ndef log_metrics(logger, epoch, elapsed, tag, metrics):\n    logger.info(\n        \"\\nEpoch {} - Evaluation time (seconds): {} - {} metrics:\\n {}\".format(\n            epoch, int(elapsed), tag, \"\\n\".join([\"\\t{}: {}\".format(k, v) for k, v in metrics.items()])\n        )\n    )\n\n\ndef log_basic_info(logger, config):\n\n    msg = \"\\n- PyTorch version: {}\".format(torch.__version__)\n    msg += \"\\n- Ignite version: {}\".format(ignite.__version__)\n    msg += \"\\n- Cuda device name: {}\".format(torch.cuda.get_device_name(idist.get_local_rank()))\n\n    logger.info(msg)\n\n    if idist.get_world_size() > 1:\n        msg = \"\\nDistributed setting:\"\n        msg += \"\\tbackend: {}\".format(idist.backend())\n        msg += \"\\trank: {}\".format(idist.get_rank())\n        msg += \"\\tworld size: {}\".format(idist.get_world_size())\n        logger.info(msg)\n\n\ndef training(local_rank, config, logger=None):\n\n    if not getattr(config, \"use_fp16\", True):\n        raise RuntimeError(\"This training script uses by default fp16 AMP\")\n\n    torch.backends.cudnn.benchmark = True\n\n    set_seed(config.seed + local_rank)\n\n    train_loader, val_loader, train_eval_loader = config.train_loader, config.val_loader, config.train_eval_loader\n\n    # Setup model, optimizer, criterion\n    model, optimizer, criterion = initialize(config)\n\n    # Setup trainer for this specific task\n    trainer = create_trainer(model, optimizer, criterion, train_loader.sampler, config, logger)\n\n    # Setup evaluators\n    num_classes = config.num_classes\n    cm_metric = ConfusionMatrix(num_classes=num_classes, average=\"recall\")\n\n    val_metrics = {\n        \"IoU\": IoU(cm_metric),\n        \"mIoU_bg\": mIoU(cm_metric),\n    }\n\n    if hasattr(config, \"val_metrics\") and isinstance(config.val_metrics, dict):\n        val_metrics.update(config.val_metrics)\n\n    evaluator, train_evaluator = create_evaluators(model, val_metrics, config)\n\n    @trainer.on(Events.EPOCH_COMPLETED(every=getattr(config, \"val_interval\", 1)) | Events.COMPLETED)\n    def run_validation():\n        epoch = trainer.state.epoch\n        state = train_evaluator.run(train_eval_loader)\n        log_metrics(logger, epoch, state.times[\"COMPLETED\"], \"Train\", state.metrics)\n        state = evaluator.run(val_loader)\n        log_metrics(logger, epoch, state.times[\"COMPLETED\"], \"Test\", state.metrics)\n\n    if getattr(config, \"start_by_validation\", False):\n        trainer.add_event_handler(Events.STARTED, run_validation)\n\n    score_metric_name = \"mIoU_bg\"\n\n    if hasattr(config, \"es_patience\"):\n        common.add_early_stopping_by_val_score(config.es_patience, evaluator, trainer, metric_name=score_metric_name)\n\n    # Store 3 best models by validation accuracy:\n    common.gen_save_best_models_by_val_score(\n        save_handler=get_save_handler(config),\n        evaluator=evaluator,\n        models=model,\n        metric_name=score_metric_name,\n        n_saved=3,\n        trainer=trainer,\n        tag=\"val\",\n    )\n\n    if idist.get_rank() == 0:\n\n        tb_logger = common.setup_tb_logging(\n            config.output_path.as_posix(),\n            trainer,\n            optimizer,\n            evaluators={\"training\": train_evaluator, \"validation\": evaluator},\n        )\n\n        if not exp_tracking.has_trains:\n            exp_tracking_logger = exp_tracking.setup_logging(\n                trainer, optimizer, evaluators={\"training\": train_evaluator, \"validation\": evaluator}\n            )\n\n        # Log val predictions:\n        tb_logger.attach(\n            evaluator,\n            log_handler=predictions_gt_images_handler(\n                img_denormalize_fn=config.img_denormalize, n_images=15, another_engine=trainer, prefix_tag=\"validation\"\n            ),\n            event_name=Events.ITERATION_COMPLETED(once=len(val_loader) // 2),\n        )\n\n    # Log confusion matrix to Trains:\n    if exp_tracking.has_trains:\n\n        @trainer.on(Events.COMPLETED)\n        def compute_and_log_cm():\n            cm = cm_metric.compute().cpu().numpy()\n\n            if idist.get_rank() == 0:\n                from trains import Task\n\n                trains_logger = Task.current_task().get_logger()\n                trains_logger.report_confusion_matrix(\n                    title=\"Final Confusion Matrix\",\n                    series=\"cm-preds-gt\",\n                    matrix=cm,\n                    iteration=trainer.state.iteration,\n                    xlabels=VOCSegmentationOpencv.target_names,\n                    ylabels=VOCSegmentationOpencv.target_names,\n                )\n\n    trainer.run(train_loader, max_epochs=config.num_epochs)\n\n    if idist.get_rank() == 0:\n        tb_logger.close()\n        if not exp_tracking.has_trains:\n            exp_tracking_logger.close()\n\n\ndef run(config, **kwargs):\n    \"\"\"This is the main method to run the training. As this training script is launched with `py_config_runner`\n    it should obligatory contain `run(config, **kwargs)` method.\n\n    \"\"\"\n\n    assert torch.cuda.is_available(), torch.cuda.is_available()\n    assert torch.backends.cudnn.enabled, \"Nvidia/Amp requires cudnn backend to be enabled.\"\n\n    with idist.Parallel(backend=\"nccl\") as parallel:\n\n        logger = setup_logger(name=\"Pascal-VOC12 Training\", distributed_rank=idist.get_rank())\n\n        assert_config(config, TRAINVAL_CONFIG)\n        # The following attributes are automatically added by py_config_runner\n        assert hasattr(config, \"config_filepath\") and isinstance(config.config_filepath, Path)\n        assert hasattr(config, \"script_filepath\") and isinstance(config.script_filepath, Path)\n\n        if idist.get_rank() == 0 and exp_tracking.has_trains:\n            from trains import Task\n\n            task = Task.init(\"Pascal-VOC12 Training\", config.config_filepath.stem)\n            task.connect_configuration(config.config_filepath.as_posix())\n\n        log_basic_info(logger, config)\n\n        config.output_path = Path(exp_tracking.get_output_path())\n        # dump python files to reproduce the run\n        exp_tracking.log_artifact(config.config_filepath.as_posix())\n        exp_tracking.log_artifact(config.script_filepath.as_posix())\n        exp_tracking.log_params(get_params(config, TRAINVAL_CONFIG))\n\n        try:\n            parallel.run(training, config, logger=logger)\n        except KeyboardInterrupt:\n            logger.info(\"Catched KeyboardInterrupt -> exit\")\n        except Exception as e:  # noqa\n            logger.exception(\"\")\n            raise e\n", "path": "examples/references/segmentation/pascal_voc2012/code/scripts/training.py"}]}