We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
'float' object has no attribute 'detach' in pytorch imagenet example.
I ran the [pytorch imagenet example](https://github.com/horovod/horovod/blob/master/examples/pytorch_imagenet_resnet50.py) but got an error that `float` number don't have `detach()` method. It seems that `loss.item()` lead to the `float` number, but I don't know how to fix that in `horovod` framework. 

Can anyone help me? Thanks a lot!

```
mpirun -np 4 \
  -H localhost:4 \
  -bind-to none -map-by slot \
  -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH -x PATH \
  -mca pml ob1 -mca btl ^openib \
  python main_hvd.py --train-dir /datasets/ILSVRC2012/images/train --val-dir /datasets/ILSVRC2012/images/val
```

```
Train Epoch     #1:   0%|          | 0/10010 [00:00<?, ?it/s]Traceback (most recent call last):
  File "main_hvd.py", line 272, in <module>
    train(epoch)
  File "main_hvd.py", line 179, in train
    train_loss.update(loss.item())
  File "main_hvd.py", line 263, in update
    self.sum += hvd.allreduce(val.detach().cpu(), name=self.name)
AttributeError: 'float' object has no attribute 'detach'
```

My environment is:
* pytorch==0.4.1
* horovod==0.16.0

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/pytorch_imagenet_resnet50.py`
Content:
```
1 from __future__ import print_function
2 
3 import torch
4 import argparse
5 import torch.backends.cudnn as cudnn
6 import torch.nn.functional as F
7 import torch.optim as optim
8 import torch.utils.data.distributed
9 from torchvision import datasets, transforms, models
10 import horovod.torch as hvd
11 import tensorboardX
12 import os
13 import math
14 from tqdm import tqdm
15 
16 # Training settings
17 parser = argparse.ArgumentParser(description='PyTorch ImageNet Example',
18                                  formatter_class=argparse.ArgumentDefaultsHelpFormatter)
19 parser.add_argument('--train-dir', default=os.path.expanduser('~/imagenet/train'),
20                     help='path to training data')
21 parser.add_argument('--val-dir', default=os.path.expanduser('~/imagenet/validation'),
22                     help='path to validation data')
23 parser.add_argument('--log-dir', default='./logs',
24                     help='tensorboard log directory')
25 parser.add_argument('--checkpoint-format', default='./checkpoint-{epoch}.pth.tar',
26                     help='checkpoint file format')
27 parser.add_argument('--fp16-allreduce', action='store_true', default=False,
28                     help='use fp16 compression during allreduce')
29 parser.add_argument('--batches-per-allreduce', type=int, default=1,
30                     help='number of batches processed locally before '
31                          'executing allreduce across workers; it multiplies '
32                          'total batch size.')
33 
34 # Default settings from https://arxiv.org/abs/1706.02677.
35 parser.add_argument('--batch-size', type=int, default=32,
36                     help='input batch size for training')
37 parser.add_argument('--val-batch-size', type=int, default=32,
38                     help='input batch size for validation')
39 parser.add_argument('--epochs', type=int, default=90,
40                     help='number of epochs to train')
41 parser.add_argument('--base-lr', type=float, default=0.0125,
42                     help='learning rate for a single GPU')
43 parser.add_argument('--warmup-epochs', type=float, default=5,
44                     help='number of warmup epochs')
45 parser.add_argument('--momentum', type=float, default=0.9,
46                     help='SGD momentum')
47 parser.add_argument('--wd', type=float, default=0.00005,
48                     help='weight decay')
49 
50 parser.add_argument('--no-cuda', action='store_true', default=False,
51                     help='disables CUDA training')
52 parser.add_argument('--seed', type=int, default=42,
53                     help='random seed')
54 
55 args = parser.parse_args()
56 args.cuda = not args.no_cuda and torch.cuda.is_available()
57 
58 allreduce_batch_size = args.batch_size * args.batches_per_allreduce
59 
60 hvd.init()
61 torch.manual_seed(args.seed)
62 
63 if args.cuda:
64     # Horovod: pin GPU to local rank.
65     torch.cuda.set_device(hvd.local_rank())
66     torch.cuda.manual_seed(args.seed)
67 
68 cudnn.benchmark = True
69 
70 # If set > 0, will resume training from a given checkpoint.
71 resume_from_epoch = 0
72 for try_epoch in range(args.epochs, 0, -1):
73     if os.path.exists(args.checkpoint_format.format(epoch=try_epoch)):
74         resume_from_epoch = try_epoch
75         break
76 
77 # Horovod: broadcast resume_from_epoch from rank 0 (which will have
78 # checkpoints) to other ranks.
79 resume_from_epoch = hvd.broadcast(torch.tensor(resume_from_epoch), root_rank=0,
80                                   name='resume_from_epoch').item()
81 
82 # Horovod: print logs on the first worker.
83 verbose = 1 if hvd.rank() == 0 else 0
84 
85 # Horovod: write TensorBoard logs on first worker.
86 log_writer = tensorboardX.SummaryWriter(args.log_dir) if hvd.rank() == 0 else None
87 
88 
89 kwargs = {'num_workers': 4, 'pin_memory': True} if args.cuda else {}
90 train_dataset = \
91     datasets.ImageFolder(args.train_dir,
92                          transform=transforms.Compose([
93                              transforms.RandomResizedCrop(224),
94                              transforms.RandomHorizontalFlip(),
95                              transforms.ToTensor(),
96                              transforms.Normalize(mean=[0.485, 0.456, 0.406],
97                                                   std=[0.229, 0.224, 0.225])
98                          ]))
99 # Horovod: use DistributedSampler to partition data among workers. Manually specify
100 # `num_replicas=hvd.size()` and `rank=hvd.rank()`.
101 train_sampler = torch.utils.data.distributed.DistributedSampler(
102     train_dataset, num_replicas=hvd.size(), rank=hvd.rank())
103 train_loader = torch.utils.data.DataLoader(
104     train_dataset, batch_size=allreduce_batch_size,
105     sampler=train_sampler, **kwargs)
106 
107 val_dataset = \
108     datasets.ImageFolder(args.val_dir,
109                          transform=transforms.Compose([
110                              transforms.Resize(256),
111                              transforms.CenterCrop(224),
112                              transforms.ToTensor(),
113                              transforms.Normalize(mean=[0.485, 0.456, 0.406],
114                                                   std=[0.229, 0.224, 0.225])
115                          ]))
116 val_sampler = torch.utils.data.distributed.DistributedSampler(
117     val_dataset, num_replicas=hvd.size(), rank=hvd.rank())
118 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size,
119                                          sampler=val_sampler, **kwargs)
120 
121 
122 # Set up standard ResNet-50 model.
123 model = models.resnet50()
124 
125 if args.cuda:
126     # Move model to GPU.
127     model.cuda()
128 
129 # Horovod: scale learning rate by the number of GPUs.
130 # Gradient Accumulation: scale learning rate by batches_per_allreduce
131 optimizer = optim.SGD(model.parameters(),
132                       lr=(args.base_lr *
133                           args.batches_per_allreduce * hvd.size()),
134                       momentum=args.momentum, weight_decay=args.wd)
135 
136 # Horovod: (optional) compression algorithm.
137 compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none
138 
139 # Horovod: wrap optimizer with DistributedOptimizer.
140 optimizer = hvd.DistributedOptimizer(
141     optimizer, named_parameters=model.named_parameters(),
142     compression=compression,
143     backward_passes_per_step=args.batches_per_allreduce)
144 
145 # Restore from a previous checkpoint, if initial_epoch is specified.
146 # Horovod: restore on the first worker which will broadcast weights to other workers.
147 if resume_from_epoch > 0 and hvd.rank() == 0:
148     filepath = args.checkpoint_format.format(epoch=resume_from_epoch)
149     checkpoint = torch.load(filepath)
150     model.load_state_dict(checkpoint['model'])
151     optimizer.load_state_dict(checkpoint['optimizer'])
152 
153 # Horovod: broadcast parameters & optimizer state.
154 hvd.broadcast_parameters(model.state_dict(), root_rank=0)
155 hvd.broadcast_optimizer_state(optimizer, root_rank=0)
156 
157 def train(epoch):
158     model.train()
159     train_sampler.set_epoch(epoch)
160     train_loss = Metric('train_loss')
161     train_accuracy = Metric('train_accuracy')
162 
163     with tqdm(total=len(train_loader),
164               desc='Train Epoch     #{}'.format(epoch + 1),
165               disable=not verbose) as t:
166         for batch_idx, (data, target) in enumerate(train_loader):
167             adjust_learning_rate(epoch, batch_idx)
168 
169             if args.cuda:
170                 data, target = data.cuda(), target.cuda()
171             optimizer.zero_grad()
172             # Split data into sub-batches of size batch_size
173             for i in range(0, len(data), args.batch_size):
174                 data_batch = data[i:i + args.batch_size]
175                 target_batch = target[i:i + args.batch_size]
176                 output = model(data_batch)
177                 train_accuracy.update(accuracy(output, target_batch))
178                 loss = F.cross_entropy(output, target_batch)
179                 train_loss.update(loss.item())
180                 # Average gradients among sub-batches
181                 loss.div_(math.ceil(float(len(data)) / args.batch_size))
182                 loss.backward()
183             # Gradient is applied across all ranks
184             optimizer.step()
185             t.set_postfix({'loss': train_loss.avg.item(),
186                            'accuracy': 100. * train_accuracy.avg.item()})
187             t.update(1)
188 
189     if log_writer:
190         log_writer.add_scalar('train/loss', train_loss.avg, epoch)
191         log_writer.add_scalar('train/accuracy', train_accuracy.avg, epoch)
192 
193 
194 def validate(epoch):
195     model.eval()
196     val_loss = Metric('val_loss')
197     val_accuracy = Metric('val_accuracy')
198 
199     with tqdm(total=len(val_loader),
200               desc='Validate Epoch  #{}'.format(epoch + 1),
201               disable=not verbose) as t:
202         with torch.no_grad():
203             for data, target in val_loader:
204                 if args.cuda:
205                     data, target = data.cuda(), target.cuda()
206                 output = model(data)
207 
208                 val_loss.update(F.cross_entropy(output, target))
209                 val_accuracy.update(accuracy(output, target))
210                 t.set_postfix({'loss': val_loss.avg.item(),
211                                'accuracy': 100. * val_accuracy.avg.item()})
212                 t.update(1)
213 
214     if log_writer:
215         log_writer.add_scalar('val/loss', val_loss.avg, epoch)
216         log_writer.add_scalar('val/accuracy', val_accuracy.avg, epoch)
217 
218 
219 # Horovod: using `lr = base_lr * hvd.size()` from the very beginning leads to worse final
220 # accuracy. Scale the learning rate `lr = base_lr` ---> `lr = base_lr * hvd.size()` during
221 # the first five epochs. See https://arxiv.org/abs/1706.02677 for details.
222 # After the warmup reduce learning rate by 10 on the 30th, 60th and 80th epochs.
223 def adjust_learning_rate(epoch, batch_idx):
224     if epoch < args.warmup_epochs:
225         epoch += float(batch_idx + 1) / len(train_loader)
226         lr_adj = 1. / hvd.size() * (epoch * (hvd.size() - 1) / args.warmup_epochs + 1)
227     elif epoch < 30:
228         lr_adj = 1.
229     elif epoch < 60:
230         lr_adj = 1e-1
231     elif epoch < 80:
232         lr_adj = 1e-2
233     else:
234         lr_adj = 1e-3
235     for param_group in optimizer.param_groups:
236         param_group['lr'] = args.base_lr * hvd.size() * args.batches_per_allreduce * lr_adj
237 
238 
239 def accuracy(output, target):
240     # get the index of the max log-probability
241     pred = output.max(1, keepdim=True)[1]
242     return pred.eq(target.view_as(pred)).cpu().float().mean()
243 
244 
245 def save_checkpoint(epoch):
246     if hvd.rank() == 0:
247         filepath = args.checkpoint_format.format(epoch=epoch + 1)
248         state = {
249             'model': model.state_dict(),
250             'optimizer': optimizer.state_dict(),
251         }
252         torch.save(state, filepath)
253 
254 
255 # Horovod: average metrics from distributed training.
256 class Metric(object):
257     def __init__(self, name):
258         self.name = name
259         self.sum = torch.tensor(0.)
260         self.n = torch.tensor(0.)
261 
262     def update(self, val):
263         self.sum += hvd.allreduce(val.detach().cpu(), name=self.name)
264         self.n += 1
265 
266     @property
267     def avg(self):
268         return self.sum / self.n
269 
270 
271 for epoch in range(resume_from_epoch, args.epochs):
272     train(epoch)
273     validate(epoch)
274     save_checkpoint(epoch)
275
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/pytorch_imagenet_resnet50.py b/examples/pytorch_imagenet_resnet50.py
--- a/examples/pytorch_imagenet_resnet50.py
+++ b/examples/pytorch_imagenet_resnet50.py
@@ -176,7 +176,7 @@
                 output = model(data_batch)
                 train_accuracy.update(accuracy(output, target_batch))
                 loss = F.cross_entropy(output, target_batch)
-                train_loss.update(loss.item())
+                train_loss.update(loss)
                 # Average gradients among sub-batches
                 loss.div_(math.ceil(float(len(data)) / args.batch_size))
                 loss.backward()

{"golden_diff": "diff --git a/examples/pytorch_imagenet_resnet50.py b/examples/pytorch_imagenet_resnet50.py\n--- a/examples/pytorch_imagenet_resnet50.py\n+++ b/examples/pytorch_imagenet_resnet50.py\n@@ -176,7 +176,7 @@\n                 output = model(data_batch)\n                 train_accuracy.update(accuracy(output, target_batch))\n                 loss = F.cross_entropy(output, target_batch)\n-                train_loss.update(loss.item())\n+                train_loss.update(loss)\n                 # Average gradients among sub-batches\n                 loss.div_(math.ceil(float(len(data)) / args.batch_size))\n                 loss.backward()\n", "issue": "'float' object has no attribute 'detach' in pytorch imagenet example.\nI ran the [pytorch imagenet example](https://github.com/horovod/horovod/blob/master/examples/pytorch_imagenet_resnet50.py) but got an error that `float` number don't have `detach()` method. It seems that `loss.item()` lead to the `float` number, but I don't know how to fix that in `horovod` framework. \r\n\r\nCan anyone help me? Thanks a lot!\r\n\r\n```\r\nmpirun -np 4 \\\r\n  -H localhost:4 \\\r\n  -bind-to none -map-by slot \\\r\n  -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH -x PATH \\\r\n  -mca pml ob1 -mca btl ^openib \\\r\n  python main_hvd.py --train-dir /datasets/ILSVRC2012/images/train --val-dir /datasets/ILSVRC2012/images/val\r\n```\r\n\r\n```\r\nTrain Epoch     #1:   0%|          | 0/10010 [00:00<?, ?it/s]Traceback (most recent call last):\r\n  File \"main_hvd.py\", line 272, in <module>\r\n    train(epoch)\r\n  File \"main_hvd.py\", line 179, in train\r\n    train_loss.update(loss.item())\r\n  File \"main_hvd.py\", line 263, in update\r\n    self.sum += hvd.allreduce(val.detach().cpu(), name=self.name)\r\nAttributeError: 'float' object has no attribute 'detach'\r\n```\r\n\r\nMy environment is:\r\n* pytorch==0.4.1\r\n* horovod==0.16.0\n", "before_files": [{"content": "from __future__ import print_function\n\nimport torch\nimport argparse\nimport torch.backends.cudnn as cudnn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data.distributed\nfrom torchvision import datasets, transforms, models\nimport horovod.torch as hvd\nimport tensorboardX\nimport os\nimport math\nfrom tqdm import tqdm\n\n# Training settings\nparser = argparse.ArgumentParser(description='PyTorch ImageNet Example',\n                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\nparser.add_argument('--train-dir', default=os.path.expanduser('~/imagenet/train'),\n                    help='path to training data')\nparser.add_argument('--val-dir', default=os.path.expanduser('~/imagenet/validation'),\n                    help='path to validation data')\nparser.add_argument('--log-dir', default='./logs',\n                    help='tensorboard log directory')\nparser.add_argument('--checkpoint-format', default='./checkpoint-{epoch}.pth.tar',\n                    help='checkpoint file format')\nparser.add_argument('--fp16-allreduce', action='store_true', default=False,\n                    help='use fp16 compression during allreduce')\nparser.add_argument('--batches-per-allreduce', type=int, default=1,\n                    help='number of batches processed locally before '\n                         'executing allreduce across workers; it multiplies '\n                         'total batch size.')\n\n# Default settings from https://arxiv.org/abs/1706.02677.\nparser.add_argument('--batch-size', type=int, default=32,\n                    help='input batch size for training')\nparser.add_argument('--val-batch-size', type=int, default=32,\n                    help='input batch size for validation')\nparser.add_argument('--epochs', type=int, default=90,\n                    help='number of epochs to train')\nparser.add_argument('--base-lr', type=float, default=0.0125,\n                    help='learning rate for a single GPU')\nparser.add_argument('--warmup-epochs', type=float, default=5,\n                    help='number of warmup epochs')\nparser.add_argument('--momentum', type=float, default=0.9,\n                    help='SGD momentum')\nparser.add_argument('--wd', type=float, default=0.00005,\n                    help='weight decay')\n\nparser.add_argument('--no-cuda', action='store_true', default=False,\n                    help='disables CUDA training')\nparser.add_argument('--seed', type=int, default=42,\n                    help='random seed')\n\nargs = parser.parse_args()\nargs.cuda = not args.no_cuda and torch.cuda.is_available()\n\nallreduce_batch_size = args.batch_size * args.batches_per_allreduce\n\nhvd.init()\ntorch.manual_seed(args.seed)\n\nif args.cuda:\n    # Horovod: pin GPU to local rank.\n    torch.cuda.set_device(hvd.local_rank())\n    torch.cuda.manual_seed(args.seed)\n\ncudnn.benchmark = True\n\n# If set > 0, will resume training from a given checkpoint.\nresume_from_epoch = 0\nfor try_epoch in range(args.epochs, 0, -1):\n    if os.path.exists(args.checkpoint_format.format(epoch=try_epoch)):\n        resume_from_epoch = try_epoch\n        break\n\n# Horovod: broadcast resume_from_epoch from rank 0 (which will have\n# checkpoints) to other ranks.\nresume_from_epoch = hvd.broadcast(torch.tensor(resume_from_epoch), root_rank=0,\n                                  name='resume_from_epoch').item()\n\n# Horovod: print logs on the first worker.\nverbose = 1 if hvd.rank() == 0 else 0\n\n# Horovod: write TensorBoard logs on first worker.\nlog_writer = tensorboardX.SummaryWriter(args.log_dir) if hvd.rank() == 0 else None\n\n\nkwargs = {'num_workers': 4, 'pin_memory': True} if args.cuda else {}\ntrain_dataset = \\\n    datasets.ImageFolder(args.train_dir,\n                         transform=transforms.Compose([\n                             transforms.RandomResizedCrop(224),\n                             transforms.RandomHorizontalFlip(),\n                             transforms.ToTensor(),\n                             transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                  std=[0.229, 0.224, 0.225])\n                         ]))\n# Horovod: use DistributedSampler to partition data among workers. Manually specify\n# `num_replicas=hvd.size()` and `rank=hvd.rank()`.\ntrain_sampler = torch.utils.data.distributed.DistributedSampler(\n    train_dataset, num_replicas=hvd.size(), rank=hvd.rank())\ntrain_loader = torch.utils.data.DataLoader(\n    train_dataset, batch_size=allreduce_batch_size,\n    sampler=train_sampler, **kwargs)\n\nval_dataset = \\\n    datasets.ImageFolder(args.val_dir,\n                         transform=transforms.Compose([\n                             transforms.Resize(256),\n                             transforms.CenterCrop(224),\n                             transforms.ToTensor(),\n                             transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                  std=[0.229, 0.224, 0.225])\n                         ]))\nval_sampler = torch.utils.data.distributed.DistributedSampler(\n    val_dataset, num_replicas=hvd.size(), rank=hvd.rank())\nval_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size,\n                                         sampler=val_sampler, **kwargs)\n\n\n# Set up standard ResNet-50 model.\nmodel = models.resnet50()\n\nif args.cuda:\n    # Move model to GPU.\n    model.cuda()\n\n# Horovod: scale learning rate by the number of GPUs.\n# Gradient Accumulation: scale learning rate by batches_per_allreduce\noptimizer = optim.SGD(model.parameters(),\n                      lr=(args.base_lr *\n                          args.batches_per_allreduce * hvd.size()),\n                      momentum=args.momentum, weight_decay=args.wd)\n\n# Horovod: (optional) compression algorithm.\ncompression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none\n\n# Horovod: wrap optimizer with DistributedOptimizer.\noptimizer = hvd.DistributedOptimizer(\n    optimizer, named_parameters=model.named_parameters(),\n    compression=compression,\n    backward_passes_per_step=args.batches_per_allreduce)\n\n# Restore from a previous checkpoint, if initial_epoch is specified.\n# Horovod: restore on the first worker which will broadcast weights to other workers.\nif resume_from_epoch > 0 and hvd.rank() == 0:\n    filepath = args.checkpoint_format.format(epoch=resume_from_epoch)\n    checkpoint = torch.load(filepath)\n    model.load_state_dict(checkpoint['model'])\n    optimizer.load_state_dict(checkpoint['optimizer'])\n\n# Horovod: broadcast parameters & optimizer state.\nhvd.broadcast_parameters(model.state_dict(), root_rank=0)\nhvd.broadcast_optimizer_state(optimizer, root_rank=0)\n\ndef train(epoch):\n    model.train()\n    train_sampler.set_epoch(epoch)\n    train_loss = Metric('train_loss')\n    train_accuracy = Metric('train_accuracy')\n\n    with tqdm(total=len(train_loader),\n              desc='Train Epoch     #{}'.format(epoch + 1),\n              disable=not verbose) as t:\n        for batch_idx, (data, target) in enumerate(train_loader):\n            adjust_learning_rate(epoch, batch_idx)\n\n            if args.cuda:\n                data, target = data.cuda(), target.cuda()\n            optimizer.zero_grad()\n            # Split data into sub-batches of size batch_size\n            for i in range(0, len(data), args.batch_size):\n                data_batch = data[i:i + args.batch_size]\n                target_batch = target[i:i + args.batch_size]\n                output = model(data_batch)\n                train_accuracy.update(accuracy(output, target_batch))\n                loss = F.cross_entropy(output, target_batch)\n                train_loss.update(loss.item())\n                # Average gradients among sub-batches\n                loss.div_(math.ceil(float(len(data)) / args.batch_size))\n                loss.backward()\n            # Gradient is applied across all ranks\n            optimizer.step()\n            t.set_postfix({'loss': train_loss.avg.item(),\n                           'accuracy': 100. * train_accuracy.avg.item()})\n            t.update(1)\n\n    if log_writer:\n        log_writer.add_scalar('train/loss', train_loss.avg, epoch)\n        log_writer.add_scalar('train/accuracy', train_accuracy.avg, epoch)\n\n\ndef validate(epoch):\n    model.eval()\n    val_loss = Metric('val_loss')\n    val_accuracy = Metric('val_accuracy')\n\n    with tqdm(total=len(val_loader),\n              desc='Validate Epoch  #{}'.format(epoch + 1),\n              disable=not verbose) as t:\n        with torch.no_grad():\n            for data, target in val_loader:\n                if args.cuda:\n                    data, target = data.cuda(), target.cuda()\n                output = model(data)\n\n                val_loss.update(F.cross_entropy(output, target))\n                val_accuracy.update(accuracy(output, target))\n                t.set_postfix({'loss': val_loss.avg.item(),\n                               'accuracy': 100. * val_accuracy.avg.item()})\n                t.update(1)\n\n    if log_writer:\n        log_writer.add_scalar('val/loss', val_loss.avg, epoch)\n        log_writer.add_scalar('val/accuracy', val_accuracy.avg, epoch)\n\n\n# Horovod: using `lr = base_lr * hvd.size()` from the very beginning leads to worse final\n# accuracy. Scale the learning rate `lr = base_lr` ---> `lr = base_lr * hvd.size()` during\n# the first five epochs. See https://arxiv.org/abs/1706.02677 for details.\n# After the warmup reduce learning rate by 10 on the 30th, 60th and 80th epochs.\ndef adjust_learning_rate(epoch, batch_idx):\n    if epoch < args.warmup_epochs:\n        epoch += float(batch_idx + 1) / len(train_loader)\n        lr_adj = 1. / hvd.size() * (epoch * (hvd.size() - 1) / args.warmup_epochs + 1)\n    elif epoch < 30:\n        lr_adj = 1.\n    elif epoch < 60:\n        lr_adj = 1e-1\n    elif epoch < 80:\n        lr_adj = 1e-2\n    else:\n        lr_adj = 1e-3\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = args.base_lr * hvd.size() * args.batches_per_allreduce * lr_adj\n\n\ndef accuracy(output, target):\n    # get the index of the max log-probability\n    pred = output.max(1, keepdim=True)[1]\n    return pred.eq(target.view_as(pred)).cpu().float().mean()\n\n\ndef save_checkpoint(epoch):\n    if hvd.rank() == 0:\n        filepath = args.checkpoint_format.format(epoch=epoch + 1)\n        state = {\n            'model': model.state_dict(),\n            'optimizer': optimizer.state_dict(),\n        }\n        torch.save(state, filepath)\n\n\n# Horovod: average metrics from distributed training.\nclass Metric(object):\n    def __init__(self, name):\n        self.name = name\n        self.sum = torch.tensor(0.)\n        self.n = torch.tensor(0.)\n\n    def update(self, val):\n        self.sum += hvd.allreduce(val.detach().cpu(), name=self.name)\n        self.n += 1\n\n    @property\n    def avg(self):\n        return self.sum / self.n\n\n\nfor epoch in range(resume_from_epoch, args.epochs):\n    train(epoch)\n    validate(epoch)\n    save_checkpoint(epoch)\n", "path": "examples/pytorch_imagenet_resnet50.py"}], "after_files": [{"content": "from __future__ import print_function\n\nimport torch\nimport argparse\nimport torch.backends.cudnn as cudnn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data.distributed\nfrom torchvision import datasets, transforms, models\nimport horovod.torch as hvd\nimport tensorboardX\nimport os\nimport math\nfrom tqdm import tqdm\n\n# Training settings\nparser = argparse.ArgumentParser(description='PyTorch ImageNet Example',\n                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\nparser.add_argument('--train-dir', default=os.path.expanduser('~/imagenet/train'),\n                    help='path to training data')\nparser.add_argument('--val-dir', default=os.path.expanduser('~/imagenet/validation'),\n                    help='path to validation data')\nparser.add_argument('--log-dir', default='./logs',\n                    help='tensorboard log directory')\nparser.add_argument('--checkpoint-format', default='./checkpoint-{epoch}.pth.tar',\n                    help='checkpoint file format')\nparser.add_argument('--fp16-allreduce', action='store_true', default=False,\n                    help='use fp16 compression during allreduce')\nparser.add_argument('--batches-per-allreduce', type=int, default=1,\n                    help='number of batches processed locally before '\n                         'executing allreduce across workers; it multiplies '\n                         'total batch size.')\n\n# Default settings from https://arxiv.org/abs/1706.02677.\nparser.add_argument('--batch-size', type=int, default=32,\n                    help='input batch size for training')\nparser.add_argument('--val-batch-size', type=int, default=32,\n                    help='input batch size for validation')\nparser.add_argument('--epochs', type=int, default=90,\n                    help='number of epochs to train')\nparser.add_argument('--base-lr', type=float, default=0.0125,\n                    help='learning rate for a single GPU')\nparser.add_argument('--warmup-epochs', type=float, default=5,\n                    help='number of warmup epochs')\nparser.add_argument('--momentum', type=float, default=0.9,\n                    help='SGD momentum')\nparser.add_argument('--wd', type=float, default=0.00005,\n                    help='weight decay')\n\nparser.add_argument('--no-cuda', action='store_true', default=False,\n                    help='disables CUDA training')\nparser.add_argument('--seed', type=int, default=42,\n                    help='random seed')\n\nargs = parser.parse_args()\nargs.cuda = not args.no_cuda and torch.cuda.is_available()\n\nallreduce_batch_size = args.batch_size * args.batches_per_allreduce\n\nhvd.init()\ntorch.manual_seed(args.seed)\n\nif args.cuda:\n    # Horovod: pin GPU to local rank.\n    torch.cuda.set_device(hvd.local_rank())\n    torch.cuda.manual_seed(args.seed)\n\ncudnn.benchmark = True\n\n# If set > 0, will resume training from a given checkpoint.\nresume_from_epoch = 0\nfor try_epoch in range(args.epochs, 0, -1):\n    if os.path.exists(args.checkpoint_format.format(epoch=try_epoch)):\n        resume_from_epoch = try_epoch\n        break\n\n# Horovod: broadcast resume_from_epoch from rank 0 (which will have\n# checkpoints) to other ranks.\nresume_from_epoch = hvd.broadcast(torch.tensor(resume_from_epoch), root_rank=0,\n                                  name='resume_from_epoch').item()\n\n# Horovod: print logs on the first worker.\nverbose = 1 if hvd.rank() == 0 else 0\n\n# Horovod: write TensorBoard logs on first worker.\nlog_writer = tensorboardX.SummaryWriter(args.log_dir) if hvd.rank() == 0 else None\n\n\nkwargs = {'num_workers': 4, 'pin_memory': True} if args.cuda else {}\ntrain_dataset = \\\n    datasets.ImageFolder(args.train_dir,\n                         transform=transforms.Compose([\n                             transforms.RandomResizedCrop(224),\n                             transforms.RandomHorizontalFlip(),\n                             transforms.ToTensor(),\n                             transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                  std=[0.229, 0.224, 0.225])\n                         ]))\n# Horovod: use DistributedSampler to partition data among workers. Manually specify\n# `num_replicas=hvd.size()` and `rank=hvd.rank()`.\ntrain_sampler = torch.utils.data.distributed.DistributedSampler(\n    train_dataset, num_replicas=hvd.size(), rank=hvd.rank())\ntrain_loader = torch.utils.data.DataLoader(\n    train_dataset, batch_size=allreduce_batch_size,\n    sampler=train_sampler, **kwargs)\n\nval_dataset = \\\n    datasets.ImageFolder(args.val_dir,\n                         transform=transforms.Compose([\n                             transforms.Resize(256),\n                             transforms.CenterCrop(224),\n                             transforms.ToTensor(),\n                             transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                  std=[0.229, 0.224, 0.225])\n                         ]))\nval_sampler = torch.utils.data.distributed.DistributedSampler(\n    val_dataset, num_replicas=hvd.size(), rank=hvd.rank())\nval_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size,\n                                         sampler=val_sampler, **kwargs)\n\n\n# Set up standard ResNet-50 model.\nmodel = models.resnet50()\n\nif args.cuda:\n    # Move model to GPU.\n    model.cuda()\n\n# Horovod: scale learning rate by the number of GPUs.\n# Gradient Accumulation: scale learning rate by batches_per_allreduce\noptimizer = optim.SGD(model.parameters(),\n                      lr=(args.base_lr *\n                          args.batches_per_allreduce * hvd.size()),\n                      momentum=args.momentum, weight_decay=args.wd)\n\n# Horovod: (optional) compression algorithm.\ncompression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none\n\n# Horovod: wrap optimizer with DistributedOptimizer.\noptimizer = hvd.DistributedOptimizer(\n    optimizer, named_parameters=model.named_parameters(),\n    compression=compression,\n    backward_passes_per_step=args.batches_per_allreduce)\n\n# Restore from a previous checkpoint, if initial_epoch is specified.\n# Horovod: restore on the first worker which will broadcast weights to other workers.\nif resume_from_epoch > 0 and hvd.rank() == 0:\n    filepath = args.checkpoint_format.format(epoch=resume_from_epoch)\n    checkpoint = torch.load(filepath)\n    model.load_state_dict(checkpoint['model'])\n    optimizer.load_state_dict(checkpoint['optimizer'])\n\n# Horovod: broadcast parameters & optimizer state.\nhvd.broadcast_parameters(model.state_dict(), root_rank=0)\nhvd.broadcast_optimizer_state(optimizer, root_rank=0)\n\ndef train(epoch):\n    model.train()\n    train_sampler.set_epoch(epoch)\n    train_loss = Metric('train_loss')\n    train_accuracy = Metric('train_accuracy')\n\n    with tqdm(total=len(train_loader),\n              desc='Train Epoch     #{}'.format(epoch + 1),\n              disable=not verbose) as t:\n        for batch_idx, (data, target) in enumerate(train_loader):\n            adjust_learning_rate(epoch, batch_idx)\n\n            if args.cuda:\n                data, target = data.cuda(), target.cuda()\n            optimizer.zero_grad()\n            # Split data into sub-batches of size batch_size\n            for i in range(0, len(data), args.batch_size):\n                data_batch = data[i:i + args.batch_size]\n                target_batch = target[i:i + args.batch_size]\n                output = model(data_batch)\n                train_accuracy.update(accuracy(output, target_batch))\n                loss = F.cross_entropy(output, target_batch)\n                train_loss.update(loss)\n                # Average gradients among sub-batches\n                loss.div_(math.ceil(float(len(data)) / args.batch_size))\n                loss.backward()\n            # Gradient is applied across all ranks\n            optimizer.step()\n            t.set_postfix({'loss': train_loss.avg.item(),\n                           'accuracy': 100. * train_accuracy.avg.item()})\n            t.update(1)\n\n    if log_writer:\n        log_writer.add_scalar('train/loss', train_loss.avg, epoch)\n        log_writer.add_scalar('train/accuracy', train_accuracy.avg, epoch)\n\n\ndef validate(epoch):\n    model.eval()\n    val_loss = Metric('val_loss')\n    val_accuracy = Metric('val_accuracy')\n\n    with tqdm(total=len(val_loader),\n              desc='Validate Epoch  #{}'.format(epoch + 1),\n              disable=not verbose) as t:\n        with torch.no_grad():\n            for data, target in val_loader:\n                if args.cuda:\n                    data, target = data.cuda(), target.cuda()\n                output = model(data)\n\n                val_loss.update(F.cross_entropy(output, target))\n                val_accuracy.update(accuracy(output, target))\n                t.set_postfix({'loss': val_loss.avg.item(),\n                               'accuracy': 100. * val_accuracy.avg.item()})\n                t.update(1)\n\n    if log_writer:\n        log_writer.add_scalar('val/loss', val_loss.avg, epoch)\n        log_writer.add_scalar('val/accuracy', val_accuracy.avg, epoch)\n\n\n# Horovod: using `lr = base_lr * hvd.size()` from the very beginning leads to worse final\n# accuracy. Scale the learning rate `lr = base_lr` ---> `lr = base_lr * hvd.size()` during\n# the first five epochs. See https://arxiv.org/abs/1706.02677 for details.\n# After the warmup reduce learning rate by 10 on the 30th, 60th and 80th epochs.\ndef adjust_learning_rate(epoch, batch_idx):\n    if epoch < args.warmup_epochs:\n        epoch += float(batch_idx + 1) / len(train_loader)\n        lr_adj = 1. / hvd.size() * (epoch * (hvd.size() - 1) / args.warmup_epochs + 1)\n    elif epoch < 30:\n        lr_adj = 1.\n    elif epoch < 60:\n        lr_adj = 1e-1\n    elif epoch < 80:\n        lr_adj = 1e-2\n    else:\n        lr_adj = 1e-3\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = args.base_lr * hvd.size() * args.batches_per_allreduce * lr_adj\n\n\ndef accuracy(output, target):\n    # get the index of the max log-probability\n    pred = output.max(1, keepdim=True)[1]\n    return pred.eq(target.view_as(pred)).cpu().float().mean()\n\n\ndef save_checkpoint(epoch):\n    if hvd.rank() == 0:\n        filepath = args.checkpoint_format.format(epoch=epoch + 1)\n        state = {\n            'model': model.state_dict(),\n            'optimizer': optimizer.state_dict(),\n        }\n        torch.save(state, filepath)\n\n\n# Horovod: average metrics from distributed training.\nclass Metric(object):\n    def __init__(self, name):\n        self.name = name\n        self.sum = torch.tensor(0.)\n        self.n = torch.tensor(0.)\n\n    def update(self, val):\n        self.sum += hvd.allreduce(val.detach().cpu(), name=self.name)\n        self.n += 1\n\n    @property\n    def avg(self):\n        return self.sum / self.n\n\n\nfor epoch in range(resume_from_epoch, args.epochs):\n    train(epoch)\n    validate(epoch)\n    save_checkpoint(epoch)\n", "path": "examples/pytorch_imagenet_resnet50.py"}]}