We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Incorrect default cuda device when using single gpu other than cuda:0
## 🐛 Bug

The default `cuda` is not set properly to the `trainer.root_gpu` in single-GPU mode. The tensors created with `device='cuda'` will be placed on the incorrect gpu, and the dataloader will acquire memory on the incorrect gpu when `pin_memory=True`.

Maybe we'll need to add
`torch.cuda.set_device(self.trainer.root_gpu)` to https://github.com/PyTorchLightning/pytorch-lightning/blob/5dfc7b157e7febab692036b7392dac8b52f41b87/pytorch_lightning/accelerators/gpu_backend.py#L24
as `DDPBackend` did:

https://github.com/PyTorchLightning/pytorch-lightning/blob/5dfc7b157e7febab692036b7392dac8b52f41b87/pytorch_lightning/accelerators/ddp_backend.py#L195

### To Reproduce

Running the following code will get 

`RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cuda:0!`

#### Code sample

```python
import pytorch_lightning as pl
import torch
from torch import nn
from torch.utils import data


class Dataset(data.Dataset):

  def __getitem__(self, item):
    return torch.zeros(1)

  def __len__(self):
    return 5


class Model(pl.LightningModule):

  def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)
    self.x = nn.Parameter(torch.zeros(1))

  def forward(self, *args, **kwargs):
    return self.x

  def training_step(self, *args, **kwargs):
    return self.x + torch.zeros(1, device='cuda')  # RuntimeError.

  def train_dataloader(self):
    return data.DataLoader(Dataset(), num_workers=1, pin_memory=True)

  def configure_optimizers(self):
    return torch.optim.SGD(self.parameters(), 1.0)


if __name__ == '__main__':
  trainer = pl.Trainer(gpus=[1], num_sanity_val_steps=0, max_epochs=1)
  model = Model()
  trainer.fit(model)

```

### Expected behavior

No `RuntimeError` occurs.

### Environment

* CUDA:
	- GPU:
	- available:
	- version:
* Packages:
	- numpy:             1.18.5
	- pyTorch_debug:     False
	- pyTorch_version:   1.6.0
	- pytorch-lightning: 0.9.0rc16
	- tensorboard:       2.3.0
	- tqdm:              4.48.2
* System:
	- OS:                Windows
	- architecture:
		- 64bit
		- WindowsPE
	- processor:
	- python:            3.7.3
	- version:           10.0.18362

### Additional context

<!-- Add any other context about the problem here. -->


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/accelerators/gpu_backend.py`
Content:
```
1 # Copyright The PyTorch Lightning team.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from pytorch_lightning.core import LightningModule
16 from pytorch_lightning.utilities import AMPType
17 
18 try:
19     from apex import amp
20 except ImportError:
21     amp = None
22 
23 
24 class GPUBackend(object):
25     amp_backend: AMPType
26 
27     def __init__(self, trainer):
28         self.trainer = trainer
29 
30     def setup(self, model):
31 
32         # call setup
33         self.trainer.call_setup_hook(model)
34 
35         model.cuda(self.trainer.root_gpu)
36 
37         # CHOOSE OPTIMIZER
38         # allow for lr schedulers as well
39         optimizers, lr_schedulers, optimizer_frequencies = self.trainer.init_optimizers(model)
40         self.trainer.optimizers = optimizers
41         self.trainer.lr_schedulers = lr_schedulers
42         self.trainer.optimizer_frequencies = optimizer_frequencies
43 
44         if self.trainer.amp_backend == AMPType.APEX:
45             model = self._setup_nvidia_apex(model)
46         return model
47 
48     def train(self, model):
49         results = self.trainer.run_pretrain_routine(model)
50         return results
51 
52     def _setup_nvidia_apex(self, model: LightningModule):
53         model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
54         self.trainer.optimizers = optimizers
55         self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)
56         return model
57
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/accelerators/gpu_backend.py b/pytorch_lightning/accelerators/gpu_backend.py
--- a/pytorch_lightning/accelerators/gpu_backend.py
+++ b/pytorch_lightning/accelerators/gpu_backend.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import torch
 from pytorch_lightning.core import LightningModule
 from pytorch_lightning.utilities import AMPType
 
@@ -32,6 +33,7 @@
         # call setup
         self.trainer.call_setup_hook(model)
 
+        torch.cuda.set_device(self.trainer.root_gpu)
         model.cuda(self.trainer.root_gpu)
 
         # CHOOSE OPTIMIZER

{"golden_diff": "diff --git a/pytorch_lightning/accelerators/gpu_backend.py b/pytorch_lightning/accelerators/gpu_backend.py\n--- a/pytorch_lightning/accelerators/gpu_backend.py\n+++ b/pytorch_lightning/accelerators/gpu_backend.py\n@@ -12,6 +12,7 @@\n # See the License for the specific language governing permissions and\n # limitations under the License.\n \n+import torch\n from pytorch_lightning.core import LightningModule\n from pytorch_lightning.utilities import AMPType\n \n@@ -32,6 +33,7 @@\n         # call setup\n         self.trainer.call_setup_hook(model)\n \n+        torch.cuda.set_device(self.trainer.root_gpu)\n         model.cuda(self.trainer.root_gpu)\n \n         # CHOOSE OPTIMIZER\n", "issue": "Incorrect default cuda device when using single gpu other than cuda:0\n## \ud83d\udc1b Bug\r\n\r\nThe default `cuda` is not set properly to the `trainer.root_gpu` in single-GPU mode. The tensors created with `device='cuda'` will be placed on the incorrect gpu, and the dataloader will acquire memory on the incorrect gpu when `pin_memory=True`.\r\n\r\nMaybe we'll need to add\r\n`torch.cuda.set_device(self.trainer.root_gpu)` to https://github.com/PyTorchLightning/pytorch-lightning/blob/5dfc7b157e7febab692036b7392dac8b52f41b87/pytorch_lightning/accelerators/gpu_backend.py#L24\r\nas `DDPBackend` did:\r\n\r\nhttps://github.com/PyTorchLightning/pytorch-lightning/blob/5dfc7b157e7febab692036b7392dac8b52f41b87/pytorch_lightning/accelerators/ddp_backend.py#L195\r\n\r\n### To Reproduce\r\n\r\nRunning the following code will get \r\n\r\n`RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cuda:0!`\r\n\r\n#### Code sample\r\n\r\n```python\r\nimport pytorch_lightning as pl\r\nimport torch\r\nfrom torch import nn\r\nfrom torch.utils import data\r\n\r\n\r\nclass Dataset(data.Dataset):\r\n\r\n  def __getitem__(self, item):\r\n    return torch.zeros(1)\r\n\r\n  def __len__(self):\r\n    return 5\r\n\r\n\r\nclass Model(pl.LightningModule):\r\n\r\n  def __init__(self, *args, **kwargs):\r\n    super().__init__(*args, **kwargs)\r\n    self.x = nn.Parameter(torch.zeros(1))\r\n\r\n  def forward(self, *args, **kwargs):\r\n    return self.x\r\n\r\n  def training_step(self, *args, **kwargs):\r\n    return self.x + torch.zeros(1, device='cuda')  # RuntimeError.\r\n\r\n  def train_dataloader(self):\r\n    return data.DataLoader(Dataset(), num_workers=1, pin_memory=True)\r\n\r\n  def configure_optimizers(self):\r\n    return torch.optim.SGD(self.parameters(), 1.0)\r\n\r\n\r\nif __name__ == '__main__':\r\n  trainer = pl.Trainer(gpus=[1], num_sanity_val_steps=0, max_epochs=1)\r\n  model = Model()\r\n  trainer.fit(model)\r\n\r\n```\r\n\r\n### Expected behavior\r\n\r\nNo `RuntimeError` occurs.\r\n\r\n### Environment\r\n\r\n* CUDA:\r\n\t- GPU:\r\n\t- available:\r\n\t- version:\r\n* Packages:\r\n\t- numpy:             1.18.5\r\n\t- pyTorch_debug:     False\r\n\t- pyTorch_version:   1.6.0\r\n\t- pytorch-lightning: 0.9.0rc16\r\n\t- tensorboard:       2.3.0\r\n\t- tqdm:              4.48.2\r\n* System:\r\n\t- OS:                Windows\r\n\t- architecture:\r\n\t\t- 64bit\r\n\t\t- WindowsPE\r\n\t- processor:\r\n\t- python:            3.7.3\r\n\t- version:           10.0.18362\r\n\r\n### Additional context\r\n\r\n<!-- Add any other context about the problem here. -->\r\n\n", "before_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.utilities import AMPType\n\ntry:\n    from apex import amp\nexcept ImportError:\n    amp = None\n\n\nclass GPUBackend(object):\n    amp_backend: AMPType\n\n    def __init__(self, trainer):\n        self.trainer = trainer\n\n    def setup(self, model):\n\n        # call setup\n        self.trainer.call_setup_hook(model)\n\n        model.cuda(self.trainer.root_gpu)\n\n        # CHOOSE OPTIMIZER\n        # allow for lr schedulers as well\n        optimizers, lr_schedulers, optimizer_frequencies = self.trainer.init_optimizers(model)\n        self.trainer.optimizers = optimizers\n        self.trainer.lr_schedulers = lr_schedulers\n        self.trainer.optimizer_frequencies = optimizer_frequencies\n\n        if self.trainer.amp_backend == AMPType.APEX:\n            model = self._setup_nvidia_apex(model)\n        return model\n\n    def train(self, model):\n        results = self.trainer.run_pretrain_routine(model)\n        return results\n\n    def _setup_nvidia_apex(self, model: LightningModule):\n        model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)\n        self.trainer.optimizers = optimizers\n        self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)\n        return model\n", "path": "pytorch_lightning/accelerators/gpu_backend.py"}], "after_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport torch\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.utilities import AMPType\n\ntry:\n    from apex import amp\nexcept ImportError:\n    amp = None\n\n\nclass GPUBackend(object):\n    amp_backend: AMPType\n\n    def __init__(self, trainer):\n        self.trainer = trainer\n\n    def setup(self, model):\n\n        # call setup\n        self.trainer.call_setup_hook(model)\n\n        torch.cuda.set_device(self.trainer.root_gpu)\n        model.cuda(self.trainer.root_gpu)\n\n        # CHOOSE OPTIMIZER\n        # allow for lr schedulers as well\n        optimizers, lr_schedulers, optimizer_frequencies = self.trainer.init_optimizers(model)\n        self.trainer.optimizers = optimizers\n        self.trainer.lr_schedulers = lr_schedulers\n        self.trainer.optimizer_frequencies = optimizer_frequencies\n\n        if self.trainer.amp_backend == AMPType.APEX:\n            model = self._setup_nvidia_apex(model)\n        return model\n\n    def train(self, model):\n        results = self.trainer.run_pretrain_routine(model)\n        return results\n\n    def _setup_nvidia_apex(self, model: LightningModule):\n        model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)\n        self.trainer.optimizers = optimizers\n        self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)\n        return model\n", "path": "pytorch_lightning/accelerators/gpu_backend.py"}]}