We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Performance drop when activating gradient clipping
Hello all,

I experienced a substantial drop in computation time when activating gradient clipping (by passing a non-zero value to the keyword argument `gradient_clip_val` when initializing the Trainer).

I noticed that in the current implementation of the `clipping_gradient` method in pytorch-lightning/trainer/training_tricks.py redundant computations are made by first computing the 2-norm and second squaring this result, which could be shortened by computing the sum of squares directly. This saves one square root and squaring operation per parameter set.

Best,
Jonas

### Environment

```
cuda:
	GPU:
	available:           False
	version:             None
packages:
	numpy:               1.18.1
	pyTorch_debug:       False
	pyTorch_version:     1.4.0
	pytorch-lightning:   0.7.4-dev
	tensorboard:         2.2.1
	tqdm:                4.45.0
system:
	OS:                  Darwin
	architecture:
		64bit
		
	processor:           i386
	python:              3.8.2
	version:             Darwin Kernel Version 19.4.0: Wed Mar  4 22:28:40 PST 2020; root:xnu-6153.101.6~15/RELEASE_X86_64
```

### Additional context

I trained a relatively small (two-layered) MLP on MNIST; perhaps this performance drop does not become that apparent when training on larger network architectures.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/trainer/training_tricks.py`
Content:
```
1 import math
2 import sys
3 from abc import ABC, abstractmethod
4 
5 import torch
6 from torch import Tensor
7 
8 from pytorch_lightning import _logger as log
9 from pytorch_lightning.callbacks import GradientAccumulationScheduler
10 
11 EPSILON = 1e-6
12 EPSILON_FP16 = 1e-5
13 
14 
15 class TrainerTrainingTricksMixin(ABC):
16 
17     # this is just a summary on variables used in this abstract class,
18     #  the proper values/initialisation should be done in child class
19     gradient_clip_val: ...
20     precision: ...
21 
22     @abstractmethod
23     def get_model(self):
24         """Warning: this is just empty shell for code implemented in other class."""
25 
26     def clip_gradients(self):
27         # this code is a modification of torch.nn.utils.clip_grad_norm_
28         # with TPU support based on https://github.com/pytorch/xla/blob/master/TROUBLESHOOTING.md
29         if self.gradient_clip_val > 0:
30             model = self.get_model()
31             parameters = model.parameters()
32             max_norm = float(self.gradient_clip_val)
33             norm_type = float(2.0)
34             if isinstance(parameters, torch.Tensor):
35                 parameters = [parameters]
36             parameters = list(filter(lambda p: p.grad is not None, parameters))
37             if norm_type == math.inf:
38                 total_norm = max(p.grad.data.abs().max() for p in parameters)
39             else:
40                 device = parameters[0].device
41                 total_norm = torch.zeros([], device=device if parameters else None)
42                 for p in parameters:
43                     param_norm = p.grad.data.norm(norm_type) ** norm_type
44                     total_norm.add_(param_norm)
45                 total_norm = (total_norm ** (1. / norm_type))
46             eps = EPSILON_FP16 if self.precision == 16 else EPSILON
47             clip_coef = torch.tensor(max_norm, device=device) / (total_norm + eps)
48             for p in parameters:
49                 p.grad.data.mul_(torch.where(clip_coef < 1, clip_coef, torch.tensor(1., device=device)))
50 
51     def print_nan_gradients(self) -> None:
52         model = self.get_model()
53         for param in model.parameters():
54             if (param.grad is not None) and torch.isnan(param.grad.float()).any():
55                 log.info(param, param.grad)
56 
57     def detect_nan_tensors(self, loss: Tensor) -> None:
58         model = self.get_model()
59 
60         # check if loss is nan
61         if not torch.isfinite(loss).all():
62             raise ValueError(
63                 'The loss returned in `training_step` is nan or inf.'
64             )
65         # check if a network weight is nan
66         for name, param in model.named_parameters():
67             if not torch.isfinite(param).all():
68                 self.print_nan_gradients()
69                 raise ValueError(
70                     f'Detected nan and/or inf values in `{name}`.'
71                     ' Check your forward pass for numerically unstable operations.'
72                 )
73 
74     def configure_accumulated_gradients(self, accumulate_grad_batches):
75         if isinstance(accumulate_grad_batches, dict):
76             self.accumulation_scheduler = GradientAccumulationScheduler(accumulate_grad_batches)
77         elif isinstance(accumulate_grad_batches, int):
78             schedule = {1: accumulate_grad_batches}
79             self.accumulation_scheduler = GradientAccumulationScheduler(schedule)
80         else:
81             raise TypeError("Gradient accumulation supports only int and dict types")
82
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/trainer/training_tricks.py b/pytorch_lightning/trainer/training_tricks.py
--- a/pytorch_lightning/trainer/training_tricks.py
+++ b/pytorch_lightning/trainer/training_tricks.py
@@ -40,7 +40,7 @@
                 device = parameters[0].device
                 total_norm = torch.zeros([], device=device if parameters else None)
                 for p in parameters:
-                    param_norm = p.grad.data.norm(norm_type) ** norm_type
+                    param_norm = p.grad.data.pow(norm_type).sum()
                     total_norm.add_(param_norm)
                 total_norm = (total_norm ** (1. / norm_type))
             eps = EPSILON_FP16 if self.precision == 16 else EPSILON

{"golden_diff": "diff --git a/pytorch_lightning/trainer/training_tricks.py b/pytorch_lightning/trainer/training_tricks.py\n--- a/pytorch_lightning/trainer/training_tricks.py\n+++ b/pytorch_lightning/trainer/training_tricks.py\n@@ -40,7 +40,7 @@\n                 device = parameters[0].device\n                 total_norm = torch.zeros([], device=device if parameters else None)\n                 for p in parameters:\n-                    param_norm = p.grad.data.norm(norm_type) ** norm_type\n+                    param_norm = p.grad.data.pow(norm_type).sum()\n                     total_norm.add_(param_norm)\n                 total_norm = (total_norm ** (1. / norm_type))\n             eps = EPSILON_FP16 if self.precision == 16 else EPSILON\n", "issue": "Performance drop when activating gradient clipping\nHello all,\r\n\r\nI experienced a substantial drop in computation time when activating gradient clipping (by passing a non-zero value to the keyword argument `gradient_clip_val` when initializing the Trainer).\r\n\r\nI noticed that in the current implementation of the `clipping_gradient` method in pytorch-lightning/trainer/training_tricks.py redundant computations are made by first computing the 2-norm and second squaring this result, which could be shortened by computing the sum of squares directly. This saves one square root and squaring operation per parameter set.\r\n\r\nBest,\r\nJonas\r\n\r\n### Environment\r\n\r\n```\r\ncuda:\r\n\tGPU:\r\n\tavailable:           False\r\n\tversion:             None\r\npackages:\r\n\tnumpy:               1.18.1\r\n\tpyTorch_debug:       False\r\n\tpyTorch_version:     1.4.0\r\n\tpytorch-lightning:   0.7.4-dev\r\n\ttensorboard:         2.2.1\r\n\ttqdm:                4.45.0\r\nsystem:\r\n\tOS:                  Darwin\r\n\tarchitecture:\r\n\t\t64bit\r\n\t\t\r\n\tprocessor:           i386\r\n\tpython:              3.8.2\r\n\tversion:             Darwin Kernel Version 19.4.0: Wed Mar  4 22:28:40 PST 2020; root:xnu-6153.101.6~15/RELEASE_X86_64\r\n```\r\n\r\n### Additional context\r\n\r\nI trained a relatively small (two-layered) MLP on MNIST; perhaps this performance drop does not become that apparent when training on larger network architectures.\n", "before_files": [{"content": "import math\nimport sys\nfrom abc import ABC, abstractmethod\n\nimport torch\nfrom torch import Tensor\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.callbacks import GradientAccumulationScheduler\n\nEPSILON = 1e-6\nEPSILON_FP16 = 1e-5\n\n\nclass TrainerTrainingTricksMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    gradient_clip_val: ...\n    precision: ...\n\n    @abstractmethod\n    def get_model(self):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def clip_gradients(self):\n        # this code is a modification of torch.nn.utils.clip_grad_norm_\n        # with TPU support based on https://github.com/pytorch/xla/blob/master/TROUBLESHOOTING.md\n        if self.gradient_clip_val > 0:\n            model = self.get_model()\n            parameters = model.parameters()\n            max_norm = float(self.gradient_clip_val)\n            norm_type = float(2.0)\n            if isinstance(parameters, torch.Tensor):\n                parameters = [parameters]\n            parameters = list(filter(lambda p: p.grad is not None, parameters))\n            if norm_type == math.inf:\n                total_norm = max(p.grad.data.abs().max() for p in parameters)\n            else:\n                device = parameters[0].device\n                total_norm = torch.zeros([], device=device if parameters else None)\n                for p in parameters:\n                    param_norm = p.grad.data.norm(norm_type) ** norm_type\n                    total_norm.add_(param_norm)\n                total_norm = (total_norm ** (1. / norm_type))\n            eps = EPSILON_FP16 if self.precision == 16 else EPSILON\n            clip_coef = torch.tensor(max_norm, device=device) / (total_norm + eps)\n            for p in parameters:\n                p.grad.data.mul_(torch.where(clip_coef < 1, clip_coef, torch.tensor(1., device=device)))\n\n    def print_nan_gradients(self) -> None:\n        model = self.get_model()\n        for param in model.parameters():\n            if (param.grad is not None) and torch.isnan(param.grad.float()).any():\n                log.info(param, param.grad)\n\n    def detect_nan_tensors(self, loss: Tensor) -> None:\n        model = self.get_model()\n\n        # check if loss is nan\n        if not torch.isfinite(loss).all():\n            raise ValueError(\n                'The loss returned in `training_step` is nan or inf.'\n            )\n        # check if a network weight is nan\n        for name, param in model.named_parameters():\n            if not torch.isfinite(param).all():\n                self.print_nan_gradients()\n                raise ValueError(\n                    f'Detected nan and/or inf values in `{name}`.'\n                    ' Check your forward pass for numerically unstable operations.'\n                )\n\n    def configure_accumulated_gradients(self, accumulate_grad_batches):\n        if isinstance(accumulate_grad_batches, dict):\n            self.accumulation_scheduler = GradientAccumulationScheduler(accumulate_grad_batches)\n        elif isinstance(accumulate_grad_batches, int):\n            schedule = {1: accumulate_grad_batches}\n            self.accumulation_scheduler = GradientAccumulationScheduler(schedule)\n        else:\n            raise TypeError(\"Gradient accumulation supports only int and dict types\")\n", "path": "pytorch_lightning/trainer/training_tricks.py"}], "after_files": [{"content": "import math\nimport sys\nfrom abc import ABC, abstractmethod\n\nimport torch\nfrom torch import Tensor\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.callbacks import GradientAccumulationScheduler\n\nEPSILON = 1e-6\nEPSILON_FP16 = 1e-5\n\n\nclass TrainerTrainingTricksMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    gradient_clip_val: ...\n    precision: ...\n\n    @abstractmethod\n    def get_model(self):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def clip_gradients(self):\n        # this code is a modification of torch.nn.utils.clip_grad_norm_\n        # with TPU support based on https://github.com/pytorch/xla/blob/master/TROUBLESHOOTING.md\n        if self.gradient_clip_val > 0:\n            model = self.get_model()\n            parameters = model.parameters()\n            max_norm = float(self.gradient_clip_val)\n            norm_type = float(2.0)\n            if isinstance(parameters, torch.Tensor):\n                parameters = [parameters]\n            parameters = list(filter(lambda p: p.grad is not None, parameters))\n            if norm_type == math.inf:\n                total_norm = max(p.grad.data.abs().max() for p in parameters)\n            else:\n                device = parameters[0].device\n                total_norm = torch.zeros([], device=device if parameters else None)\n                for p in parameters:\n                    param_norm = p.grad.data.pow(norm_type).sum()\n                    total_norm.add_(param_norm)\n                total_norm = (total_norm ** (1. / norm_type))\n            eps = EPSILON_FP16 if self.precision == 16 else EPSILON\n            clip_coef = torch.tensor(max_norm, device=device) / (total_norm + eps)\n            for p in parameters:\n                p.grad.data.mul_(torch.where(clip_coef < 1, clip_coef, torch.tensor(1., device=device)))\n\n    def print_nan_gradients(self) -> None:\n        model = self.get_model()\n        for param in model.parameters():\n            if (param.grad is not None) and torch.isnan(param.grad.float()).any():\n                log.info(param, param.grad)\n\n    def detect_nan_tensors(self, loss: Tensor) -> None:\n        model = self.get_model()\n\n        # check if loss is nan\n        if not torch.isfinite(loss).all():\n            raise ValueError(\n                'The loss returned in `training_step` is nan or inf.'\n            )\n        # check if a network weight is nan\n        for name, param in model.named_parameters():\n            if not torch.isfinite(param).all():\n                self.print_nan_gradients()\n                raise ValueError(\n                    f'Detected nan and/or inf values in `{name}`.'\n                    ' Check your forward pass for numerically unstable operations.'\n                )\n\n    def configure_accumulated_gradients(self, accumulate_grad_batches):\n        if isinstance(accumulate_grad_batches, dict):\n            self.accumulation_scheduler = GradientAccumulationScheduler(accumulate_grad_batches)\n        elif isinstance(accumulate_grad_batches, int):\n            schedule = {1: accumulate_grad_batches}\n            self.accumulation_scheduler = GradientAccumulationScheduler(schedule)\n        else:\n            raise TypeError(\"Gradient accumulation supports only int and dict types\")\n", "path": "pytorch_lightning/trainer/training_tricks.py"}]}