We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[Bug] TypeError when calling `backward` on `gpytorch.functions.logdet` 
# 🐛 Bug

I think there's a missing `list()` in https://github.com/cornellius-gp/gpytorch/blob/master/gpytorch/functions/_inv_quad_log_det.py#L221. I'm not super familiar with gpytorch internals so hopefully this is correct -- if so, happy to contribute the one-liner fix. 

## To reproduce

** Code snippet to reproduce **
```python
### works (I'm guessing something dispatches elsewhere for small matrices?)
import torch
from torch.autograd import backward
import gpytorch
from gpytorch.functions import logdet, inv_matmul
n = 100
inp = torch.arange(n, dtype=torch.float)
kern = gpytorch.kernels.RBFKernel()(inp)
ld = logdet(kern)
backward(ld)

### doesn't work
import torch
from torch.autograd import backward
import gpytorch
from gpytorch.functions import logdet, inv_matmul
n = 1000
inp = torch.arange(n, dtype=torch.float)
kern = gpytorch.kernels.RBFKernel()(inp)
ld = logdet(kern)
backward(ld)
```

** Stack trace/error message **
```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-46-593fbced29ac> in <module>()
      3 kern = gpytorch.kernels.RBFKernel()(inp)
      4 ld = logdet(kern)
----> 5 backward(ld)

<PATH SNIPPED>/lib/python3.7/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
     91     Variable._execution_engine.run_backward(
     92         tensors, grad_tensors, retain_graph, create_graph,
---> 93         allow_unreachable=True)  # allow_unreachable flag
     94 
     95 

<PATH SNIPPED>/lib/python3.7/site-packages/torch/autograd/function.py in apply(self, *args)
     75 
     76     def apply(self, *args):
---> 77         return self._forward_cls.backward(self, *args)
     78 
     79 

<PATH SNIPPED>lib/python3.7/site-packages/gpytorch/functions/_inv_quad_log_det.py in backward(ctx, inv_quad_grad_output, logdet_grad_output)
    221             res = matrix_arg_grads
    222 
--> 223         return tuple([None] * 9 + res)

TypeError: can only concatenate list (not "tuple") to list
```

## Expected Behavior

No error. 

## System information

**Please complete the following information:**
- GPyTorch version: 0.3.2 <!-- GPyTorch Version (run `print(gpytorch.__version__)` -->
- pytorch version 1.1.0. 
- Mac OSX. 



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gpytorch/functions/_inv_quad_log_det.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 import torch
4 from torch.autograd import Function
5 from ..utils.lanczos import lanczos_tridiag_to_diag
6 from ..utils.stochastic_lq import StochasticLQ
7 from .. import settings
8 
9 
10 class InvQuadLogDet(Function):
11     """
12     Given a PSD matrix A (or a batch of PSD matrices A), this function computes one or both
13     of the following
14     - The matrix solves A^{-1} b
15     - logdet(A)
16     """
17     @staticmethod
18     def forward(
19         ctx,
20         representation_tree,
21         dtype,
22         device,
23         matrix_shape,
24         batch_shape=torch.Size(),
25         inv_quad=False,
26         logdet=False,
27         probe_vectors=None,
28         probe_vector_norms=None,
29         *args
30     ):
31         """
32         *args - The arguments representing the PSD matrix A (or batch of PSD matrices A)
33         If self.inv_quad is true, the first entry in *args is inv_quad_rhs (Tensor)
34         - the RHS of the matrix solves.
35 
36         Returns:
37         - (Scalar) The inverse quadratic form (or None, if self.inv_quad is False)
38         - (Scalar) The log determinant (or None, self.if logdet is False)
39         """
40 
41         if not (inv_quad or logdet):
42             raise RuntimeError("Either inv_quad or logdet must be true (or both)")
43 
44         ctx.representation_tree = representation_tree
45         ctx.dtype = dtype
46         ctx.device = device
47         ctx.matrix_shape = matrix_shape
48         ctx.batch_shape = batch_shape
49         ctx.inv_quad = inv_quad
50         ctx.logdet = logdet
51 
52         matrix_args = None
53         inv_quad_rhs = None
54         if ctx.inv_quad:
55             matrix_args = args[1:]
56             inv_quad_rhs = args[0]
57         else:
58             matrix_args = args
59 
60         # Get closure for matmul
61         lazy_tsr = ctx.representation_tree(*matrix_args)
62         with torch.no_grad():
63             preconditioner, precond_lt, logdet_correction = lazy_tsr._preconditioner()
64 
65         ctx.preconditioner = preconditioner
66 
67         if (probe_vectors is None or probe_vector_norms is None) and logdet:
68             num_random_probes = settings.num_trace_samples.value()
69             if preconditioner is None:
70                 probe_vectors = torch.empty(matrix_shape[-1], num_random_probes, dtype=dtype, device=device)
71                 probe_vectors.bernoulli_().mul_(2).add_(-1)
72                 probe_vector_norms = torch.norm(probe_vectors, 2, dim=-2, keepdim=True)
73                 if batch_shape is not None:
74                     probe_vectors = probe_vectors.expand(*batch_shape, matrix_shape[-1], num_random_probes)
75                     probe_vector_norms = probe_vector_norms.expand(*batch_shape, 1, num_random_probes)
76             else:
77                 probe_vectors = precond_lt.zero_mean_mvn_samples(num_random_probes)
78                 probe_vectors = probe_vectors.unsqueeze(-2).transpose(0, -2).squeeze(0).transpose(-2, -1)
79                 probe_vector_norms = torch.norm(probe_vectors, p=2, dim=-2, keepdim=True)
80             probe_vectors = probe_vectors.div(probe_vector_norms)
81 
82         ctx.probe_vectors = probe_vectors
83         ctx.probe_vector_norms = probe_vector_norms
84 
85         if ctx.logdet and not ctx.probe_vectors.numel():
86             raise RuntimeError("Probe vectors were not supplied for logdet computation")
87 
88         # Collect terms for LinearCG
89         # We use LinearCG for both matrix solves and for stochastically estimating the log det
90         rhs_list = []
91         num_random_probes = 0
92         num_inv_quad_solves = 0
93 
94         # RHS for logdet
95         if ctx.logdet:
96             rhs_list.append(ctx.probe_vectors)
97             num_random_probes = ctx.probe_vectors.size(-1)
98 
99         # RHS for inv_quad
100         ctx.is_vector = False
101         if ctx.inv_quad:
102             if inv_quad_rhs.ndimension() == 1:
103                 inv_quad_rhs = inv_quad_rhs.unsqueeze(-1)
104                 ctx.is_vector = True
105             rhs_list.append(inv_quad_rhs)
106             num_inv_quad_solves = inv_quad_rhs.size(-1)
107 
108         # Perform solves (for inv_quad) and tridiagonalization (for estimating logdet)
109         rhs = torch.cat(rhs_list, -1)
110         t_mat = None
111         if ctx.logdet and settings.skip_logdet_forward.off():
112             solves, t_mat = lazy_tsr._solve(rhs, preconditioner, num_tridiag=num_random_probes)
113 
114         else:
115             solves = lazy_tsr._solve(rhs, preconditioner, num_tridiag=0)
116 
117         # Final values to return
118         logdet_term = torch.zeros(lazy_tsr.batch_shape, dtype=ctx.dtype, device=ctx.device)
119         inv_quad_term = torch.zeros(lazy_tsr.batch_shape, dtype=ctx.dtype, device=ctx.device)
120 
121         # Compute logdet from tridiagonalization
122         if ctx.logdet and settings.skip_logdet_forward.off():
123             if torch.any(torch.isnan(t_mat)).item():
124                 logdet_term = torch.tensor(float("nan"), dtype=ctx.dtype, device=ctx.device)
125             else:
126                 if ctx.batch_shape is None:
127                     t_mat = t_mat.unsqueeze(1)
128                 eigenvalues, eigenvectors = lanczos_tridiag_to_diag(t_mat)
129                 slq = StochasticLQ()
130                 logdet_term, = slq.evaluate(ctx.matrix_shape, eigenvalues, eigenvectors, [lambda x: x.log()])
131 
132                 # Add correction
133                 if logdet_correction is not None:
134                     logdet_term = logdet_term + logdet_correction
135 
136         # Extract inv_quad solves from all the solves
137         if ctx.inv_quad:
138             inv_quad_solves = solves.narrow(-1, num_random_probes, num_inv_quad_solves)
139             inv_quad_term = (inv_quad_solves * inv_quad_rhs).sum(-2)
140 
141         ctx.num_random_probes = num_random_probes
142         ctx.num_inv_quad_solves = num_inv_quad_solves
143 
144         to_save = list(matrix_args) + [solves, ]
145         ctx.save_for_backward(*to_save)
146 
147         if settings.memory_efficient.off():
148             ctx._lazy_tsr = lazy_tsr
149 
150         return inv_quad_term, logdet_term
151 
152     @staticmethod
153     def backward(ctx, inv_quad_grad_output, logdet_grad_output):
154         matrix_arg_grads = None
155         inv_quad_rhs_grad = None
156 
157         # Which backward passes should we compute?
158         compute_inv_quad_grad = inv_quad_grad_output.abs().sum() and ctx.inv_quad
159         compute_logdet_grad = logdet_grad_output.abs().sum() and ctx.logdet
160 
161         # Get input arguments, and get gradients in the proper form
162         matrix_args = ctx.saved_tensors[:-1]
163         solves = ctx.saved_tensors[-1]
164 
165         if hasattr(ctx, "_lazy_tsr"):
166             lazy_tsr = ctx._lazy_tsr
167         else:
168             lazy_tsr = ctx.representation_tree(*matrix_args)
169 
170         # Fix grad_output sizes
171         if ctx.inv_quad:
172             inv_quad_grad_output = inv_quad_grad_output.unsqueeze(-2)
173         if compute_logdet_grad:
174             logdet_grad_output = logdet_grad_output.unsqueeze(-1)
175             logdet_grad_output.unsqueeze_(-1)
176 
177         # Divide up the solves
178         probe_vector_solves = None
179         inv_quad_solves = None
180         neg_inv_quad_solves_times_grad_out = None
181         if compute_logdet_grad:
182             coef = 1.0 / ctx.probe_vectors.size(-1)
183             probe_vector_solves = solves.narrow(-1, 0, ctx.num_random_probes).mul(coef)
184             probe_vector_solves.mul_(ctx.probe_vector_norms).mul_(logdet_grad_output)
185             probe_vectors = ctx.probe_vectors.mul(ctx.probe_vector_norms)
186         if ctx.inv_quad:
187             inv_quad_solves = solves.narrow(-1, ctx.num_random_probes, ctx.num_inv_quad_solves)
188             neg_inv_quad_solves_times_grad_out = inv_quad_solves.mul(inv_quad_grad_output).mul_(-1)
189 
190         # input_1 gradient
191         if any(ctx.needs_input_grad):
192             # Collect terms for arg grads
193             left_factors_list = []
194             right_factors_list = []
195 
196             if compute_logdet_grad:
197                 left_factors_list.append(probe_vector_solves)
198                 if ctx.preconditioner is not None:
199                     probe_vectors = ctx.preconditioner(probe_vectors)
200                 right_factors_list.append(probe_vectors)
201 
202             if compute_inv_quad_grad:
203                 left_factors_list.append(neg_inv_quad_solves_times_grad_out)
204                 right_factors_list.append(inv_quad_solves)
205 
206             left_factors = torch.cat(left_factors_list, -1)
207             right_factors = torch.cat(right_factors_list, -1)
208             matrix_arg_grads = lazy_tsr._quad_form_derivative(left_factors, right_factors)
209 
210         # input_2 gradients
211         if compute_inv_quad_grad and ctx.needs_input_grad[9]:
212             inv_quad_rhs_grad = neg_inv_quad_solves_times_grad_out.mul_(-2)
213         elif ctx.inv_quad:
214             inv_quad_rhs_grad = torch.zeros_like(inv_quad_solves)
215         if ctx.is_vector:
216             inv_quad_rhs_grad.squeeze_(-1)
217 
218         if ctx.inv_quad:
219             res = [inv_quad_rhs_grad] + list(matrix_arg_grads)
220         else:
221             res = matrix_arg_grads
222 
223         return tuple([None] * 9 + res)
224
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gpytorch/functions/_inv_quad_log_det.py b/gpytorch/functions/_inv_quad_log_det.py
--- a/gpytorch/functions/_inv_quad_log_det.py
+++ b/gpytorch/functions/_inv_quad_log_det.py
@@ -218,6 +218,6 @@
         if ctx.inv_quad:
             res = [inv_quad_rhs_grad] + list(matrix_arg_grads)
         else:
-            res = matrix_arg_grads
+            res = list(matrix_arg_grads)
 
         return tuple([None] * 9 + res)

{"golden_diff": "diff --git a/gpytorch/functions/_inv_quad_log_det.py b/gpytorch/functions/_inv_quad_log_det.py\n--- a/gpytorch/functions/_inv_quad_log_det.py\n+++ b/gpytorch/functions/_inv_quad_log_det.py\n@@ -218,6 +218,6 @@\n         if ctx.inv_quad:\n             res = [inv_quad_rhs_grad] + list(matrix_arg_grads)\n         else:\n-            res = matrix_arg_grads\n+            res = list(matrix_arg_grads)\n \n         return tuple([None] * 9 + res)\n", "issue": "[Bug] TypeError when calling `backward` on `gpytorch.functions.logdet` \n# \ud83d\udc1b Bug\r\n\r\nI think there's a missing `list()` in https://github.com/cornellius-gp/gpytorch/blob/master/gpytorch/functions/_inv_quad_log_det.py#L221. I'm not super familiar with gpytorch internals so hopefully this is correct -- if so, happy to contribute the one-liner fix. \r\n\r\n## To reproduce\r\n\r\n** Code snippet to reproduce **\r\n```python\r\n### works (I'm guessing something dispatches elsewhere for small matrices?)\r\nimport torch\r\nfrom torch.autograd import backward\r\nimport gpytorch\r\nfrom gpytorch.functions import logdet, inv_matmul\r\nn = 100\r\ninp = torch.arange(n, dtype=torch.float)\r\nkern = gpytorch.kernels.RBFKernel()(inp)\r\nld = logdet(kern)\r\nbackward(ld)\r\n\r\n### doesn't work\r\nimport torch\r\nfrom torch.autograd import backward\r\nimport gpytorch\r\nfrom gpytorch.functions import logdet, inv_matmul\r\nn = 1000\r\ninp = torch.arange(n, dtype=torch.float)\r\nkern = gpytorch.kernels.RBFKernel()(inp)\r\nld = logdet(kern)\r\nbackward(ld)\r\n```\r\n\r\n** Stack trace/error message **\r\n```\r\n---------------------------------------------------------------------------\r\nTypeError                                 Traceback (most recent call last)\r\n<ipython-input-46-593fbced29ac> in <module>()\r\n      3 kern = gpytorch.kernels.RBFKernel()(inp)\r\n      4 ld = logdet(kern)\r\n----> 5 backward(ld)\r\n\r\n<PATH SNIPPED>/lib/python3.7/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\r\n     91     Variable._execution_engine.run_backward(\r\n     92         tensors, grad_tensors, retain_graph, create_graph,\r\n---> 93         allow_unreachable=True)  # allow_unreachable flag\r\n     94 \r\n     95 \r\n\r\n<PATH SNIPPED>/lib/python3.7/site-packages/torch/autograd/function.py in apply(self, *args)\r\n     75 \r\n     76     def apply(self, *args):\r\n---> 77         return self._forward_cls.backward(self, *args)\r\n     78 \r\n     79 \r\n\r\n<PATH SNIPPED>lib/python3.7/site-packages/gpytorch/functions/_inv_quad_log_det.py in backward(ctx, inv_quad_grad_output, logdet_grad_output)\r\n    221             res = matrix_arg_grads\r\n    222 \r\n--> 223         return tuple([None] * 9 + res)\r\n\r\nTypeError: can only concatenate list (not \"tuple\") to list\r\n```\r\n\r\n## Expected Behavior\r\n\r\nNo error. \r\n\r\n## System information\r\n\r\n**Please complete the following information:**\r\n- GPyTorch version: 0.3.2 <!-- GPyTorch Version (run `print(gpytorch.__version__)` -->\r\n- pytorch version 1.1.0. \r\n- Mac OSX. \r\n\r\n\n", "before_files": [{"content": "#!/usr/bin/env python3\n\nimport torch\nfrom torch.autograd import Function\nfrom ..utils.lanczos import lanczos_tridiag_to_diag\nfrom ..utils.stochastic_lq import StochasticLQ\nfrom .. import settings\n\n\nclass InvQuadLogDet(Function):\n    \"\"\"\n    Given a PSD matrix A (or a batch of PSD matrices A), this function computes one or both\n    of the following\n    - The matrix solves A^{-1} b\n    - logdet(A)\n    \"\"\"\n    @staticmethod\n    def forward(\n        ctx,\n        representation_tree,\n        dtype,\n        device,\n        matrix_shape,\n        batch_shape=torch.Size(),\n        inv_quad=False,\n        logdet=False,\n        probe_vectors=None,\n        probe_vector_norms=None,\n        *args\n    ):\n        \"\"\"\n        *args - The arguments representing the PSD matrix A (or batch of PSD matrices A)\n        If self.inv_quad is true, the first entry in *args is inv_quad_rhs (Tensor)\n        - the RHS of the matrix solves.\n\n        Returns:\n        - (Scalar) The inverse quadratic form (or None, if self.inv_quad is False)\n        - (Scalar) The log determinant (or None, self.if logdet is False)\n        \"\"\"\n\n        if not (inv_quad or logdet):\n            raise RuntimeError(\"Either inv_quad or logdet must be true (or both)\")\n\n        ctx.representation_tree = representation_tree\n        ctx.dtype = dtype\n        ctx.device = device\n        ctx.matrix_shape = matrix_shape\n        ctx.batch_shape = batch_shape\n        ctx.inv_quad = inv_quad\n        ctx.logdet = logdet\n\n        matrix_args = None\n        inv_quad_rhs = None\n        if ctx.inv_quad:\n            matrix_args = args[1:]\n            inv_quad_rhs = args[0]\n        else:\n            matrix_args = args\n\n        # Get closure for matmul\n        lazy_tsr = ctx.representation_tree(*matrix_args)\n        with torch.no_grad():\n            preconditioner, precond_lt, logdet_correction = lazy_tsr._preconditioner()\n\n        ctx.preconditioner = preconditioner\n\n        if (probe_vectors is None or probe_vector_norms is None) and logdet:\n            num_random_probes = settings.num_trace_samples.value()\n            if preconditioner is None:\n                probe_vectors = torch.empty(matrix_shape[-1], num_random_probes, dtype=dtype, device=device)\n                probe_vectors.bernoulli_().mul_(2).add_(-1)\n                probe_vector_norms = torch.norm(probe_vectors, 2, dim=-2, keepdim=True)\n                if batch_shape is not None:\n                    probe_vectors = probe_vectors.expand(*batch_shape, matrix_shape[-1], num_random_probes)\n                    probe_vector_norms = probe_vector_norms.expand(*batch_shape, 1, num_random_probes)\n            else:\n                probe_vectors = precond_lt.zero_mean_mvn_samples(num_random_probes)\n                probe_vectors = probe_vectors.unsqueeze(-2).transpose(0, -2).squeeze(0).transpose(-2, -1)\n                probe_vector_norms = torch.norm(probe_vectors, p=2, dim=-2, keepdim=True)\n            probe_vectors = probe_vectors.div(probe_vector_norms)\n\n        ctx.probe_vectors = probe_vectors\n        ctx.probe_vector_norms = probe_vector_norms\n\n        if ctx.logdet and not ctx.probe_vectors.numel():\n            raise RuntimeError(\"Probe vectors were not supplied for logdet computation\")\n\n        # Collect terms for LinearCG\n        # We use LinearCG for both matrix solves and for stochastically estimating the log det\n        rhs_list = []\n        num_random_probes = 0\n        num_inv_quad_solves = 0\n\n        # RHS for logdet\n        if ctx.logdet:\n            rhs_list.append(ctx.probe_vectors)\n            num_random_probes = ctx.probe_vectors.size(-1)\n\n        # RHS for inv_quad\n        ctx.is_vector = False\n        if ctx.inv_quad:\n            if inv_quad_rhs.ndimension() == 1:\n                inv_quad_rhs = inv_quad_rhs.unsqueeze(-1)\n                ctx.is_vector = True\n            rhs_list.append(inv_quad_rhs)\n            num_inv_quad_solves = inv_quad_rhs.size(-1)\n\n        # Perform solves (for inv_quad) and tridiagonalization (for estimating logdet)\n        rhs = torch.cat(rhs_list, -1)\n        t_mat = None\n        if ctx.logdet and settings.skip_logdet_forward.off():\n            solves, t_mat = lazy_tsr._solve(rhs, preconditioner, num_tridiag=num_random_probes)\n\n        else:\n            solves = lazy_tsr._solve(rhs, preconditioner, num_tridiag=0)\n\n        # Final values to return\n        logdet_term = torch.zeros(lazy_tsr.batch_shape, dtype=ctx.dtype, device=ctx.device)\n        inv_quad_term = torch.zeros(lazy_tsr.batch_shape, dtype=ctx.dtype, device=ctx.device)\n\n        # Compute logdet from tridiagonalization\n        if ctx.logdet and settings.skip_logdet_forward.off():\n            if torch.any(torch.isnan(t_mat)).item():\n                logdet_term = torch.tensor(float(\"nan\"), dtype=ctx.dtype, device=ctx.device)\n            else:\n                if ctx.batch_shape is None:\n                    t_mat = t_mat.unsqueeze(1)\n                eigenvalues, eigenvectors = lanczos_tridiag_to_diag(t_mat)\n                slq = StochasticLQ()\n                logdet_term, = slq.evaluate(ctx.matrix_shape, eigenvalues, eigenvectors, [lambda x: x.log()])\n\n                # Add correction\n                if logdet_correction is not None:\n                    logdet_term = logdet_term + logdet_correction\n\n        # Extract inv_quad solves from all the solves\n        if ctx.inv_quad:\n            inv_quad_solves = solves.narrow(-1, num_random_probes, num_inv_quad_solves)\n            inv_quad_term = (inv_quad_solves * inv_quad_rhs).sum(-2)\n\n        ctx.num_random_probes = num_random_probes\n        ctx.num_inv_quad_solves = num_inv_quad_solves\n\n        to_save = list(matrix_args) + [solves, ]\n        ctx.save_for_backward(*to_save)\n\n        if settings.memory_efficient.off():\n            ctx._lazy_tsr = lazy_tsr\n\n        return inv_quad_term, logdet_term\n\n    @staticmethod\n    def backward(ctx, inv_quad_grad_output, logdet_grad_output):\n        matrix_arg_grads = None\n        inv_quad_rhs_grad = None\n\n        # Which backward passes should we compute?\n        compute_inv_quad_grad = inv_quad_grad_output.abs().sum() and ctx.inv_quad\n        compute_logdet_grad = logdet_grad_output.abs().sum() and ctx.logdet\n\n        # Get input arguments, and get gradients in the proper form\n        matrix_args = ctx.saved_tensors[:-1]\n        solves = ctx.saved_tensors[-1]\n\n        if hasattr(ctx, \"_lazy_tsr\"):\n            lazy_tsr = ctx._lazy_tsr\n        else:\n            lazy_tsr = ctx.representation_tree(*matrix_args)\n\n        # Fix grad_output sizes\n        if ctx.inv_quad:\n            inv_quad_grad_output = inv_quad_grad_output.unsqueeze(-2)\n        if compute_logdet_grad:\n            logdet_grad_output = logdet_grad_output.unsqueeze(-1)\n            logdet_grad_output.unsqueeze_(-1)\n\n        # Divide up the solves\n        probe_vector_solves = None\n        inv_quad_solves = None\n        neg_inv_quad_solves_times_grad_out = None\n        if compute_logdet_grad:\n            coef = 1.0 / ctx.probe_vectors.size(-1)\n            probe_vector_solves = solves.narrow(-1, 0, ctx.num_random_probes).mul(coef)\n            probe_vector_solves.mul_(ctx.probe_vector_norms).mul_(logdet_grad_output)\n            probe_vectors = ctx.probe_vectors.mul(ctx.probe_vector_norms)\n        if ctx.inv_quad:\n            inv_quad_solves = solves.narrow(-1, ctx.num_random_probes, ctx.num_inv_quad_solves)\n            neg_inv_quad_solves_times_grad_out = inv_quad_solves.mul(inv_quad_grad_output).mul_(-1)\n\n        # input_1 gradient\n        if any(ctx.needs_input_grad):\n            # Collect terms for arg grads\n            left_factors_list = []\n            right_factors_list = []\n\n            if compute_logdet_grad:\n                left_factors_list.append(probe_vector_solves)\n                if ctx.preconditioner is not None:\n                    probe_vectors = ctx.preconditioner(probe_vectors)\n                right_factors_list.append(probe_vectors)\n\n            if compute_inv_quad_grad:\n                left_factors_list.append(neg_inv_quad_solves_times_grad_out)\n                right_factors_list.append(inv_quad_solves)\n\n            left_factors = torch.cat(left_factors_list, -1)\n            right_factors = torch.cat(right_factors_list, -1)\n            matrix_arg_grads = lazy_tsr._quad_form_derivative(left_factors, right_factors)\n\n        # input_2 gradients\n        if compute_inv_quad_grad and ctx.needs_input_grad[9]:\n            inv_quad_rhs_grad = neg_inv_quad_solves_times_grad_out.mul_(-2)\n        elif ctx.inv_quad:\n            inv_quad_rhs_grad = torch.zeros_like(inv_quad_solves)\n        if ctx.is_vector:\n            inv_quad_rhs_grad.squeeze_(-1)\n\n        if ctx.inv_quad:\n            res = [inv_quad_rhs_grad] + list(matrix_arg_grads)\n        else:\n            res = matrix_arg_grads\n\n        return tuple([None] * 9 + res)\n", "path": "gpytorch/functions/_inv_quad_log_det.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\nimport torch\nfrom torch.autograd import Function\nfrom ..utils.lanczos import lanczos_tridiag_to_diag\nfrom ..utils.stochastic_lq import StochasticLQ\nfrom .. import settings\n\n\nclass InvQuadLogDet(Function):\n    \"\"\"\n    Given a PSD matrix A (or a batch of PSD matrices A), this function computes one or both\n    of the following\n    - The matrix solves A^{-1} b\n    - logdet(A)\n    \"\"\"\n    @staticmethod\n    def forward(\n        ctx,\n        representation_tree,\n        dtype,\n        device,\n        matrix_shape,\n        batch_shape=torch.Size(),\n        inv_quad=False,\n        logdet=False,\n        probe_vectors=None,\n        probe_vector_norms=None,\n        *args\n    ):\n        \"\"\"\n        *args - The arguments representing the PSD matrix A (or batch of PSD matrices A)\n        If self.inv_quad is true, the first entry in *args is inv_quad_rhs (Tensor)\n        - the RHS of the matrix solves.\n\n        Returns:\n        - (Scalar) The inverse quadratic form (or None, if self.inv_quad is False)\n        - (Scalar) The log determinant (or None, self.if logdet is False)\n        \"\"\"\n\n        if not (inv_quad or logdet):\n            raise RuntimeError(\"Either inv_quad or logdet must be true (or both)\")\n\n        ctx.representation_tree = representation_tree\n        ctx.dtype = dtype\n        ctx.device = device\n        ctx.matrix_shape = matrix_shape\n        ctx.batch_shape = batch_shape\n        ctx.inv_quad = inv_quad\n        ctx.logdet = logdet\n\n        matrix_args = None\n        inv_quad_rhs = None\n        if ctx.inv_quad:\n            matrix_args = args[1:]\n            inv_quad_rhs = args[0]\n        else:\n            matrix_args = args\n\n        # Get closure for matmul\n        lazy_tsr = ctx.representation_tree(*matrix_args)\n        with torch.no_grad():\n            preconditioner, precond_lt, logdet_correction = lazy_tsr._preconditioner()\n\n        ctx.preconditioner = preconditioner\n\n        if (probe_vectors is None or probe_vector_norms is None) and logdet:\n            num_random_probes = settings.num_trace_samples.value()\n            if preconditioner is None:\n                probe_vectors = torch.empty(matrix_shape[-1], num_random_probes, dtype=dtype, device=device)\n                probe_vectors.bernoulli_().mul_(2).add_(-1)\n                probe_vector_norms = torch.norm(probe_vectors, 2, dim=-2, keepdim=True)\n                if batch_shape is not None:\n                    probe_vectors = probe_vectors.expand(*batch_shape, matrix_shape[-1], num_random_probes)\n                    probe_vector_norms = probe_vector_norms.expand(*batch_shape, 1, num_random_probes)\n            else:\n                probe_vectors = precond_lt.zero_mean_mvn_samples(num_random_probes)\n                probe_vectors = probe_vectors.unsqueeze(-2).transpose(0, -2).squeeze(0).transpose(-2, -1)\n                probe_vector_norms = torch.norm(probe_vectors, p=2, dim=-2, keepdim=True)\n            probe_vectors = probe_vectors.div(probe_vector_norms)\n\n        ctx.probe_vectors = probe_vectors\n        ctx.probe_vector_norms = probe_vector_norms\n\n        if ctx.logdet and not ctx.probe_vectors.numel():\n            raise RuntimeError(\"Probe vectors were not supplied for logdet computation\")\n\n        # Collect terms for LinearCG\n        # We use LinearCG for both matrix solves and for stochastically estimating the log det\n        rhs_list = []\n        num_random_probes = 0\n        num_inv_quad_solves = 0\n\n        # RHS for logdet\n        if ctx.logdet:\n            rhs_list.append(ctx.probe_vectors)\n            num_random_probes = ctx.probe_vectors.size(-1)\n\n        # RHS for inv_quad\n        ctx.is_vector = False\n        if ctx.inv_quad:\n            if inv_quad_rhs.ndimension() == 1:\n                inv_quad_rhs = inv_quad_rhs.unsqueeze(-1)\n                ctx.is_vector = True\n            rhs_list.append(inv_quad_rhs)\n            num_inv_quad_solves = inv_quad_rhs.size(-1)\n\n        # Perform solves (for inv_quad) and tridiagonalization (for estimating logdet)\n        rhs = torch.cat(rhs_list, -1)\n        t_mat = None\n        if ctx.logdet and settings.skip_logdet_forward.off():\n            solves, t_mat = lazy_tsr._solve(rhs, preconditioner, num_tridiag=num_random_probes)\n\n        else:\n            solves = lazy_tsr._solve(rhs, preconditioner, num_tridiag=0)\n\n        # Final values to return\n        logdet_term = torch.zeros(lazy_tsr.batch_shape, dtype=ctx.dtype, device=ctx.device)\n        inv_quad_term = torch.zeros(lazy_tsr.batch_shape, dtype=ctx.dtype, device=ctx.device)\n\n        # Compute logdet from tridiagonalization\n        if ctx.logdet and settings.skip_logdet_forward.off():\n            if torch.any(torch.isnan(t_mat)).item():\n                logdet_term = torch.tensor(float(\"nan\"), dtype=ctx.dtype, device=ctx.device)\n            else:\n                if ctx.batch_shape is None:\n                    t_mat = t_mat.unsqueeze(1)\n                eigenvalues, eigenvectors = lanczos_tridiag_to_diag(t_mat)\n                slq = StochasticLQ()\n                logdet_term, = slq.evaluate(ctx.matrix_shape, eigenvalues, eigenvectors, [lambda x: x.log()])\n\n                # Add correction\n                if logdet_correction is not None:\n                    logdet_term = logdet_term + logdet_correction\n\n        # Extract inv_quad solves from all the solves\n        if ctx.inv_quad:\n            inv_quad_solves = solves.narrow(-1, num_random_probes, num_inv_quad_solves)\n            inv_quad_term = (inv_quad_solves * inv_quad_rhs).sum(-2)\n\n        ctx.num_random_probes = num_random_probes\n        ctx.num_inv_quad_solves = num_inv_quad_solves\n\n        to_save = list(matrix_args) + [solves, ]\n        ctx.save_for_backward(*to_save)\n\n        if settings.memory_efficient.off():\n            ctx._lazy_tsr = lazy_tsr\n\n        return inv_quad_term, logdet_term\n\n    @staticmethod\n    def backward(ctx, inv_quad_grad_output, logdet_grad_output):\n        matrix_arg_grads = None\n        inv_quad_rhs_grad = None\n\n        # Which backward passes should we compute?\n        compute_inv_quad_grad = inv_quad_grad_output.abs().sum() and ctx.inv_quad\n        compute_logdet_grad = logdet_grad_output.abs().sum() and ctx.logdet\n\n        # Get input arguments, and get gradients in the proper form\n        matrix_args = ctx.saved_tensors[:-1]\n        solves = ctx.saved_tensors[-1]\n\n        if hasattr(ctx, \"_lazy_tsr\"):\n            lazy_tsr = ctx._lazy_tsr\n        else:\n            lazy_tsr = ctx.representation_tree(*matrix_args)\n\n        # Fix grad_output sizes\n        if ctx.inv_quad:\n            inv_quad_grad_output = inv_quad_grad_output.unsqueeze(-2)\n        if compute_logdet_grad:\n            logdet_grad_output = logdet_grad_output.unsqueeze(-1)\n            logdet_grad_output.unsqueeze_(-1)\n\n        # Divide up the solves\n        probe_vector_solves = None\n        inv_quad_solves = None\n        neg_inv_quad_solves_times_grad_out = None\n        if compute_logdet_grad:\n            coef = 1.0 / ctx.probe_vectors.size(-1)\n            probe_vector_solves = solves.narrow(-1, 0, ctx.num_random_probes).mul(coef)\n            probe_vector_solves.mul_(ctx.probe_vector_norms).mul_(logdet_grad_output)\n            probe_vectors = ctx.probe_vectors.mul(ctx.probe_vector_norms)\n        if ctx.inv_quad:\n            inv_quad_solves = solves.narrow(-1, ctx.num_random_probes, ctx.num_inv_quad_solves)\n            neg_inv_quad_solves_times_grad_out = inv_quad_solves.mul(inv_quad_grad_output).mul_(-1)\n\n        # input_1 gradient\n        if any(ctx.needs_input_grad):\n            # Collect terms for arg grads\n            left_factors_list = []\n            right_factors_list = []\n\n            if compute_logdet_grad:\n                left_factors_list.append(probe_vector_solves)\n                if ctx.preconditioner is not None:\n                    probe_vectors = ctx.preconditioner(probe_vectors)\n                right_factors_list.append(probe_vectors)\n\n            if compute_inv_quad_grad:\n                left_factors_list.append(neg_inv_quad_solves_times_grad_out)\n                right_factors_list.append(inv_quad_solves)\n\n            left_factors = torch.cat(left_factors_list, -1)\n            right_factors = torch.cat(right_factors_list, -1)\n            matrix_arg_grads = lazy_tsr._quad_form_derivative(left_factors, right_factors)\n\n        # input_2 gradients\n        if compute_inv_quad_grad and ctx.needs_input_grad[9]:\n            inv_quad_rhs_grad = neg_inv_quad_solves_times_grad_out.mul_(-2)\n        elif ctx.inv_quad:\n            inv_quad_rhs_grad = torch.zeros_like(inv_quad_solves)\n        if ctx.is_vector:\n            inv_quad_rhs_grad.squeeze_(-1)\n\n        if ctx.inv_quad:\n            res = [inv_quad_rhs_grad] + list(matrix_arg_grads)\n        else:\n            res = list(matrix_arg_grads)\n\n        return tuple([None] * 9 + res)\n", "path": "gpytorch/functions/_inv_quad_log_det.py"}]}