We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Calibration error does not support negative logits
Hi there,

It appears that `CalibrationError` currently does not support some of the input data types mentioned in [this table](https://torchmetrics.readthedocs.io/en/stable/pages/classification.html#input-types) from the docs. In particular, it seems to break when fed with 

- `preds=logits` where `logits` is a `(N, C)` float32 tensor with potentially negative values.
- `preds=predictions` where `predictions` is a `(N,)` int tensor with the predicted labels.

It still works with softmax-ed logits (`preds=logits.softmax(-1)`) or, generally, with  and `(N,)`-dimensional float32 tensors (i.e. the binary input in the input data table mentioned above).

To reproduce:

```python
N, C = 10, 3
targets = torch.randint(C, (N,))

# (N, C) non-negative: works
preds = torch.rand((N, C))  # non-negative
CalibrationError()(preds=preds, target=targets)

# (N, C) potentially negative: fails
preds = torch.randn((N, C))  # potetially negative
CalibrationError()(preds=preds, target=targets)

# (N,) int type: fails
CalibrationError()(preds=targets, target=targets)

# (N,) float type non-negative: works
preds = torch.rand((N,))  # binary non-negative
CalibrationError()(preds=preds, target=targets)

# (N,) float type potentially negative: fails
preds = torch.randn((N,))  # binary potentially negative
CalibrationError()(preds=preds, target=targets)
```

Torchmetrics: v0.8
Pytorch: v1.11

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchmetrics/functional/classification/calibration_error.py`
Content:
```
1 # Copyright The PyTorch Lightning team.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 from typing import Tuple
15 
16 import torch
17 from torch import Tensor
18 
19 from torchmetrics.utilities.checks import _input_format_classification
20 from torchmetrics.utilities.enums import DataType
21 from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_8
22 
23 
24 def _binning_with_loop(
25     confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor
26 ) -> Tuple[Tensor, Tensor, Tensor]:
27     """Compute calibration bins using for loops. Use for pytorch < 1.6.
28 
29     Args:
30         confidences: The confidence (i.e. predicted prob) of the top1 prediction.
31         accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.
32         bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.
33 
34     Returns:
35         tuple with binned accuracy, binned confidence and binned probabilities
36     """
37     conf_bin = torch.zeros_like(bin_boundaries)
38     acc_bin = torch.zeros_like(bin_boundaries)
39     prop_bin = torch.zeros_like(bin_boundaries)
40     for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])):
41         # Calculated confidence and accuracy in each bin
42         in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())
43         prop_in_bin = in_bin.float().mean()
44         if prop_in_bin.item() > 0:
45             acc_bin[i] = accuracies[in_bin].float().mean()
46             conf_bin[i] = confidences[in_bin].mean()
47             prop_bin[i] = prop_in_bin
48     return acc_bin, conf_bin, prop_bin
49 
50 
51 def _binning_bucketize(
52     confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor
53 ) -> Tuple[Tensor, Tensor, Tensor]:
54     """Compute calibration bins using ``torch.bucketize``. Use for pytorch >= 1.6.
55 
56     Args:
57         confidences: The confidence (i.e. predicted prob) of the top1 prediction.
58         accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.
59         bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.
60 
61     Returns:
62         tuple with binned accuracy, binned confidence and binned probabilities
63     """
64     acc_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)
65     conf_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)
66     count_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)
67 
68     indices = torch.bucketize(confidences, bin_boundaries) - 1
69 
70     count_bin.scatter_add_(dim=0, index=indices, src=torch.ones_like(confidences))
71 
72     conf_bin.scatter_add_(dim=0, index=indices, src=confidences)
73     conf_bin = torch.nan_to_num(conf_bin / count_bin)
74 
75     acc_bin.scatter_add_(dim=0, index=indices, src=accuracies)
76     acc_bin = torch.nan_to_num(acc_bin / count_bin)
77 
78     prop_bin = count_bin / count_bin.sum()
79     return acc_bin, conf_bin, prop_bin
80 
81 
82 def _ce_compute(
83     confidences: Tensor,
84     accuracies: Tensor,
85     bin_boundaries: Tensor,
86     norm: str = "l1",
87     debias: bool = False,
88 ) -> Tensor:
89     """Computes the calibration error given the provided bin boundaries and norm.
90 
91     Args:
92         confidences: The confidence (i.e. predicted prob) of the top1 prediction.
93         accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.
94         bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.
95         norm: Norm function to use when computing calibration error. Defaults to "l1".
96         debias: Apply debiasing to L2 norm computation as in
97             `Verified Uncertainty Calibration`_. Defaults to False.
98 
99     Raises:
100         ValueError: If an unsupported norm function is provided.
101 
102     Returns:
103         Tensor: Calibration error scalar.
104     """
105     if norm not in {"l1", "l2", "max"}:
106         raise ValueError(f"Norm {norm} is not supported. Please select from l1, l2, or max. ")
107 
108     if _TORCH_GREATER_EQUAL_1_8:
109         acc_bin, conf_bin, prop_bin = _binning_bucketize(confidences, accuracies, bin_boundaries)
110     else:
111         acc_bin, conf_bin, prop_bin = _binning_with_loop(confidences, accuracies, bin_boundaries)
112 
113     if norm == "l1":
114         ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin)
115     elif norm == "max":
116         ce = torch.max(torch.abs(acc_bin - conf_bin))
117     elif norm == "l2":
118         ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin)
119         # NOTE: debiasing is disabled in the wrapper functions. This implementation differs from that in sklearn.
120         if debias:
121             # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from
122             # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/
123             debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1)
124             ce += torch.sum(torch.nan_to_num(debias_bins))  # replace nans with zeros if nothing appeared in a bin
125         ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0)
126     return ce
127 
128 
129 def _ce_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]:
130     """Given a predictions and targets tensor, computes the confidences of the top-1 prediction and records their
131     correctness.
132 
133     Args:
134         preds:  Input ``softmaxed`` predictions.
135         target: Labels.
136 
137     Raises:
138         ValueError: If the dataset shape is not binary, multiclass, or multidimensional-multiclass.
139 
140     Returns:
141         tuple with confidences and accuracies
142     """
143     _, _, mode = _input_format_classification(preds, target)
144 
145     if mode == DataType.BINARY:
146         confidences, accuracies = preds, target
147     elif mode == DataType.MULTICLASS:
148         confidences, predictions = preds.max(dim=1)
149         accuracies = predictions.eq(target)
150     elif mode == DataType.MULTIDIM_MULTICLASS:
151         # reshape tensors
152         # for preds, move the class dimension to the final axis and flatten the rest
153         confidences, predictions = torch.transpose(preds, 1, -1).flatten(0, -2).max(dim=1)
154         # for targets, just flatten the target
155         accuracies = predictions.eq(target.flatten())
156     else:
157         raise ValueError(
158             f"Calibration error is not well-defined for data with size {preds.size()} and targets {target.size()}."
159         )
160     # must be cast to float for ddp allgather to work
161     return confidences.float(), accuracies.float()
162 
163 
164 def calibration_error(preds: Tensor, target: Tensor, n_bins: int = 15, norm: str = "l1") -> Tensor:
165     r"""`Computes the Top-label Calibration Error`_
166 
167     Three different norms are implemented, each corresponding to variations on the calibration error metric.
168 
169     L1 norm (Expected Calibration Error)
170 
171     .. math::
172         \text{ECE} = \sum_i^N b_i \|(p_i - c_i)\|
173 
174     Infinity norm (Maximum Calibration Error)
175 
176     .. math::
177         \text{MCE} =  \max_{i} (p_i - c_i)
178 
179     L2 norm (Root Mean Square Calibration Error)
180 
181     .. math::
182         \text{RMSCE} = \sqrt{\sum_i^N b_i(p_i - c_i)^2}
183 
184     Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,
185     :math:`c_i` is the average confidence of predictions in bin :math:`i`, and
186     :math:`b_i` is the fraction of data points in bin :math:`i`.
187 
188     .. note:
189         L2-norm debiasing is not yet supported.
190 
191     Args:
192         preds: Model output probabilities.
193         target: Ground-truth target class labels.
194         n_bins: Number of bins to use when computing t.
195         norm: Norm used to compare empirical and expected probability bins.
196             Defaults to "l1", or Expected Calibration Error.
197     """
198     if norm not in ("l1", "l2", "max"):
199         raise ValueError(f"Norm {norm} is not supported. Please select from l1, l2, or max. ")
200 
201     if not isinstance(n_bins, int) or n_bins <= 0:
202         raise ValueError(f"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}")
203 
204     confidences, accuracies = _ce_update(preds, target)
205 
206     bin_boundaries = torch.linspace(0, 1, n_bins + 1, dtype=torch.float, device=preds.device)
207 
208     return _ce_compute(confidences, accuracies, bin_boundaries, norm=norm)
209
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchmetrics/functional/classification/calibration_error.py b/torchmetrics/functional/classification/calibration_error.py
--- a/torchmetrics/functional/classification/calibration_error.py
+++ b/torchmetrics/functional/classification/calibration_error.py
@@ -143,8 +143,12 @@
     _, _, mode = _input_format_classification(preds, target)
 
     if mode == DataType.BINARY:
+        if not ((0 <= preds) * (preds <= 1)).all():
+            preds = preds.sigmoid()
         confidences, accuracies = preds, target
     elif mode == DataType.MULTICLASS:
+        if not ((0 <= preds) * (preds <= 1)).all():
+            preds = preds.softmax(dim=1)
         confidences, predictions = preds.max(dim=1)
         accuracies = predictions.eq(target)
     elif mode == DataType.MULTIDIM_MULTICLASS:

{"golden_diff": "diff --git a/torchmetrics/functional/classification/calibration_error.py b/torchmetrics/functional/classification/calibration_error.py\n--- a/torchmetrics/functional/classification/calibration_error.py\n+++ b/torchmetrics/functional/classification/calibration_error.py\n@@ -143,8 +143,12 @@\n     _, _, mode = _input_format_classification(preds, target)\n \n     if mode == DataType.BINARY:\n+        if not ((0 <= preds) * (preds <= 1)).all():\n+            preds = preds.sigmoid()\n         confidences, accuracies = preds, target\n     elif mode == DataType.MULTICLASS:\n+        if not ((0 <= preds) * (preds <= 1)).all():\n+            preds = preds.softmax(dim=1)\n         confidences, predictions = preds.max(dim=1)\n         accuracies = predictions.eq(target)\n     elif mode == DataType.MULTIDIM_MULTICLASS:\n", "issue": "Calibration error does not support negative logits\nHi there,\r\n\r\nIt appears that `CalibrationError` currently does not support some of the input data types mentioned in [this table](https://torchmetrics.readthedocs.io/en/stable/pages/classification.html#input-types) from the docs. In particular, it seems to break when fed with \r\n\r\n- `preds=logits` where `logits` is a `(N, C)` float32 tensor with potentially negative values.\r\n- `preds=predictions` where `predictions` is a `(N,)` int tensor with the predicted labels.\r\n\r\nIt still works with softmax-ed logits (`preds=logits.softmax(-1)`) or, generally, with  and `(N,)`-dimensional float32 tensors (i.e. the binary input in the input data table mentioned above).\r\n\r\nTo reproduce:\r\n\r\n```python\r\nN, C = 10, 3\r\ntargets = torch.randint(C, (N,))\r\n\r\n# (N, C) non-negative: works\r\npreds = torch.rand((N, C))  # non-negative\r\nCalibrationError()(preds=preds, target=targets)\r\n\r\n# (N, C) potentially negative: fails\r\npreds = torch.randn((N, C))  # potetially negative\r\nCalibrationError()(preds=preds, target=targets)\r\n\r\n# (N,) int type: fails\r\nCalibrationError()(preds=targets, target=targets)\r\n\r\n# (N,) float type non-negative: works\r\npreds = torch.rand((N,))  # binary non-negative\r\nCalibrationError()(preds=preds, target=targets)\r\n\r\n# (N,) float type potentially negative: fails\r\npreds = torch.randn((N,))  # binary potentially negative\r\nCalibrationError()(preds=preds, target=targets)\r\n```\r\n\r\nTorchmetrics: v0.8\r\nPytorch: v1.11\n", "before_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torchmetrics.utilities.checks import _input_format_classification\nfrom torchmetrics.utilities.enums import DataType\nfrom torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_8\n\n\ndef _binning_with_loop(\n    confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor\n) -> Tuple[Tensor, Tensor, Tensor]:\n    \"\"\"Compute calibration bins using for loops. Use for pytorch < 1.6.\n\n    Args:\n        confidences: The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.\n\n    Returns:\n        tuple with binned accuracy, binned confidence and binned probabilities\n    \"\"\"\n    conf_bin = torch.zeros_like(bin_boundaries)\n    acc_bin = torch.zeros_like(bin_boundaries)\n    prop_bin = torch.zeros_like(bin_boundaries)\n    for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])):\n        # Calculated confidence and accuracy in each bin\n        in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())\n        prop_in_bin = in_bin.float().mean()\n        if prop_in_bin.item() > 0:\n            acc_bin[i] = accuracies[in_bin].float().mean()\n            conf_bin[i] = confidences[in_bin].mean()\n            prop_bin[i] = prop_in_bin\n    return acc_bin, conf_bin, prop_bin\n\n\ndef _binning_bucketize(\n    confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor\n) -> Tuple[Tensor, Tensor, Tensor]:\n    \"\"\"Compute calibration bins using ``torch.bucketize``. Use for pytorch >= 1.6.\n\n    Args:\n        confidences: The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.\n\n    Returns:\n        tuple with binned accuracy, binned confidence and binned probabilities\n    \"\"\"\n    acc_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)\n    conf_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)\n    count_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)\n\n    indices = torch.bucketize(confidences, bin_boundaries) - 1\n\n    count_bin.scatter_add_(dim=0, index=indices, src=torch.ones_like(confidences))\n\n    conf_bin.scatter_add_(dim=0, index=indices, src=confidences)\n    conf_bin = torch.nan_to_num(conf_bin / count_bin)\n\n    acc_bin.scatter_add_(dim=0, index=indices, src=accuracies)\n    acc_bin = torch.nan_to_num(acc_bin / count_bin)\n\n    prop_bin = count_bin / count_bin.sum()\n    return acc_bin, conf_bin, prop_bin\n\n\ndef _ce_compute(\n    confidences: Tensor,\n    accuracies: Tensor,\n    bin_boundaries: Tensor,\n    norm: str = \"l1\",\n    debias: bool = False,\n) -> Tensor:\n    \"\"\"Computes the calibration error given the provided bin boundaries and norm.\n\n    Args:\n        confidences: The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.\n        norm: Norm function to use when computing calibration error. Defaults to \"l1\".\n        debias: Apply debiasing to L2 norm computation as in\n            `Verified Uncertainty Calibration`_. Defaults to False.\n\n    Raises:\n        ValueError: If an unsupported norm function is provided.\n\n    Returns:\n        Tensor: Calibration error scalar.\n    \"\"\"\n    if norm not in {\"l1\", \"l2\", \"max\"}:\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    if _TORCH_GREATER_EQUAL_1_8:\n        acc_bin, conf_bin, prop_bin = _binning_bucketize(confidences, accuracies, bin_boundaries)\n    else:\n        acc_bin, conf_bin, prop_bin = _binning_with_loop(confidences, accuracies, bin_boundaries)\n\n    if norm == \"l1\":\n        ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin)\n    elif norm == \"max\":\n        ce = torch.max(torch.abs(acc_bin - conf_bin))\n    elif norm == \"l2\":\n        ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin)\n        # NOTE: debiasing is disabled in the wrapper functions. This implementation differs from that in sklearn.\n        if debias:\n            # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from\n            # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/\n            debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1)\n            ce += torch.sum(torch.nan_to_num(debias_bins))  # replace nans with zeros if nothing appeared in a bin\n        ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0)\n    return ce\n\n\ndef _ce_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]:\n    \"\"\"Given a predictions and targets tensor, computes the confidences of the top-1 prediction and records their\n    correctness.\n\n    Args:\n        preds:  Input ``softmaxed`` predictions.\n        target: Labels.\n\n    Raises:\n        ValueError: If the dataset shape is not binary, multiclass, or multidimensional-multiclass.\n\n    Returns:\n        tuple with confidences and accuracies\n    \"\"\"\n    _, _, mode = _input_format_classification(preds, target)\n\n    if mode == DataType.BINARY:\n        confidences, accuracies = preds, target\n    elif mode == DataType.MULTICLASS:\n        confidences, predictions = preds.max(dim=1)\n        accuracies = predictions.eq(target)\n    elif mode == DataType.MULTIDIM_MULTICLASS:\n        # reshape tensors\n        # for preds, move the class dimension to the final axis and flatten the rest\n        confidences, predictions = torch.transpose(preds, 1, -1).flatten(0, -2).max(dim=1)\n        # for targets, just flatten the target\n        accuracies = predictions.eq(target.flatten())\n    else:\n        raise ValueError(\n            f\"Calibration error is not well-defined for data with size {preds.size()} and targets {target.size()}.\"\n        )\n    # must be cast to float for ddp allgather to work\n    return confidences.float(), accuracies.float()\n\n\ndef calibration_error(preds: Tensor, target: Tensor, n_bins: int = 15, norm: str = \"l1\") -> Tensor:\n    r\"\"\"`Computes the Top-label Calibration Error`_\n\n    Three different norms are implemented, each corresponding to variations on the calibration error metric.\n\n    L1 norm (Expected Calibration Error)\n\n    .. math::\n        \\text{ECE} = \\sum_i^N b_i \\|(p_i - c_i)\\|\n\n    Infinity norm (Maximum Calibration Error)\n\n    .. math::\n        \\text{MCE} =  \\max_{i} (p_i - c_i)\n\n    L2 norm (Root Mean Square Calibration Error)\n\n    .. math::\n        \\text{RMSCE} = \\sqrt{\\sum_i^N b_i(p_i - c_i)^2}\n\n    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,\n    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and\n    :math:`b_i` is the fraction of data points in bin :math:`i`.\n\n    .. note:\n        L2-norm debiasing is not yet supported.\n\n    Args:\n        preds: Model output probabilities.\n        target: Ground-truth target class labels.\n        n_bins: Number of bins to use when computing t.\n        norm: Norm used to compare empirical and expected probability bins.\n            Defaults to \"l1\", or Expected Calibration Error.\n    \"\"\"\n    if norm not in (\"l1\", \"l2\", \"max\"):\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    if not isinstance(n_bins, int) or n_bins <= 0:\n        raise ValueError(f\"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}\")\n\n    confidences, accuracies = _ce_update(preds, target)\n\n    bin_boundaries = torch.linspace(0, 1, n_bins + 1, dtype=torch.float, device=preds.device)\n\n    return _ce_compute(confidences, accuracies, bin_boundaries, norm=norm)\n", "path": "torchmetrics/functional/classification/calibration_error.py"}], "after_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torchmetrics.utilities.checks import _input_format_classification\nfrom torchmetrics.utilities.enums import DataType\nfrom torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_8\n\n\ndef _binning_with_loop(\n    confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor\n) -> Tuple[Tensor, Tensor, Tensor]:\n    \"\"\"Compute calibration bins using for loops. Use for pytorch < 1.6.\n\n    Args:\n        confidences: The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.\n\n    Returns:\n        tuple with binned accuracy, binned confidence and binned probabilities\n    \"\"\"\n    conf_bin = torch.zeros_like(bin_boundaries)\n    acc_bin = torch.zeros_like(bin_boundaries)\n    prop_bin = torch.zeros_like(bin_boundaries)\n    for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])):\n        # Calculated confidence and accuracy in each bin\n        in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())\n        prop_in_bin = in_bin.float().mean()\n        if prop_in_bin.item() > 0:\n            acc_bin[i] = accuracies[in_bin].float().mean()\n            conf_bin[i] = confidences[in_bin].mean()\n            prop_bin[i] = prop_in_bin\n    return acc_bin, conf_bin, prop_bin\n\n\ndef _binning_bucketize(\n    confidences: Tensor, accuracies: Tensor, bin_boundaries: Tensor\n) -> Tuple[Tensor, Tensor, Tensor]:\n    \"\"\"Compute calibration bins using ``torch.bucketize``. Use for pytorch >= 1.6.\n\n    Args:\n        confidences: The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.\n\n    Returns:\n        tuple with binned accuracy, binned confidence and binned probabilities\n    \"\"\"\n    acc_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)\n    conf_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)\n    count_bin = torch.zeros(len(bin_boundaries) - 1, device=confidences.device, dtype=confidences.dtype)\n\n    indices = torch.bucketize(confidences, bin_boundaries) - 1\n\n    count_bin.scatter_add_(dim=0, index=indices, src=torch.ones_like(confidences))\n\n    conf_bin.scatter_add_(dim=0, index=indices, src=confidences)\n    conf_bin = torch.nan_to_num(conf_bin / count_bin)\n\n    acc_bin.scatter_add_(dim=0, index=indices, src=accuracies)\n    acc_bin = torch.nan_to_num(acc_bin / count_bin)\n\n    prop_bin = count_bin / count_bin.sum()\n    return acc_bin, conf_bin, prop_bin\n\n\ndef _ce_compute(\n    confidences: Tensor,\n    accuracies: Tensor,\n    bin_boundaries: Tensor,\n    norm: str = \"l1\",\n    debias: bool = False,\n) -> Tensor:\n    \"\"\"Computes the calibration error given the provided bin boundaries and norm.\n\n    Args:\n        confidences: The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies: 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries: Bin boundaries separating the ``linspace`` from 0 to 1.\n        norm: Norm function to use when computing calibration error. Defaults to \"l1\".\n        debias: Apply debiasing to L2 norm computation as in\n            `Verified Uncertainty Calibration`_. Defaults to False.\n\n    Raises:\n        ValueError: If an unsupported norm function is provided.\n\n    Returns:\n        Tensor: Calibration error scalar.\n    \"\"\"\n    if norm not in {\"l1\", \"l2\", \"max\"}:\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    if _TORCH_GREATER_EQUAL_1_8:\n        acc_bin, conf_bin, prop_bin = _binning_bucketize(confidences, accuracies, bin_boundaries)\n    else:\n        acc_bin, conf_bin, prop_bin = _binning_with_loop(confidences, accuracies, bin_boundaries)\n\n    if norm == \"l1\":\n        ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin)\n    elif norm == \"max\":\n        ce = torch.max(torch.abs(acc_bin - conf_bin))\n    elif norm == \"l2\":\n        ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin)\n        # NOTE: debiasing is disabled in the wrapper functions. This implementation differs from that in sklearn.\n        if debias:\n            # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from\n            # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/\n            debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1)\n            ce += torch.sum(torch.nan_to_num(debias_bins))  # replace nans with zeros if nothing appeared in a bin\n        ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0)\n    return ce\n\n\ndef _ce_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]:\n    \"\"\"Given a predictions and targets tensor, computes the confidences of the top-1 prediction and records their\n    correctness.\n\n    Args:\n        preds:  Input ``softmaxed`` predictions.\n        target: Labels.\n\n    Raises:\n        ValueError: If the dataset shape is not binary, multiclass, or multidimensional-multiclass.\n\n    Returns:\n        tuple with confidences and accuracies\n    \"\"\"\n    _, _, mode = _input_format_classification(preds, target)\n\n    if mode == DataType.BINARY:\n        if not ((0 <= preds) * (preds <= 1)).all():\n            preds = preds.sigmoid()\n        confidences, accuracies = preds, target\n    elif mode == DataType.MULTICLASS:\n        if not ((0 <= preds) * (preds <= 1)).all():\n            preds = preds.softmax(dim=1)\n        confidences, predictions = preds.max(dim=1)\n        accuracies = predictions.eq(target)\n    elif mode == DataType.MULTIDIM_MULTICLASS:\n        # reshape tensors\n        # for preds, move the class dimension to the final axis and flatten the rest\n        confidences, predictions = torch.transpose(preds, 1, -1).flatten(0, -2).max(dim=1)\n        # for targets, just flatten the target\n        accuracies = predictions.eq(target.flatten())\n    else:\n        raise ValueError(\n            f\"Calibration error is not well-defined for data with size {preds.size()} and targets {target.size()}.\"\n        )\n    # must be cast to float for ddp allgather to work\n    return confidences.float(), accuracies.float()\n\n\ndef calibration_error(preds: Tensor, target: Tensor, n_bins: int = 15, norm: str = \"l1\") -> Tensor:\n    r\"\"\"`Computes the Top-label Calibration Error`_\n\n    Three different norms are implemented, each corresponding to variations on the calibration error metric.\n\n    L1 norm (Expected Calibration Error)\n\n    .. math::\n        \\text{ECE} = \\sum_i^N b_i \\|(p_i - c_i)\\|\n\n    Infinity norm (Maximum Calibration Error)\n\n    .. math::\n        \\text{MCE} =  \\max_{i} (p_i - c_i)\n\n    L2 norm (Root Mean Square Calibration Error)\n\n    .. math::\n        \\text{RMSCE} = \\sqrt{\\sum_i^N b_i(p_i - c_i)^2}\n\n    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,\n    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and\n    :math:`b_i` is the fraction of data points in bin :math:`i`.\n\n    .. note:\n        L2-norm debiasing is not yet supported.\n\n    Args:\n        preds: Model output probabilities.\n        target: Ground-truth target class labels.\n        n_bins: Number of bins to use when computing t.\n        norm: Norm used to compare empirical and expected probability bins.\n            Defaults to \"l1\", or Expected Calibration Error.\n    \"\"\"\n    if norm not in (\"l1\", \"l2\", \"max\"):\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    if not isinstance(n_bins, int) or n_bins <= 0:\n        raise ValueError(f\"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}\")\n\n    confidences, accuracies = _ce_update(preds, target)\n\n    bin_boundaries = torch.linspace(0, 1, n_bins + 1, dtype=torch.float, device=preds.device)\n\n    return _ce_compute(confidences, accuracies, bin_boundaries, norm=norm)\n", "path": "torchmetrics/functional/classification/calibration_error.py"}]}