We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Typo in CalibrationError metric docs
I just noticed typos in the CE metrics - the max calibration error and RMS calibration error labels should be switched, and there's a square root sign missing from the root mean squared error (whoops). That's my mistake, I'll submit a PR to fix. 


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchmetrics/classification/calibration_error.py`
Content:
```
1 # Copyright The PyTorch Lightning team.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 from typing import Any, List, Optional
15 
16 import torch
17 from torch import Tensor
18 
19 from torchmetrics.functional.classification.calibration_error import _ce_compute, _ce_update
20 from torchmetrics.metric import Metric
21 from torchmetrics.utilities.data import dim_zero_cat
22 
23 
24 class CalibrationError(Metric):
25     r"""
26 
27     `Computes the Top-label Calibration Error`_
28     Three different norms are implemented, each corresponding to variations on the calibration error metric.
29 
30     L1 norm (Expected Calibration Error)
31 
32     .. math::
33         \text{ECE} = \frac{1}{N}\sum_i^N \|(p_i - c_i)\|
34 
35     Infinity norm (Maximum Calibration Error)
36 
37     .. math::
38         \text{RMSCE} =  \max_{i} (p_i - c_i)
39 
40     L2 norm (Root Mean Square Calibration Error)
41 
42     .. math::
43         \text{MCE} = \frac{1}{N}\sum_i^N (p_i - c_i)^2
44 
45     Where :math:`p_i` is the top-1 prediction accuracy in bin i
46     and :math:`c_i` is the average confidence of predictions in bin i.
47 
48     .. note::
49         L2-norm debiasing is not yet supported.
50 
51     Args:
52         n_bins: Number of bins to use when computing probabilites and accuracies.
53         norm: Norm used to compare empirical and expected probability bins.
54             Defaults to "l1", or Expected Calibration Error.
55         debias: Applies debiasing term, only implemented for l2 norm. Defaults to True.
56         compute_on_step:  Forward only calls ``update()`` and return None if this is set to False.
57         dist_sync_on_step: Synchronize metric state across processes at each ``forward()``
58             before returning the value at the step
59         process_group: Specify the process group on which synchronization is called.
60     """
61     DISTANCES = {"l1", "l2", "max"}
62     higher_is_better = False
63     confidences: List[Tensor]
64     accuracies: List[Tensor]
65 
66     def __init__(
67         self,
68         n_bins: int = 15,
69         norm: str = "l1",
70         compute_on_step: bool = False,
71         dist_sync_on_step: bool = False,
72         process_group: Optional[Any] = None,
73     ):
74 
75         super().__init__(
76             compute_on_step=compute_on_step,
77             dist_sync_on_step=dist_sync_on_step,
78             process_group=process_group,
79             dist_sync_fn=None,
80         )
81 
82         if norm not in self.DISTANCES:
83             raise ValueError(f"Norm {norm} is not supported. Please select from l1, l2, or max. ")
84 
85         if not isinstance(n_bins, int) or n_bins <= 0:
86             raise ValueError(f"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}")
87         self.n_bins = n_bins
88         self.register_buffer("bin_boundaries", torch.linspace(0, 1, n_bins + 1))
89         self.norm = norm
90 
91         self.add_state("confidences", [], dist_reduce_fx="cat")
92         self.add_state("accuracies", [], dist_reduce_fx="cat")
93 
94     def update(self, preds: Tensor, target: Tensor) -> None:  # type: ignore
95         """Computes top-level confidences and accuracies for the input probabilites and appends them to internal
96         state.
97 
98         Args:
99             preds (Tensor): Model output probabilities.
100             target (Tensor): Ground-truth target class labels.
101         """
102         confidences, accuracies = _ce_update(preds, target)
103 
104         self.confidences.append(confidences)
105         self.accuracies.append(accuracies)
106 
107     def compute(self) -> Tensor:
108         """Computes calibration error across all confidences and accuracies.
109 
110         Returns:
111             Tensor: Calibration error across previously collected examples.
112         """
113         confidences = dim_zero_cat(self.confidences)
114         accuracies = dim_zero_cat(self.accuracies)
115         return _ce_compute(confidences, accuracies, self.bin_boundaries, norm=self.norm)
116
```

Path: `torchmetrics/functional/classification/calibration_error.py`
Content:
```
1 # Copyright The PyTorch Lightning team.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 from typing import Tuple
15 
16 import torch
17 from torch import FloatTensor, Tensor
18 
19 from torchmetrics.utilities.checks import _input_format_classification
20 from torchmetrics.utilities.enums import DataType
21 
22 
23 def _ce_compute(
24     confidences: FloatTensor,
25     accuracies: FloatTensor,
26     bin_boundaries: FloatTensor,
27     norm: str = "l1",
28     debias: bool = False,
29 ) -> Tensor:
30     """Computes the calibration error given the provided bin boundaries and norm.
31 
32     Args:
33         confidences (FloatTensor): The confidence (i.e. predicted prob) of the top1 prediction.
34         accuracies (FloatTensor): 1.0 if the top-1 prediction was correct, 0.0 otherwise.
35         bin_boundaries (FloatTensor): Bin boundaries separating the linspace from 0 to 1.
36         norm (str, optional): Norm function to use when computing calibration error. Defaults to "l1".
37         debias (bool, optional): Apply debiasing to L2 norm computation as in
38             `Verified Uncertainty Calibration`_. Defaults to False.
39 
40     Raises:
41         ValueError: If an unsupported norm function is provided.
42 
43     Returns:
44         Tensor: Calibration error scalar.
45     """
46     if norm not in {"l1", "l2", "max"}:
47         raise ValueError(f"Norm {norm} is not supported. Please select from l1, l2, or max. ")
48 
49     conf_bin = torch.zeros_like(bin_boundaries)
50     acc_bin = torch.zeros_like(bin_boundaries)
51     prop_bin = torch.zeros_like(bin_boundaries)
52     for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])):
53         # Calculated confidence and accuracy in each bin
54         in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())
55         prop_in_bin = in_bin.float().mean()
56         if prop_in_bin.item() > 0:
57             acc_bin[i] = accuracies[in_bin].float().mean()
58             conf_bin[i] = confidences[in_bin].mean()
59             prop_bin[i] = prop_in_bin
60 
61     if norm == "l1":
62         ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin)
63     elif norm == "max":
64         ce = torch.max(torch.abs(acc_bin - conf_bin))
65     elif norm == "l2":
66         ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin)
67         # NOTE: debiasing is disabled in the wrapper functions. This implementation differs from that in sklearn.
68         if debias:
69             # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from
70             # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/
71             debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1)
72             ce += torch.sum(torch.nan_to_num(debias_bins))  # replace nans with zeros if nothing appeared in a bin
73         ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0)
74     return ce
75 
76 
77 def _ce_update(preds: Tensor, target: Tensor) -> Tuple[FloatTensor, FloatTensor]:
78     """Given a predictions and targets tensor, computes the confidences of the top-1 prediction and records their
79     correctness.
80 
81     Args:
82         preds (Tensor):  Input softmaxed predictions.
83         target (Tensor): Labels.
84 
85     Raises:
86         ValueError: If the dataset shape is not binary, multiclass, or multidimensional-multiclass.
87 
88     Returns:
89         Tuple[FloatTensor, FloatTensor]: [description]
90     """
91     _, _, mode = _input_format_classification(preds, target)
92 
93     if mode == DataType.BINARY:
94         confidences, accuracies = preds, target
95     elif mode == DataType.MULTICLASS:
96         confidences, predictions = preds.max(dim=1)
97         accuracies = predictions.eq(target)
98     elif mode == DataType.MULTIDIM_MULTICLASS:
99         # reshape tensors
100         # for preds, move the class dimension to the final axis and flatten the rest
101         confidences, predictions = torch.transpose(preds, 1, -1).flatten(0, -2).max(dim=1)
102         # for targets, just flatten the target
103         accuracies = predictions.eq(target.flatten())
104     else:
105         raise ValueError(
106             f"Calibration error is not well-defined for data with size {preds.size()} and targets {target.size()}."
107         )
108     # must be cast to float for ddp allgather to work
109     return confidences.float(), accuracies.float()
110 
111 
112 def calibration_error(preds: Tensor, target: Tensor, n_bins: int = 15, norm: str = "l1") -> Tensor:
113     r"""
114     `Computes the Top-label Calibration Error`_
115 
116     Three different norms are implemented, each corresponding to variations on the calibration error metric.
117 
118     L1 norm (Expected Calibration Error)
119 
120     .. math::
121         \text{ECE} = \frac{1}{N}\sum_i^N \|(p_i - c_i)\|
122 
123     Infinity norm (Maximum Calibration Error)
124 
125     .. math::
126         \text{RMSCE} =  \max_{i} (p_i - c_i)
127 
128     L2 norm (Root Mean Square Calibration Error)
129 
130     .. math::
131         \text{MCE} = \frac{1}{N}\sum_i^N (p_i - c_i)^2
132 
133     Where :math:`p_i` is the top-1 prediction accuracy in
134     bin i and :math:`c_i` is the average confidence of predictions in bin i.
135 
136     .. note:
137         L2-norm debiasing is not yet supported.
138 
139     Args:
140         preds (Tensor): Model output probabilities.
141         target (Tensor): Ground-truth target class labels.
142         n_bins (int, optional): Number of bins to use when computing t. Defaults to 15.
143         norm (str, optional): Norm used to compare empirical and expected probability bins.
144             Defaults to "l1", or Expected Calibration Error.
145     """
146     if norm not in ("l1", "l2", "max"):
147         raise ValueError(f"Norm {norm} is not supported. Please select from l1, l2, or max. ")
148 
149     if not isinstance(n_bins, int) or n_bins <= 0:
150         raise ValueError(f"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}")
151 
152     confidences, accuracies = _ce_update(preds, target)
153 
154     bin_boundaries = torch.linspace(0, 1, n_bins + 1, dtype=torch.float, device=preds.device)
155 
156     return _ce_compute(confidences, accuracies, bin_boundaries, norm=norm)
157
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchmetrics/classification/calibration_error.py b/torchmetrics/classification/calibration_error.py
--- a/torchmetrics/classification/calibration_error.py
+++ b/torchmetrics/classification/calibration_error.py
@@ -30,20 +30,21 @@
     L1 norm (Expected Calibration Error)
 
     .. math::
-        \text{ECE} = \frac{1}{N}\sum_i^N \|(p_i - c_i)\|
+        \text{ECE} = \sum_i^N b_i \|(p_i - c_i)\|
 
     Infinity norm (Maximum Calibration Error)
 
     .. math::
-        \text{RMSCE} =  \max_{i} (p_i - c_i)
+        \text{MCE} =  \max_{i} (p_i - c_i)
 
     L2 norm (Root Mean Square Calibration Error)
 
     .. math::
-        \text{MCE} = \frac{1}{N}\sum_i^N (p_i - c_i)^2
+        \text{RMSCE} = \sqrt{\sum_i^N b_i(p_i - c_i)^2}
 
-    Where :math:`p_i` is the top-1 prediction accuracy in bin i
-    and :math:`c_i` is the average confidence of predictions in bin i.
+    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,
+    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and
+    :math:`b_i` is the fraction of data points in bin :math:`i`.
 
     .. note::
         L2-norm debiasing is not yet supported.
diff --git a/torchmetrics/functional/classification/calibration_error.py b/torchmetrics/functional/classification/calibration_error.py
--- a/torchmetrics/functional/classification/calibration_error.py
+++ b/torchmetrics/functional/classification/calibration_error.py
@@ -118,20 +118,21 @@
     L1 norm (Expected Calibration Error)
 
     .. math::
-        \text{ECE} = \frac{1}{N}\sum_i^N \|(p_i - c_i)\|
+        \text{ECE} = \sum_i^N b_i \|(p_i - c_i)\|
 
     Infinity norm (Maximum Calibration Error)
 
     .. math::
-        \text{RMSCE} =  \max_{i} (p_i - c_i)
+        \text{MCE} =  \max_{i} (p_i - c_i)
 
     L2 norm (Root Mean Square Calibration Error)
 
     .. math::
-        \text{MCE} = \frac{1}{N}\sum_i^N (p_i - c_i)^2
+        \text{RMSCE} = \sqrt{\sum_i^N b_i(p_i - c_i)^2}
 
-    Where :math:`p_i` is the top-1 prediction accuracy in
-    bin i and :math:`c_i` is the average confidence of predictions in bin i.
+    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,
+    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and
+    :math:`b_i` is the fraction of data points in bin :math:`i`.
 
     .. note:
         L2-norm debiasing is not yet supported.

{"golden_diff": "diff --git a/torchmetrics/classification/calibration_error.py b/torchmetrics/classification/calibration_error.py\n--- a/torchmetrics/classification/calibration_error.py\n+++ b/torchmetrics/classification/calibration_error.py\n@@ -30,20 +30,21 @@\n     L1 norm (Expected Calibration Error)\n \n     .. math::\n-        \\text{ECE} = \\frac{1}{N}\\sum_i^N \\|(p_i - c_i)\\|\n+        \\text{ECE} = \\sum_i^N b_i \\|(p_i - c_i)\\|\n \n     Infinity norm (Maximum Calibration Error)\n \n     .. math::\n-        \\text{RMSCE} =  \\max_{i} (p_i - c_i)\n+        \\text{MCE} =  \\max_{i} (p_i - c_i)\n \n     L2 norm (Root Mean Square Calibration Error)\n \n     .. math::\n-        \\text{MCE} = \\frac{1}{N}\\sum_i^N (p_i - c_i)^2\n+        \\text{RMSCE} = \\sqrt{\\sum_i^N b_i(p_i - c_i)^2}\n \n-    Where :math:`p_i` is the top-1 prediction accuracy in bin i\n-    and :math:`c_i` is the average confidence of predictions in bin i.\n+    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,\n+    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and\n+    :math:`b_i` is the fraction of data points in bin :math:`i`.\n \n     .. note::\n         L2-norm debiasing is not yet supported.\ndiff --git a/torchmetrics/functional/classification/calibration_error.py b/torchmetrics/functional/classification/calibration_error.py\n--- a/torchmetrics/functional/classification/calibration_error.py\n+++ b/torchmetrics/functional/classification/calibration_error.py\n@@ -118,20 +118,21 @@\n     L1 norm (Expected Calibration Error)\n \n     .. math::\n-        \\text{ECE} = \\frac{1}{N}\\sum_i^N \\|(p_i - c_i)\\|\n+        \\text{ECE} = \\sum_i^N b_i \\|(p_i - c_i)\\|\n \n     Infinity norm (Maximum Calibration Error)\n \n     .. math::\n-        \\text{RMSCE} =  \\max_{i} (p_i - c_i)\n+        \\text{MCE} =  \\max_{i} (p_i - c_i)\n \n     L2 norm (Root Mean Square Calibration Error)\n \n     .. math::\n-        \\text{MCE} = \\frac{1}{N}\\sum_i^N (p_i - c_i)^2\n+        \\text{RMSCE} = \\sqrt{\\sum_i^N b_i(p_i - c_i)^2}\n \n-    Where :math:`p_i` is the top-1 prediction accuracy in\n-    bin i and :math:`c_i` is the average confidence of predictions in bin i.\n+    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,\n+    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and\n+    :math:`b_i` is the fraction of data points in bin :math:`i`.\n \n     .. note:\n         L2-norm debiasing is not yet supported.\n", "issue": "Typo in CalibrationError metric docs\nI just noticed typos in the CE metrics - the max calibration error and RMS calibration error labels should be switched, and there's a square root sign missing from the root mean squared error (whoops). That's my mistake, I'll submit a PR to fix. \r\n\n", "before_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Any, List, Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torchmetrics.functional.classification.calibration_error import _ce_compute, _ce_update\nfrom torchmetrics.metric import Metric\nfrom torchmetrics.utilities.data import dim_zero_cat\n\n\nclass CalibrationError(Metric):\n    r\"\"\"\n\n    `Computes the Top-label Calibration Error`_\n    Three different norms are implemented, each corresponding to variations on the calibration error metric.\n\n    L1 norm (Expected Calibration Error)\n\n    .. math::\n        \\text{ECE} = \\frac{1}{N}\\sum_i^N \\|(p_i - c_i)\\|\n\n    Infinity norm (Maximum Calibration Error)\n\n    .. math::\n        \\text{RMSCE} =  \\max_{i} (p_i - c_i)\n\n    L2 norm (Root Mean Square Calibration Error)\n\n    .. math::\n        \\text{MCE} = \\frac{1}{N}\\sum_i^N (p_i - c_i)^2\n\n    Where :math:`p_i` is the top-1 prediction accuracy in bin i\n    and :math:`c_i` is the average confidence of predictions in bin i.\n\n    .. note::\n        L2-norm debiasing is not yet supported.\n\n    Args:\n        n_bins: Number of bins to use when computing probabilites and accuracies.\n        norm: Norm used to compare empirical and expected probability bins.\n            Defaults to \"l1\", or Expected Calibration Error.\n        debias: Applies debiasing term, only implemented for l2 norm. Defaults to True.\n        compute_on_step:  Forward only calls ``update()`` and return None if this is set to False.\n        dist_sync_on_step: Synchronize metric state across processes at each ``forward()``\n            before returning the value at the step\n        process_group: Specify the process group on which synchronization is called.\n    \"\"\"\n    DISTANCES = {\"l1\", \"l2\", \"max\"}\n    higher_is_better = False\n    confidences: List[Tensor]\n    accuracies: List[Tensor]\n\n    def __init__(\n        self,\n        n_bins: int = 15,\n        norm: str = \"l1\",\n        compute_on_step: bool = False,\n        dist_sync_on_step: bool = False,\n        process_group: Optional[Any] = None,\n    ):\n\n        super().__init__(\n            compute_on_step=compute_on_step,\n            dist_sync_on_step=dist_sync_on_step,\n            process_group=process_group,\n            dist_sync_fn=None,\n        )\n\n        if norm not in self.DISTANCES:\n            raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n        if not isinstance(n_bins, int) or n_bins <= 0:\n            raise ValueError(f\"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}\")\n        self.n_bins = n_bins\n        self.register_buffer(\"bin_boundaries\", torch.linspace(0, 1, n_bins + 1))\n        self.norm = norm\n\n        self.add_state(\"confidences\", [], dist_reduce_fx=\"cat\")\n        self.add_state(\"accuracies\", [], dist_reduce_fx=\"cat\")\n\n    def update(self, preds: Tensor, target: Tensor) -> None:  # type: ignore\n        \"\"\"Computes top-level confidences and accuracies for the input probabilites and appends them to internal\n        state.\n\n        Args:\n            preds (Tensor): Model output probabilities.\n            target (Tensor): Ground-truth target class labels.\n        \"\"\"\n        confidences, accuracies = _ce_update(preds, target)\n\n        self.confidences.append(confidences)\n        self.accuracies.append(accuracies)\n\n    def compute(self) -> Tensor:\n        \"\"\"Computes calibration error across all confidences and accuracies.\n\n        Returns:\n            Tensor: Calibration error across previously collected examples.\n        \"\"\"\n        confidences = dim_zero_cat(self.confidences)\n        accuracies = dim_zero_cat(self.accuracies)\n        return _ce_compute(confidences, accuracies, self.bin_boundaries, norm=self.norm)\n", "path": "torchmetrics/classification/calibration_error.py"}, {"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Tuple\n\nimport torch\nfrom torch import FloatTensor, Tensor\n\nfrom torchmetrics.utilities.checks import _input_format_classification\nfrom torchmetrics.utilities.enums import DataType\n\n\ndef _ce_compute(\n    confidences: FloatTensor,\n    accuracies: FloatTensor,\n    bin_boundaries: FloatTensor,\n    norm: str = \"l1\",\n    debias: bool = False,\n) -> Tensor:\n    \"\"\"Computes the calibration error given the provided bin boundaries and norm.\n\n    Args:\n        confidences (FloatTensor): The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies (FloatTensor): 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries (FloatTensor): Bin boundaries separating the linspace from 0 to 1.\n        norm (str, optional): Norm function to use when computing calibration error. Defaults to \"l1\".\n        debias (bool, optional): Apply debiasing to L2 norm computation as in\n            `Verified Uncertainty Calibration`_. Defaults to False.\n\n    Raises:\n        ValueError: If an unsupported norm function is provided.\n\n    Returns:\n        Tensor: Calibration error scalar.\n    \"\"\"\n    if norm not in {\"l1\", \"l2\", \"max\"}:\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    conf_bin = torch.zeros_like(bin_boundaries)\n    acc_bin = torch.zeros_like(bin_boundaries)\n    prop_bin = torch.zeros_like(bin_boundaries)\n    for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])):\n        # Calculated confidence and accuracy in each bin\n        in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())\n        prop_in_bin = in_bin.float().mean()\n        if prop_in_bin.item() > 0:\n            acc_bin[i] = accuracies[in_bin].float().mean()\n            conf_bin[i] = confidences[in_bin].mean()\n            prop_bin[i] = prop_in_bin\n\n    if norm == \"l1\":\n        ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin)\n    elif norm == \"max\":\n        ce = torch.max(torch.abs(acc_bin - conf_bin))\n    elif norm == \"l2\":\n        ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin)\n        # NOTE: debiasing is disabled in the wrapper functions. This implementation differs from that in sklearn.\n        if debias:\n            # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from\n            # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/\n            debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1)\n            ce += torch.sum(torch.nan_to_num(debias_bins))  # replace nans with zeros if nothing appeared in a bin\n        ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0)\n    return ce\n\n\ndef _ce_update(preds: Tensor, target: Tensor) -> Tuple[FloatTensor, FloatTensor]:\n    \"\"\"Given a predictions and targets tensor, computes the confidences of the top-1 prediction and records their\n    correctness.\n\n    Args:\n        preds (Tensor):  Input softmaxed predictions.\n        target (Tensor): Labels.\n\n    Raises:\n        ValueError: If the dataset shape is not binary, multiclass, or multidimensional-multiclass.\n\n    Returns:\n        Tuple[FloatTensor, FloatTensor]: [description]\n    \"\"\"\n    _, _, mode = _input_format_classification(preds, target)\n\n    if mode == DataType.BINARY:\n        confidences, accuracies = preds, target\n    elif mode == DataType.MULTICLASS:\n        confidences, predictions = preds.max(dim=1)\n        accuracies = predictions.eq(target)\n    elif mode == DataType.MULTIDIM_MULTICLASS:\n        # reshape tensors\n        # for preds, move the class dimension to the final axis and flatten the rest\n        confidences, predictions = torch.transpose(preds, 1, -1).flatten(0, -2).max(dim=1)\n        # for targets, just flatten the target\n        accuracies = predictions.eq(target.flatten())\n    else:\n        raise ValueError(\n            f\"Calibration error is not well-defined for data with size {preds.size()} and targets {target.size()}.\"\n        )\n    # must be cast to float for ddp allgather to work\n    return confidences.float(), accuracies.float()\n\n\ndef calibration_error(preds: Tensor, target: Tensor, n_bins: int = 15, norm: str = \"l1\") -> Tensor:\n    r\"\"\"\n    `Computes the Top-label Calibration Error`_\n\n    Three different norms are implemented, each corresponding to variations on the calibration error metric.\n\n    L1 norm (Expected Calibration Error)\n\n    .. math::\n        \\text{ECE} = \\frac{1}{N}\\sum_i^N \\|(p_i - c_i)\\|\n\n    Infinity norm (Maximum Calibration Error)\n\n    .. math::\n        \\text{RMSCE} =  \\max_{i} (p_i - c_i)\n\n    L2 norm (Root Mean Square Calibration Error)\n\n    .. math::\n        \\text{MCE} = \\frac{1}{N}\\sum_i^N (p_i - c_i)^2\n\n    Where :math:`p_i` is the top-1 prediction accuracy in\n    bin i and :math:`c_i` is the average confidence of predictions in bin i.\n\n    .. note:\n        L2-norm debiasing is not yet supported.\n\n    Args:\n        preds (Tensor): Model output probabilities.\n        target (Tensor): Ground-truth target class labels.\n        n_bins (int, optional): Number of bins to use when computing t. Defaults to 15.\n        norm (str, optional): Norm used to compare empirical and expected probability bins.\n            Defaults to \"l1\", or Expected Calibration Error.\n    \"\"\"\n    if norm not in (\"l1\", \"l2\", \"max\"):\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    if not isinstance(n_bins, int) or n_bins <= 0:\n        raise ValueError(f\"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}\")\n\n    confidences, accuracies = _ce_update(preds, target)\n\n    bin_boundaries = torch.linspace(0, 1, n_bins + 1, dtype=torch.float, device=preds.device)\n\n    return _ce_compute(confidences, accuracies, bin_boundaries, norm=norm)\n", "path": "torchmetrics/functional/classification/calibration_error.py"}], "after_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Any, List, Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torchmetrics.functional.classification.calibration_error import _ce_compute, _ce_update\nfrom torchmetrics.metric import Metric\nfrom torchmetrics.utilities.data import dim_zero_cat\n\n\nclass CalibrationError(Metric):\n    r\"\"\"\n\n    `Computes the Top-label Calibration Error`_\n    Three different norms are implemented, each corresponding to variations on the calibration error metric.\n\n    L1 norm (Expected Calibration Error)\n\n    .. math::\n        \\text{ECE} = \\sum_i^N b_i \\|(p_i - c_i)\\|\n\n    Infinity norm (Maximum Calibration Error)\n\n    .. math::\n        \\text{MCE} =  \\max_{i} (p_i - c_i)\n\n    L2 norm (Root Mean Square Calibration Error)\n\n    .. math::\n        \\text{RMSCE} = \\sqrt{\\sum_i^N b_i(p_i - c_i)^2}\n\n    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,\n    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and\n    :math:`b_i` is the fraction of data points in bin :math:`i`.\n\n    .. note::\n        L2-norm debiasing is not yet supported.\n\n    Args:\n        n_bins: Number of bins to use when computing probabilites and accuracies.\n        norm: Norm used to compare empirical and expected probability bins.\n            Defaults to \"l1\", or Expected Calibration Error.\n        debias: Applies debiasing term, only implemented for l2 norm. Defaults to True.\n        compute_on_step:  Forward only calls ``update()`` and return None if this is set to False.\n        dist_sync_on_step: Synchronize metric state across processes at each ``forward()``\n            before returning the value at the step\n        process_group: Specify the process group on which synchronization is called.\n    \"\"\"\n    DISTANCES = {\"l1\", \"l2\", \"max\"}\n    higher_is_better = False\n    confidences: List[Tensor]\n    accuracies: List[Tensor]\n\n    def __init__(\n        self,\n        n_bins: int = 15,\n        norm: str = \"l1\",\n        compute_on_step: bool = False,\n        dist_sync_on_step: bool = False,\n        process_group: Optional[Any] = None,\n    ):\n\n        super().__init__(\n            compute_on_step=compute_on_step,\n            dist_sync_on_step=dist_sync_on_step,\n            process_group=process_group,\n            dist_sync_fn=None,\n        )\n\n        if norm not in self.DISTANCES:\n            raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n        if not isinstance(n_bins, int) or n_bins <= 0:\n            raise ValueError(f\"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}\")\n        self.n_bins = n_bins\n        self.register_buffer(\"bin_boundaries\", torch.linspace(0, 1, n_bins + 1))\n        self.norm = norm\n\n        self.add_state(\"confidences\", [], dist_reduce_fx=\"cat\")\n        self.add_state(\"accuracies\", [], dist_reduce_fx=\"cat\")\n\n    def update(self, preds: Tensor, target: Tensor) -> None:  # type: ignore\n        \"\"\"Computes top-level confidences and accuracies for the input probabilites and appends them to internal\n        state.\n\n        Args:\n            preds (Tensor): Model output probabilities.\n            target (Tensor): Ground-truth target class labels.\n        \"\"\"\n        confidences, accuracies = _ce_update(preds, target)\n\n        self.confidences.append(confidences)\n        self.accuracies.append(accuracies)\n\n    def compute(self) -> Tensor:\n        \"\"\"Computes calibration error across all confidences and accuracies.\n\n        Returns:\n            Tensor: Calibration error across previously collected examples.\n        \"\"\"\n        confidences = dim_zero_cat(self.confidences)\n        accuracies = dim_zero_cat(self.accuracies)\n        return _ce_compute(confidences, accuracies, self.bin_boundaries, norm=self.norm)\n", "path": "torchmetrics/classification/calibration_error.py"}, {"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Tuple\n\nimport torch\nfrom torch import FloatTensor, Tensor\n\nfrom torchmetrics.utilities.checks import _input_format_classification\nfrom torchmetrics.utilities.enums import DataType\n\n\ndef _ce_compute(\n    confidences: FloatTensor,\n    accuracies: FloatTensor,\n    bin_boundaries: FloatTensor,\n    norm: str = \"l1\",\n    debias: bool = False,\n) -> Tensor:\n    \"\"\"Computes the calibration error given the provided bin boundaries and norm.\n\n    Args:\n        confidences (FloatTensor): The confidence (i.e. predicted prob) of the top1 prediction.\n        accuracies (FloatTensor): 1.0 if the top-1 prediction was correct, 0.0 otherwise.\n        bin_boundaries (FloatTensor): Bin boundaries separating the linspace from 0 to 1.\n        norm (str, optional): Norm function to use when computing calibration error. Defaults to \"l1\".\n        debias (bool, optional): Apply debiasing to L2 norm computation as in\n            `Verified Uncertainty Calibration`_. Defaults to False.\n\n    Raises:\n        ValueError: If an unsupported norm function is provided.\n\n    Returns:\n        Tensor: Calibration error scalar.\n    \"\"\"\n    if norm not in {\"l1\", \"l2\", \"max\"}:\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    conf_bin = torch.zeros_like(bin_boundaries)\n    acc_bin = torch.zeros_like(bin_boundaries)\n    prop_bin = torch.zeros_like(bin_boundaries)\n    for i, (bin_lower, bin_upper) in enumerate(zip(bin_boundaries[:-1], bin_boundaries[1:])):\n        # Calculated confidence and accuracy in each bin\n        in_bin = confidences.gt(bin_lower.item()) * confidences.le(bin_upper.item())\n        prop_in_bin = in_bin.float().mean()\n        if prop_in_bin.item() > 0:\n            acc_bin[i] = accuracies[in_bin].float().mean()\n            conf_bin[i] = confidences[in_bin].mean()\n            prop_bin[i] = prop_in_bin\n\n    if norm == \"l1\":\n        ce = torch.sum(torch.abs(acc_bin - conf_bin) * prop_bin)\n    elif norm == \"max\":\n        ce = torch.max(torch.abs(acc_bin - conf_bin))\n    elif norm == \"l2\":\n        ce = torch.sum(torch.pow(acc_bin - conf_bin, 2) * prop_bin)\n        # NOTE: debiasing is disabled in the wrapper functions. This implementation differs from that in sklearn.\n        if debias:\n            # the order here (acc_bin - 1 ) vs (1 - acc_bin) is flipped from\n            # the equation in Verified Uncertainty Prediction (Kumar et al 2019)/\n            debias_bins = (acc_bin * (acc_bin - 1) * prop_bin) / (prop_bin * accuracies.size()[0] - 1)\n            ce += torch.sum(torch.nan_to_num(debias_bins))  # replace nans with zeros if nothing appeared in a bin\n        ce = torch.sqrt(ce) if ce > 0 else torch.tensor(0)\n    return ce\n\n\ndef _ce_update(preds: Tensor, target: Tensor) -> Tuple[FloatTensor, FloatTensor]:\n    \"\"\"Given a predictions and targets tensor, computes the confidences of the top-1 prediction and records their\n    correctness.\n\n    Args:\n        preds (Tensor):  Input softmaxed predictions.\n        target (Tensor): Labels.\n\n    Raises:\n        ValueError: If the dataset shape is not binary, multiclass, or multidimensional-multiclass.\n\n    Returns:\n        Tuple[FloatTensor, FloatTensor]: [description]\n    \"\"\"\n    _, _, mode = _input_format_classification(preds, target)\n\n    if mode == DataType.BINARY:\n        confidences, accuracies = preds, target\n    elif mode == DataType.MULTICLASS:\n        confidences, predictions = preds.max(dim=1)\n        accuracies = predictions.eq(target)\n    elif mode == DataType.MULTIDIM_MULTICLASS:\n        # reshape tensors\n        # for preds, move the class dimension to the final axis and flatten the rest\n        confidences, predictions = torch.transpose(preds, 1, -1).flatten(0, -2).max(dim=1)\n        # for targets, just flatten the target\n        accuracies = predictions.eq(target.flatten())\n    else:\n        raise ValueError(\n            f\"Calibration error is not well-defined for data with size {preds.size()} and targets {target.size()}.\"\n        )\n    # must be cast to float for ddp allgather to work\n    return confidences.float(), accuracies.float()\n\n\ndef calibration_error(preds: Tensor, target: Tensor, n_bins: int = 15, norm: str = \"l1\") -> Tensor:\n    r\"\"\"\n    `Computes the Top-label Calibration Error`_\n\n    Three different norms are implemented, each corresponding to variations on the calibration error metric.\n\n    L1 norm (Expected Calibration Error)\n\n    .. math::\n        \\text{ECE} = \\sum_i^N b_i \\|(p_i - c_i)\\|\n\n    Infinity norm (Maximum Calibration Error)\n\n    .. math::\n        \\text{MCE} =  \\max_{i} (p_i - c_i)\n\n    L2 norm (Root Mean Square Calibration Error)\n\n    .. math::\n        \\text{RMSCE} = \\sqrt{\\sum_i^N b_i(p_i - c_i)^2}\n\n    Where :math:`p_i` is the top-1 prediction accuracy in bin :math:`i`,\n    :math:`c_i` is the average confidence of predictions in bin :math:`i`, and\n    :math:`b_i` is the fraction of data points in bin :math:`i`.\n\n    .. note:\n        L2-norm debiasing is not yet supported.\n\n    Args:\n        preds (Tensor): Model output probabilities.\n        target (Tensor): Ground-truth target class labels.\n        n_bins (int, optional): Number of bins to use when computing t. Defaults to 15.\n        norm (str, optional): Norm used to compare empirical and expected probability bins.\n            Defaults to \"l1\", or Expected Calibration Error.\n    \"\"\"\n    if norm not in (\"l1\", \"l2\", \"max\"):\n        raise ValueError(f\"Norm {norm} is not supported. Please select from l1, l2, or max. \")\n\n    if not isinstance(n_bins, int) or n_bins <= 0:\n        raise ValueError(f\"Expected argument `n_bins` to be a int larger than 0 but got {n_bins}\")\n\n    confidences, accuracies = _ce_update(preds, target)\n\n    bin_boundaries = torch.linspace(0, 1, n_bins + 1, dtype=torch.float, device=preds.device)\n\n    return _ce_compute(confidences, accuracies, bin_boundaries, norm=norm)\n", "path": "torchmetrics/functional/classification/calibration_error.py"}]}