We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
fix return type of get_label_quality_scores in token classification
The function only returns a `Tuple[np.ndarray, list]`, but it is annotated with:

https://github.com/cleanlab/cleanlab/blob/fad4eb266dee8b9e2925d3f0d74fe4a81939eb8a/cleanlab/token_classification/rank.py#L36

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `cleanlab/token_classification/rank.py`
Content:
```
1 # Copyright (C) 2017-2022  Cleanlab Inc.
2 # This file is part of cleanlab.
3 #
4 # cleanlab is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # cleanlab is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.
16 
17 """
18 Methods to rank and score sentences in a token classification dataset (text data), based on how likely they are to contain label errors.
19 """
20 
21 import pandas as pd
22 import numpy as np
23 from typing import List, Optional, Union, Tuple
24 
25 from cleanlab.rank import get_label_quality_scores as main_get_label_quality_scores
26 
27 
28 def get_label_quality_scores(
29     labels: list,
30     pred_probs: list,
31     *,
32     tokens: Optional[list] = None,
33     token_score_method: str = "self_confidence",
34     sentence_score_method: str = "min",
35     sentence_score_kwargs: dict = {},
36 ) -> Union[np.ndarray, Tuple[np.ndarray, list]]:
37     """
38     Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset.
39 
40     Each score is between 0 and 1.
41 
42     Lower scores indicate token labels that are less likely to be correct, or sentences that are more likely to contain a mislabeled token.
43 
44     Parameters
45     ----------
46     labels:
47         Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence.
48 
49         For a dataset with K classes, each label must be in 0, 1, ..., K-1.
50 
51     pred_probs:
52         List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens.
53 
54         Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence,
55         and contains model-predicted probabilities that `t` belongs to each of the K possible classes.
56 
57         Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1.
58 
59     tokens:
60         Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.
61 
62         These strings are used to annotated the returned `token_scores` object, see its documentation for more information.
63 
64     sentence_score_method: {"min", "softmin"}, default="min"
65         Method to aggregate individual token label quality scores into a single score for the sentence.
66 
67         - `min`: sentence score = minimum of token scores in the sentence
68         - `softmin`: sentence score = ``<s, softmax(1-s, t)>``, where `s` denotes the token label scores of the sentence, and ``<a, b> == np.dot(a, b)``.
69           Here parameter `t` controls the softmax temperature, such that the score converges toward `min` as ``t -> 0``.
70           Unlike `min`, `softmin` is affected by the scores of all tokens in the sentence.
71 
72     token_score_method: {"self_confidence", "normalized_margin", "confidence_weighted_entropy"}, default="self_confidence"
73         Label quality scoring method for each token.
74 
75         See :py:func:`cleanlab.rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` documentation for more info.
76 
77     sentence_score_kwargs:
78         Optional keyword arguments for `sentence_score_method` function (for advanced users only).
79 
80         See `cleanlab.token_classification.rank._softmin_sentence_score` for more info about keyword arguments supported for that scoring method.
81 
82     Returns
83     -------
84     sentence_scores:
85         Array of shape ``(N, )`` of scores between 0 and 1, one per sentence in the dataset.
86 
87         Lower scores indicate sentences more likely to contain a label issue.
88 
89     token_scores:
90         List of ``pd.Series``, such that `token_info[i]` contains the
91         label quality scores for individual tokens in the `i`-th sentence.
92 
93         If `tokens` strings were provided, they are used as index for each ``Series``.
94 
95     Examples
96     --------
97     >>> import numpy as np
98     >>> from cleanlab.token_classification.rank import get_label_quality_scores
99     >>> labels = [[0, 0, 1], [0, 1]]
100     >>> pred_probs = [
101     ...     np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]),
102     ...     np.array([[0.8, 0.2], [0.8, 0.2]]),
103     ... ]
104     >>> sentence_scores, token_scores = get_label_quality_scores(labels, pred_probs)
105     >>> sentence_scores
106     array([0.7, 0.2])
107     >>> token_scores
108     [0    0.90
109     1    0.70
110     2    0.95
111     dtype: float64, 0    0.8
112     1    0.2
113     dtype: float64]
114     """
115     methods = ["min", "softmin"]
116     assert sentence_score_method in methods, "Select from the following methods:\n%s" % "\n".join(
117         methods
118     )
119 
120     labels_flatten = np.array([l for label in labels for l in label])
121     pred_probs_flatten = np.array([p for pred_prob in pred_probs for p in pred_prob])
122 
123     sentence_length = [len(label) for label in labels]
124 
125     def nested_list(x, sentence_length):
126         i = iter(x)
127         return [[next(i) for _ in range(length)] for length in sentence_length]
128 
129     token_scores = main_get_label_quality_scores(
130         labels=labels_flatten, pred_probs=pred_probs_flatten, method=token_score_method
131     )
132     scores_nl = nested_list(token_scores, sentence_length)
133 
134     if sentence_score_method == "min":
135         sentence_scores = np.array(list(map(np.min, scores_nl)))
136     else:
137         assert sentence_score_method == "softmin"
138         temperature = sentence_score_kwargs.get("temperature", 0.05)
139         sentence_scores = _softmin_sentence_score(scores_nl, temperature=temperature)
140 
141     if tokens:
142         token_info = [pd.Series(scores, index=token) for scores, token in zip(scores_nl, tokens)]
143     else:
144         token_info = [pd.Series(scores) for scores in scores_nl]
145     return sentence_scores, token_info
146 
147 
148 def issues_from_scores(
149     sentence_scores: np.ndarray, *, token_scores: Optional[list] = None, threshold: float = 0.1
150 ) -> Union[list, np.ndarray]:
151     """
152     Converts scores output by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`
153     to a list of issues of similar format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`.
154 
155     Only considers as issues those tokens with label quality score lower than `threshold`.
156 
157     Issues are sorted by label quality score, from most severe to least.
158 
159     Parameters
160     ----------
161     sentence_scores:
162         Array of shape `(N, )` of overall sentence scores, where `N` is the number of sentences in the dataset.
163 
164         Same format as the `sentence_scores` returned by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`.
165 
166     token_scores:
167         Optional list such that `token_scores[i]` contains the individual token scores for the `i`-th sentence.
168 
169         Same format as the `token_scores` returned by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`.
170 
171     threshold:
172         Tokens (or sentences, if `token_scores` is not provided) with quality scores above the `threshold` are not
173         included in the result.
174 
175     Returns
176     ---------
177     issues:
178         List of label issues identified by comparing quality scores to threshold, such that each element is a tuple ``(i, j)``, which
179         indicates that the `j`-th token of the `i`-th sentence has a label issue.
180 
181         These tuples are ordered in `issues` list based on the token label quality score.
182 
183         Use :py:func:`token_classification.summary.display_issues <cleanlab.token_classification.summary.display_issues>`
184         to view these issues within the original sentences.
185 
186         If `token_scores` is not provided, returns array of integer indices (rather than tuples) of the sentences whose label quality score
187         falls below the `threshold` (also sorted by overall label quality score of each sentence).
188 
189     Examples
190     --------
191     >>> import numpy as np
192     >>> from cleanlab.token_classification.rank import issues_from_scores
193     >>> sentence_scores = np.array([0.1, 0.3, 0.6, 0.2, 0.05, 0.9, 0.8, 0.0125, 0.5, 0.6])
194     >>> issues_from_scores(sentence_scores)
195     array([7, 4])
196 
197     Changing the score threshold
198 
199     >>> issues_from_scores(sentence_scores, threshold=0.5)
200     array([7, 4, 0, 3, 1])
201 
202     Providing token scores along with sentence scores finds issues at the token level
203 
204     >>> token_scores = [
205     ...     [0.9, 0.6],
206     ...     [0.0, 0.8, 0.8],
207     ...     [0.8, 0.8],
208     ...     [0.1, 0.02, 0.3, 0.4],
209     ...     [0.1, 0.2, 0.03, 0.4],
210     ...     [0.1, 0.2, 0.3, 0.04],
211     ...     [0.1, 0.2, 0.4],
212     ...     [0.3, 0.4],
213     ...     [0.08, 0.2, 0.5, 0.4],
214     ...     [0.1, 0.2, 0.3, 0.4],
215     ... ]
216     >>> issues_from_scores(sentence_scores, token_scores=token_scores)
217     [(1, 0), (3, 1), (4, 2), (5, 3), (8, 0)]
218     """
219     if token_scores:
220         issues_with_scores = []
221         for sentence_index, scores in enumerate(token_scores):
222             for token_index, score in enumerate(scores):
223                 if score < threshold:
224                     issues_with_scores.append((sentence_index, token_index, score))
225 
226         issues_with_scores = sorted(issues_with_scores, key=lambda x: x[2])
227         issues = [(i, j) for i, j, _ in issues_with_scores]
228         return issues
229 
230     else:
231         ranking = np.argsort(sentence_scores)
232         cutoff = 0
233         while sentence_scores[ranking[cutoff]] < threshold and cutoff < len(ranking):
234             cutoff += 1
235         return ranking[:cutoff]
236 
237 
238 def _softmin_sentence_score(
239     token_scores: List[np.ndarray], *, temperature: float = 0.05
240 ) -> np.ndarray:
241     """
242     Sentence overall label quality scoring using the "softmin" method.
243 
244     Parameters
245     ----------
246     token_scores:
247         Per-token label quality scores in nested list format,
248         where `token_scores[i]` is a list of scores for each toke in the i'th sentence.
249 
250     temperature:
251         Temperature of the softmax function.
252 
253         Lower values encourage this method to converge toward the label quality score of the token with the lowest quality label in the sentence.
254 
255         Higher values encourage this method to converge toward the average label quality score of all tokens in the sentence.
256 
257     Returns
258     ---------
259     sentence_scores:
260         Array of shape ``(N, )``, where N is the number of sentences in the dataset, with one overall label quality score for each sentence.
261 
262     Examples
263     ---------
264     >>> from cleanlab.token_classification.rank import _softmin_sentence_score
265     >>> token_scores = [[0.9, 0.6], [0.0, 0.8, 0.8], [0.8]]
266     >>> _softmin_sentence_score(token_scores)
267     array([6.00741787e-01, 1.80056239e-07, 8.00000000e-01])
268     """
269     if temperature == 0:
270         return np.array([np.min(scores) for scores in token_scores])
271 
272     if temperature == np.inf:
273         return np.array([np.mean(scores) for scores in token_scores])
274 
275     def softmax(scores: np.ndarray) -> np.ndarray:
276         exp_scores = np.exp(scores / temperature)
277         return exp_scores / np.sum(exp_scores)
278 
279     def fun(scores: np.ndarray) -> float:
280         return np.dot(scores, softmax(1 - np.array(scores)))
281 
282     sentence_scores = list(map(fun, token_scores))
283     return np.array(sentence_scores)
284
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/cleanlab/token_classification/rank.py b/cleanlab/token_classification/rank.py
--- a/cleanlab/token_classification/rank.py
+++ b/cleanlab/token_classification/rank.py
@@ -33,7 +33,7 @@
     token_score_method: str = "self_confidence",
     sentence_score_method: str = "min",
     sentence_score_kwargs: dict = {},
-) -> Union[np.ndarray, Tuple[np.ndarray, list]]:
+) -> Tuple[np.ndarray, list]:
     """
     Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset.

{"golden_diff": "diff --git a/cleanlab/token_classification/rank.py b/cleanlab/token_classification/rank.py\n--- a/cleanlab/token_classification/rank.py\n+++ b/cleanlab/token_classification/rank.py\n@@ -33,7 +33,7 @@\n     token_score_method: str = \"self_confidence\",\n     sentence_score_method: str = \"min\",\n     sentence_score_kwargs: dict = {},\n-) -> Union[np.ndarray, Tuple[np.ndarray, list]]:\n+) -> Tuple[np.ndarray, list]:\n     \"\"\"\n     Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset.\n", "issue": "fix return type of get_label_quality_scores in token classification\nThe function only returns a `Tuple[np.ndarray, list]`, but it is annotated with:\r\n\r\nhttps://github.com/cleanlab/cleanlab/blob/fad4eb266dee8b9e2925d3f0d74fe4a81939eb8a/cleanlab/token_classification/rank.py#L36\n", "before_files": [{"content": "# Copyright (C) 2017-2022  Cleanlab Inc.\n# This file is part of cleanlab.\n#\n# cleanlab is free software: you can redistribute it and/or modify\n# it under the terms of the GNU Affero General Public License as published\n# by the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# cleanlab is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU Affero General Public License for more details.\n#\n# You should have received a copy of the GNU Affero General Public License\n# along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.\n\n\"\"\"\nMethods to rank and score sentences in a token classification dataset (text data), based on how likely they are to contain label errors.\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nfrom typing import List, Optional, Union, Tuple\n\nfrom cleanlab.rank import get_label_quality_scores as main_get_label_quality_scores\n\n\ndef get_label_quality_scores(\n    labels: list,\n    pred_probs: list,\n    *,\n    tokens: Optional[list] = None,\n    token_score_method: str = \"self_confidence\",\n    sentence_score_method: str = \"min\",\n    sentence_score_kwargs: dict = {},\n) -> Union[np.ndarray, Tuple[np.ndarray, list]]:\n    \"\"\"\n    Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset.\n\n    Each score is between 0 and 1.\n\n    Lower scores indicate token labels that are less likely to be correct, or sentences that are more likely to contain a mislabeled token.\n\n    Parameters\n    ----------\n    labels:\n        Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence.\n\n        For a dataset with K classes, each label must be in 0, 1, ..., K-1.\n\n    pred_probs:\n        List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens.\n\n        Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence,\n        and contains model-predicted probabilities that `t` belongs to each of the K possible classes.\n\n        Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1.\n\n    tokens:\n        Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.\n\n        These strings are used to annotated the returned `token_scores` object, see its documentation for more information.\n\n    sentence_score_method: {\"min\", \"softmin\"}, default=\"min\"\n        Method to aggregate individual token label quality scores into a single score for the sentence.\n\n        - `min`: sentence score = minimum of token scores in the sentence\n        - `softmin`: sentence score = ``<s, softmax(1-s, t)>``, where `s` denotes the token label scores of the sentence, and ``<a, b> == np.dot(a, b)``.\n          Here parameter `t` controls the softmax temperature, such that the score converges toward `min` as ``t -> 0``.\n          Unlike `min`, `softmin` is affected by the scores of all tokens in the sentence.\n\n    token_score_method: {\"self_confidence\", \"normalized_margin\", \"confidence_weighted_entropy\"}, default=\"self_confidence\"\n        Label quality scoring method for each token.\n\n        See :py:func:`cleanlab.rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` documentation for more info.\n\n    sentence_score_kwargs:\n        Optional keyword arguments for `sentence_score_method` function (for advanced users only).\n\n        See `cleanlab.token_classification.rank._softmin_sentence_score` for more info about keyword arguments supported for that scoring method.\n\n    Returns\n    -------\n    sentence_scores:\n        Array of shape ``(N, )`` of scores between 0 and 1, one per sentence in the dataset.\n\n        Lower scores indicate sentences more likely to contain a label issue.\n\n    token_scores:\n        List of ``pd.Series``, such that `token_info[i]` contains the\n        label quality scores for individual tokens in the `i`-th sentence.\n\n        If `tokens` strings were provided, they are used as index for each ``Series``.\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from cleanlab.token_classification.rank import get_label_quality_scores\n    >>> labels = [[0, 0, 1], [0, 1]]\n    >>> pred_probs = [\n    ...     np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]),\n    ...     np.array([[0.8, 0.2], [0.8, 0.2]]),\n    ... ]\n    >>> sentence_scores, token_scores = get_label_quality_scores(labels, pred_probs)\n    >>> sentence_scores\n    array([0.7, 0.2])\n    >>> token_scores\n    [0    0.90\n    1    0.70\n    2    0.95\n    dtype: float64, 0    0.8\n    1    0.2\n    dtype: float64]\n    \"\"\"\n    methods = [\"min\", \"softmin\"]\n    assert sentence_score_method in methods, \"Select from the following methods:\\n%s\" % \"\\n\".join(\n        methods\n    )\n\n    labels_flatten = np.array([l for label in labels for l in label])\n    pred_probs_flatten = np.array([p for pred_prob in pred_probs for p in pred_prob])\n\n    sentence_length = [len(label) for label in labels]\n\n    def nested_list(x, sentence_length):\n        i = iter(x)\n        return [[next(i) for _ in range(length)] for length in sentence_length]\n\n    token_scores = main_get_label_quality_scores(\n        labels=labels_flatten, pred_probs=pred_probs_flatten, method=token_score_method\n    )\n    scores_nl = nested_list(token_scores, sentence_length)\n\n    if sentence_score_method == \"min\":\n        sentence_scores = np.array(list(map(np.min, scores_nl)))\n    else:\n        assert sentence_score_method == \"softmin\"\n        temperature = sentence_score_kwargs.get(\"temperature\", 0.05)\n        sentence_scores = _softmin_sentence_score(scores_nl, temperature=temperature)\n\n    if tokens:\n        token_info = [pd.Series(scores, index=token) for scores, token in zip(scores_nl, tokens)]\n    else:\n        token_info = [pd.Series(scores) for scores in scores_nl]\n    return sentence_scores, token_info\n\n\ndef issues_from_scores(\n    sentence_scores: np.ndarray, *, token_scores: Optional[list] = None, threshold: float = 0.1\n) -> Union[list, np.ndarray]:\n    \"\"\"\n    Converts scores output by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`\n    to a list of issues of similar format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`.\n\n    Only considers as issues those tokens with label quality score lower than `threshold`.\n\n    Issues are sorted by label quality score, from most severe to least.\n\n    Parameters\n    ----------\n    sentence_scores:\n        Array of shape `(N, )` of overall sentence scores, where `N` is the number of sentences in the dataset.\n\n        Same format as the `sentence_scores` returned by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`.\n\n    token_scores:\n        Optional list such that `token_scores[i]` contains the individual token scores for the `i`-th sentence.\n\n        Same format as the `token_scores` returned by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`.\n\n    threshold:\n        Tokens (or sentences, if `token_scores` is not provided) with quality scores above the `threshold` are not\n        included in the result.\n\n    Returns\n    ---------\n    issues:\n        List of label issues identified by comparing quality scores to threshold, such that each element is a tuple ``(i, j)``, which\n        indicates that the `j`-th token of the `i`-th sentence has a label issue.\n\n        These tuples are ordered in `issues` list based on the token label quality score.\n\n        Use :py:func:`token_classification.summary.display_issues <cleanlab.token_classification.summary.display_issues>`\n        to view these issues within the original sentences.\n\n        If `token_scores` is not provided, returns array of integer indices (rather than tuples) of the sentences whose label quality score\n        falls below the `threshold` (also sorted by overall label quality score of each sentence).\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from cleanlab.token_classification.rank import issues_from_scores\n    >>> sentence_scores = np.array([0.1, 0.3, 0.6, 0.2, 0.05, 0.9, 0.8, 0.0125, 0.5, 0.6])\n    >>> issues_from_scores(sentence_scores)\n    array([7, 4])\n\n    Changing the score threshold\n\n    >>> issues_from_scores(sentence_scores, threshold=0.5)\n    array([7, 4, 0, 3, 1])\n\n    Providing token scores along with sentence scores finds issues at the token level\n\n    >>> token_scores = [\n    ...     [0.9, 0.6],\n    ...     [0.0, 0.8, 0.8],\n    ...     [0.8, 0.8],\n    ...     [0.1, 0.02, 0.3, 0.4],\n    ...     [0.1, 0.2, 0.03, 0.4],\n    ...     [0.1, 0.2, 0.3, 0.04],\n    ...     [0.1, 0.2, 0.4],\n    ...     [0.3, 0.4],\n    ...     [0.08, 0.2, 0.5, 0.4],\n    ...     [0.1, 0.2, 0.3, 0.4],\n    ... ]\n    >>> issues_from_scores(sentence_scores, token_scores=token_scores)\n    [(1, 0), (3, 1), (4, 2), (5, 3), (8, 0)]\n    \"\"\"\n    if token_scores:\n        issues_with_scores = []\n        for sentence_index, scores in enumerate(token_scores):\n            for token_index, score in enumerate(scores):\n                if score < threshold:\n                    issues_with_scores.append((sentence_index, token_index, score))\n\n        issues_with_scores = sorted(issues_with_scores, key=lambda x: x[2])\n        issues = [(i, j) for i, j, _ in issues_with_scores]\n        return issues\n\n    else:\n        ranking = np.argsort(sentence_scores)\n        cutoff = 0\n        while sentence_scores[ranking[cutoff]] < threshold and cutoff < len(ranking):\n            cutoff += 1\n        return ranking[:cutoff]\n\n\ndef _softmin_sentence_score(\n    token_scores: List[np.ndarray], *, temperature: float = 0.05\n) -> np.ndarray:\n    \"\"\"\n    Sentence overall label quality scoring using the \"softmin\" method.\n\n    Parameters\n    ----------\n    token_scores:\n        Per-token label quality scores in nested list format,\n        where `token_scores[i]` is a list of scores for each toke in the i'th sentence.\n\n    temperature:\n        Temperature of the softmax function.\n\n        Lower values encourage this method to converge toward the label quality score of the token with the lowest quality label in the sentence.\n\n        Higher values encourage this method to converge toward the average label quality score of all tokens in the sentence.\n\n    Returns\n    ---------\n    sentence_scores:\n        Array of shape ``(N, )``, where N is the number of sentences in the dataset, with one overall label quality score for each sentence.\n\n    Examples\n    ---------\n    >>> from cleanlab.token_classification.rank import _softmin_sentence_score\n    >>> token_scores = [[0.9, 0.6], [0.0, 0.8, 0.8], [0.8]]\n    >>> _softmin_sentence_score(token_scores)\n    array([6.00741787e-01, 1.80056239e-07, 8.00000000e-01])\n    \"\"\"\n    if temperature == 0:\n        return np.array([np.min(scores) for scores in token_scores])\n\n    if temperature == np.inf:\n        return np.array([np.mean(scores) for scores in token_scores])\n\n    def softmax(scores: np.ndarray) -> np.ndarray:\n        exp_scores = np.exp(scores / temperature)\n        return exp_scores / np.sum(exp_scores)\n\n    def fun(scores: np.ndarray) -> float:\n        return np.dot(scores, softmax(1 - np.array(scores)))\n\n    sentence_scores = list(map(fun, token_scores))\n    return np.array(sentence_scores)\n", "path": "cleanlab/token_classification/rank.py"}], "after_files": [{"content": "# Copyright (C) 2017-2022  Cleanlab Inc.\n# This file is part of cleanlab.\n#\n# cleanlab is free software: you can redistribute it and/or modify\n# it under the terms of the GNU Affero General Public License as published\n# by the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# cleanlab is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU Affero General Public License for more details.\n#\n# You should have received a copy of the GNU Affero General Public License\n# along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.\n\n\"\"\"\nMethods to rank and score sentences in a token classification dataset (text data), based on how likely they are to contain label errors.\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nfrom typing import List, Optional, Union, Tuple\n\nfrom cleanlab.rank import get_label_quality_scores as main_get_label_quality_scores\n\n\ndef get_label_quality_scores(\n    labels: list,\n    pred_probs: list,\n    *,\n    tokens: Optional[list] = None,\n    token_score_method: str = \"self_confidence\",\n    sentence_score_method: str = \"min\",\n    sentence_score_kwargs: dict = {},\n) -> Tuple[np.ndarray, list]:\n    \"\"\"\n    Returns overall quality scores for the labels in each sentence, as well as for the individual tokens' labels in a token classification dataset.\n\n    Each score is between 0 and 1.\n\n    Lower scores indicate token labels that are less likely to be correct, or sentences that are more likely to contain a mislabeled token.\n\n    Parameters\n    ----------\n    labels:\n        Nested list of given labels for all tokens, such that `labels[i]` is a list of labels, one for each token in the `i`-th sentence.\n\n        For a dataset with K classes, each label must be in 0, 1, ..., K-1.\n\n    pred_probs:\n        List of np arrays, such that `pred_probs[i]` has shape ``(T, K)`` if the `i`-th sentence contains T tokens.\n\n        Each row of `pred_probs[i]` corresponds to a token `t` in the `i`-th sentence,\n        and contains model-predicted probabilities that `t` belongs to each of the K possible classes.\n\n        Columns of each `pred_probs[i]` should be ordered such that the probabilities correspond to class 0, 1, ..., K-1.\n\n    tokens:\n        Nested list such that `tokens[i]` is a list of tokens (strings/words) that comprise the `i`-th sentence.\n\n        These strings are used to annotated the returned `token_scores` object, see its documentation for more information.\n\n    sentence_score_method: {\"min\", \"softmin\"}, default=\"min\"\n        Method to aggregate individual token label quality scores into a single score for the sentence.\n\n        - `min`: sentence score = minimum of token scores in the sentence\n        - `softmin`: sentence score = ``<s, softmax(1-s, t)>``, where `s` denotes the token label scores of the sentence, and ``<a, b> == np.dot(a, b)``.\n          Here parameter `t` controls the softmax temperature, such that the score converges toward `min` as ``t -> 0``.\n          Unlike `min`, `softmin` is affected by the scores of all tokens in the sentence.\n\n    token_score_method: {\"self_confidence\", \"normalized_margin\", \"confidence_weighted_entropy\"}, default=\"self_confidence\"\n        Label quality scoring method for each token.\n\n        See :py:func:`cleanlab.rank.get_label_quality_scores <cleanlab.rank.get_label_quality_scores>` documentation for more info.\n\n    sentence_score_kwargs:\n        Optional keyword arguments for `sentence_score_method` function (for advanced users only).\n\n        See `cleanlab.token_classification.rank._softmin_sentence_score` for more info about keyword arguments supported for that scoring method.\n\n    Returns\n    -------\n    sentence_scores:\n        Array of shape ``(N, )`` of scores between 0 and 1, one per sentence in the dataset.\n\n        Lower scores indicate sentences more likely to contain a label issue.\n\n    token_scores:\n        List of ``pd.Series``, such that `token_info[i]` contains the\n        label quality scores for individual tokens in the `i`-th sentence.\n\n        If `tokens` strings were provided, they are used as index for each ``Series``.\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from cleanlab.token_classification.rank import get_label_quality_scores\n    >>> labels = [[0, 0, 1], [0, 1]]\n    >>> pred_probs = [\n    ...     np.array([[0.9, 0.1], [0.7, 0.3], [0.05, 0.95]]),\n    ...     np.array([[0.8, 0.2], [0.8, 0.2]]),\n    ... ]\n    >>> sentence_scores, token_scores = get_label_quality_scores(labels, pred_probs)\n    >>> sentence_scores\n    array([0.7, 0.2])\n    >>> token_scores\n    [0    0.90\n    1    0.70\n    2    0.95\n    dtype: float64, 0    0.8\n    1    0.2\n    dtype: float64]\n    \"\"\"\n    methods = [\"min\", \"softmin\"]\n    assert sentence_score_method in methods, \"Select from the following methods:\\n%s\" % \"\\n\".join(\n        methods\n    )\n\n    labels_flatten = np.array([l for label in labels for l in label])\n    pred_probs_flatten = np.array([p for pred_prob in pred_probs for p in pred_prob])\n\n    sentence_length = [len(label) for label in labels]\n\n    def nested_list(x, sentence_length):\n        i = iter(x)\n        return [[next(i) for _ in range(length)] for length in sentence_length]\n\n    token_scores = main_get_label_quality_scores(\n        labels=labels_flatten, pred_probs=pred_probs_flatten, method=token_score_method\n    )\n    scores_nl = nested_list(token_scores, sentence_length)\n\n    if sentence_score_method == \"min\":\n        sentence_scores = np.array(list(map(np.min, scores_nl)))\n    else:\n        assert sentence_score_method == \"softmin\"\n        temperature = sentence_score_kwargs.get(\"temperature\", 0.05)\n        sentence_scores = _softmin_sentence_score(scores_nl, temperature=temperature)\n\n    if tokens:\n        token_info = [pd.Series(scores, index=token) for scores, token in zip(scores_nl, tokens)]\n    else:\n        token_info = [pd.Series(scores) for scores in scores_nl]\n    return sentence_scores, token_info\n\n\ndef issues_from_scores(\n    sentence_scores: np.ndarray, *, token_scores: Optional[list] = None, threshold: float = 0.1\n) -> Union[list, np.ndarray]:\n    \"\"\"\n    Converts scores output by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`\n    to a list of issues of similar format as output by :py:func:`token_classification.filter.find_label_issues <cleanlab.token_classification.filter.find_label_issues>`.\n\n    Only considers as issues those tokens with label quality score lower than `threshold`.\n\n    Issues are sorted by label quality score, from most severe to least.\n\n    Parameters\n    ----------\n    sentence_scores:\n        Array of shape `(N, )` of overall sentence scores, where `N` is the number of sentences in the dataset.\n\n        Same format as the `sentence_scores` returned by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`.\n\n    token_scores:\n        Optional list such that `token_scores[i]` contains the individual token scores for the `i`-th sentence.\n\n        Same format as the `token_scores` returned by :py:func:`token_classification.rank.get_label_quality_scores <cleanlab.token_classification.rank.get_label_quality_scores>`.\n\n    threshold:\n        Tokens (or sentences, if `token_scores` is not provided) with quality scores above the `threshold` are not\n        included in the result.\n\n    Returns\n    ---------\n    issues:\n        List of label issues identified by comparing quality scores to threshold, such that each element is a tuple ``(i, j)``, which\n        indicates that the `j`-th token of the `i`-th sentence has a label issue.\n\n        These tuples are ordered in `issues` list based on the token label quality score.\n\n        Use :py:func:`token_classification.summary.display_issues <cleanlab.token_classification.summary.display_issues>`\n        to view these issues within the original sentences.\n\n        If `token_scores` is not provided, returns array of integer indices (rather than tuples) of the sentences whose label quality score\n        falls below the `threshold` (also sorted by overall label quality score of each sentence).\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from cleanlab.token_classification.rank import issues_from_scores\n    >>> sentence_scores = np.array([0.1, 0.3, 0.6, 0.2, 0.05, 0.9, 0.8, 0.0125, 0.5, 0.6])\n    >>> issues_from_scores(sentence_scores)\n    array([7, 4])\n\n    Changing the score threshold\n\n    >>> issues_from_scores(sentence_scores, threshold=0.5)\n    array([7, 4, 0, 3, 1])\n\n    Providing token scores along with sentence scores finds issues at the token level\n\n    >>> token_scores = [\n    ...     [0.9, 0.6],\n    ...     [0.0, 0.8, 0.8],\n    ...     [0.8, 0.8],\n    ...     [0.1, 0.02, 0.3, 0.4],\n    ...     [0.1, 0.2, 0.03, 0.4],\n    ...     [0.1, 0.2, 0.3, 0.04],\n    ...     [0.1, 0.2, 0.4],\n    ...     [0.3, 0.4],\n    ...     [0.08, 0.2, 0.5, 0.4],\n    ...     [0.1, 0.2, 0.3, 0.4],\n    ... ]\n    >>> issues_from_scores(sentence_scores, token_scores=token_scores)\n    [(1, 0), (3, 1), (4, 2), (5, 3), (8, 0)]\n    \"\"\"\n    if token_scores:\n        issues_with_scores = []\n        for sentence_index, scores in enumerate(token_scores):\n            for token_index, score in enumerate(scores):\n                if score < threshold:\n                    issues_with_scores.append((sentence_index, token_index, score))\n\n        issues_with_scores = sorted(issues_with_scores, key=lambda x: x[2])\n        issues = [(i, j) for i, j, _ in issues_with_scores]\n        return issues\n\n    else:\n        ranking = np.argsort(sentence_scores)\n        cutoff = 0\n        while sentence_scores[ranking[cutoff]] < threshold and cutoff < len(ranking):\n            cutoff += 1\n        return ranking[:cutoff]\n\n\ndef _softmin_sentence_score(\n    token_scores: List[np.ndarray], *, temperature: float = 0.05\n) -> np.ndarray:\n    \"\"\"\n    Sentence overall label quality scoring using the \"softmin\" method.\n\n    Parameters\n    ----------\n    token_scores:\n        Per-token label quality scores in nested list format,\n        where `token_scores[i]` is a list of scores for each toke in the i'th sentence.\n\n    temperature:\n        Temperature of the softmax function.\n\n        Lower values encourage this method to converge toward the label quality score of the token with the lowest quality label in the sentence.\n\n        Higher values encourage this method to converge toward the average label quality score of all tokens in the sentence.\n\n    Returns\n    ---------\n    sentence_scores:\n        Array of shape ``(N, )``, where N is the number of sentences in the dataset, with one overall label quality score for each sentence.\n\n    Examples\n    ---------\n    >>> from cleanlab.token_classification.rank import _softmin_sentence_score\n    >>> token_scores = [[0.9, 0.6], [0.0, 0.8, 0.8], [0.8]]\n    >>> _softmin_sentence_score(token_scores)\n    array([6.00741787e-01, 1.80056239e-07, 8.00000000e-01])\n    \"\"\"\n    if temperature == 0:\n        return np.array([np.min(scores) for scores in token_scores])\n\n    if temperature == np.inf:\n        return np.array([np.mean(scores) for scores in token_scores])\n\n    def softmax(scores: np.ndarray) -> np.ndarray:\n        exp_scores = np.exp(scores / temperature)\n        return exp_scores / np.sum(exp_scores)\n\n    def fun(scores: np.ndarray) -> float:\n        return np.dot(scores, softmax(1 - np.array(scores)))\n\n    sentence_scores = list(map(fun, token_scores))\n    return np.array(sentence_scores)\n", "path": "cleanlab/token_classification/rank.py"}]}