{
    "file_diffs": [
        {
            "old_file_content": "from collections import namedtuple\n\nimport numpy as np\n\nfrom AnyQt.QtCore import Qt, QSize\nfrom AnyQt.QtWidgets import QListWidget, QSizePolicy\n\nimport pyqtgraph as pg\n\nfrom orangewidget.utils.visual_settings_dlg import VisualSettingsDialog\n\nfrom Orange.base import Model\nfrom Orange.classification import ThresholdClassifier, CalibratedLearner\nfrom Orange.evaluation import Results\nfrom Orange.evaluation.performance_curves import Curves\nfrom Orange.widgets import widget, gui, settings\nfrom Orange.widgets.evaluate.contexthandlers import \\\n    EvaluationResultsContextHandler\nfrom Orange.widgets.evaluate.utils import results_for_preview\nfrom Orange.widgets.utils import colorpalettes\nfrom Orange.widgets.utils.widgetpreview import WidgetPreview\nfrom Orange.widgets.visualize.utils.customizableplot import \\\n    CommonParameterSetter\nfrom Orange.widgets.visualize.utils.plotutils import GraphicsView, PlotItem\nfrom Orange.widgets.widget import Input, Output, Msg\nfrom Orange.widgets import report\n\n\nMetricDefinition = namedtuple(\n    \"metric_definition\",\n    (\"name\", \"functions\", \"short_names\", \"explanation\"))\n\nMetrics = [MetricDefinition(*args) for args in (\n    (\"Calibration curve\", None, (), \"\"),\n    (\"Classification accuracy\", (Curves.ca, ), (), \"\"),\n    (\"F1\", (Curves.f1, ), (), \"\"),\n    (\"Sensitivity and specificity\",\n     (Curves.sensitivity, Curves.specificity),\n     (\"sens\", \"spec\"),\n     \"<p><b>Sensitivity</b> (falling) is the proportion of correctly \"\n     \"detected positive instances (TP&nbsp;/&nbsp;P).</p>\"\n     \"<p><b>Specificity</b> (rising) is the proportion of detected \"\n     \"negative instances (TN&nbsp;/&nbsp;N).</p>\"),\n    (\"Precision and recall\",\n     (Curves.precision, Curves.recall),\n     (\"prec\", \"recall\"),\n     \"<p><b>Precision</b> (rising) is the fraction of retrieved instances \"\n     \"that are relevant, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>\"\n     \"<p><b>Recall</b> (falling) is the proportion of discovered relevant \"\n     \"instances, TP&nbsp;/&nbsp;P.</p>\"),\n    (\"Pos and neg predictive value\",\n     (Curves.ppv, Curves.npv),\n     (\"PPV\", \"TPV\"),\n     \"<p><b>Positive predictive value</b> (rising) is the proportion of \"\n     \"correct positives, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>\"\n     \"<p><b>Negative predictive value</b> is the proportion of correct \"\n     \"negatives, TN&nbsp;/&nbsp;(TN&nbsp;+&nbsp;FN).</p>\"),\n    (\"True and false positive rate\",\n     (Curves.tpr, Curves.fpr),\n     (\"TPR\", \"FPR\"),\n     \"<p><b>True and false positive rate</b> are proportions of detected \"\n     \"and omitted positive instances</p>\"),\n)]\n\n\nclass ParameterSetter(CommonParameterSetter):\n\n    def __init__(self, master):\n        super().__init__()\n        self.master = master\n\n    def update_setters(self):\n        self.initial_settings = {\n            self.LABELS_BOX: {\n                self.FONT_FAMILY_LABEL: self.FONT_FAMILY_SETTING,\n                self.TITLE_LABEL: self.FONT_SETTING,\n                self.AXIS_TITLE_LABEL: self.FONT_SETTING,\n                self.AXIS_TICKS_LABEL: self.FONT_SETTING,\n            },\n            self.ANNOT_BOX: {\n                self.TITLE_LABEL: {self.TITLE_LABEL: (\"\", \"\")},\n            }\n        }\n\n    @property\n    def title_item(self):\n        return self.master.titleLabel\n\n    @property\n    def axis_items(self):\n        return [value[\"item\"] for value in self.master.axes.values()]\n\n\nclass OWCalibrationPlot(widget.OWWidget):\n    name = \"Calibration Plot\"\n    description = \"Calibration plot based on evaluation of classifiers.\"\n    icon = \"icons/CalibrationPlot.svg\"\n    priority = 1030\n    keywords = []\n\n    class Inputs:\n        evaluation_results = Input(\"Evaluation Results\", Results)\n\n    class Outputs:\n        calibrated_model = Output(\"Calibrated Model\", Model)\n\n    class Error(widget.OWWidget.Error):\n        non_discrete_target = Msg(\"Calibration plot requires a categorical \"\n                                  \"target variable.\")\n        empty_input = widget.Msg(\"Empty result on input. Nothing to display.\")\n        nan_classes = \\\n            widget.Msg(\"Remove test data instances with unknown classes.\")\n        all_target_class = widget.Msg(\n            \"All data instances belong to target class.\")\n        no_target_class = widget.Msg(\n            \"No data instances belong to target class.\")\n\n    class Warning(widget.OWWidget.Warning):\n        omitted_folds = widget.Msg(\n            \"Test folds where all data belongs to (non)-target are not shown.\")\n        omitted_nan_prob_points = widget.Msg(\n            \"Instance for which the model couldn't compute probabilities are\"\n            \"skipped.\")\n        no_valid_data = widget.Msg(\"No valid data for model(s) {}\")\n\n    class Information(widget.OWWidget.Information):\n        no_output = Msg(\"Can't output a model: {}\")\n\n    settingsHandler = EvaluationResultsContextHandler()\n    target_index = settings.ContextSetting(0)\n    selected_classifiers = settings.ContextSetting([])\n    score = settings.Setting(0)\n    output_calibration = settings.Setting(0)\n    fold_curves = settings.Setting(False)\n    display_rug = settings.Setting(True)\n    threshold = settings.Setting(0.5)\n    visual_settings = settings.Setting({}, schema_only=True)\n    auto_commit = settings.Setting(True)\n\n    graph_name = \"plot\"\n\n    def __init__(self):\n        super().__init__()\n\n        self.results = None\n        self.scores = None\n        self.classifier_names = []\n        self.colors = []\n        self.line = None\n\n        self._last_score_value = -1\n\n        box = gui.vBox(self.controlArea, box=\"Settings\")\n        self.target_cb = gui.comboBox(\n            box, self, \"target_index\", label=\"Target:\",\n            orientation=Qt.Horizontal, callback=self.target_index_changed,\n            contentsLength=8, searchable=True)\n        gui.checkBox(\n            box, self, \"display_rug\", \"Show rug\",\n            callback=self._on_display_rug_changed)\n        gui.checkBox(\n            box, self, \"fold_curves\", \"Curves for individual folds\",\n            callback=self._replot)\n\n        self.classifiers_list_box = gui.listBox(\n            self.controlArea, self, \"selected_classifiers\", \"classifier_names\",\n            box=\"Classifier\", selectionMode=QListWidget.ExtendedSelection,\n            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),\n            sizeHint=QSize(150, 40),\n            callback=self._on_selection_changed)\n\n        box = gui.vBox(self.controlArea, \"Metrics\")\n        combo = gui.comboBox(\n            box, self, \"score\", items=(metric.name for metric in Metrics),\n            callback=self.score_changed)\n\n        self.explanation = gui.widgetLabel(\n            box, wordWrap=True, fixedWidth=combo.sizeHint().width())\n        self.explanation.setContentsMargins(8, 8, 0, 0)\n        font = self.explanation.font()\n        font.setPointSizeF(0.85 * font.pointSizeF())\n        self.explanation.setFont(font)\n\n        gui.radioButtons(\n            box, self, value=\"output_calibration\",\n            btnLabels=(\"Sigmoid calibration\", \"Isotonic calibration\"),\n            label=\"Output model calibration\", callback=self.commit.deferred)\n\n        self.info_box = gui.widgetBox(self.controlArea, \"Info\")\n        self.info_label = gui.widgetLabel(self.info_box)\n\n        gui.auto_apply(self.buttonsArea, self, \"auto_commit\")\n\n        self.plotview = GraphicsView()\n        self.plot = PlotItem(enableMenu=False)\n        self.plot.parameter_setter = ParameterSetter(self.plot)\n        self.plot.setMouseEnabled(False, False)\n        self.plot.hideButtons()\n        for axis_name in (\"bottom\", \"left\"):\n            axis = self.plot.getAxis(axis_name)\n            # Remove the condition (that is, allow setting this for bottom\n            # axis) when pyqtgraph is fixed\n            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930\n            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932\n            if axis_name != \"bottom\":  # remove if when pyqtgraph is fixed\n                axis.setStyle(stopAxisAtTick=(True, True))\n\n        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)\n        self.plotview.setCentralItem(self.plot)\n\n        self.mainArea.layout().addWidget(self.plotview)\n        self._set_explanation()\n\n        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)\n\n    @Inputs.evaluation_results\n    def set_results(self, results):\n        self.closeContext()\n        self.clear()\n        self.Error.clear()\n        self.Information.clear()\n\n        self.results = None\n        if results is not None:\n            if not results.domain.has_discrete_class:\n                self.Error.non_discrete_target()\n            elif not results.actual.size:\n                self.Error.empty_input()\n            elif np.any(np.isnan(results.actual)):\n                self.Error.nan_classes()\n            else:\n                self.results = results\n                self._initialize(results)\n                class_var = self.results.domain.class_var\n                self.target_index = int(len(class_var.values) == 2)\n                self.openContext(class_var, self.classifier_names)\n                self._replot()\n\n        self.commit.now()\n\n    def clear(self):\n        self.plot.clear()\n        self.results = None\n        self.classifier_names = []\n        self.selected_classifiers = []\n        self.target_cb.clear()\n        self.colors = []\n\n    def target_index_changed(self):\n        if len(self.results.domain.class_var.values) == 2:\n            self.threshold = 1 - self.threshold\n        self._set_explanation()\n        self._replot()\n        self.commit.deferred()\n\n    def score_changed(self):\n        self._set_explanation()\n        self._replot()\n        if self._last_score_value != self.score:\n            self.commit.deferred()\n            self._last_score_value = self.score\n\n    def _set_explanation(self):\n        explanation = Metrics[self.score].explanation\n        if explanation:\n            self.explanation.setText(explanation)\n            self.explanation.show()\n        else:\n            self.explanation.hide()\n\n        if self.score == 0:\n            self.controls.output_calibration.show()\n            self.info_box.hide()\n        else:\n            self.controls.output_calibration.hide()\n            self.info_box.show()\n\n        axis = self.plot.getAxis(\"bottom\")\n        axis.setLabel(\"Predicted probability\" if self.score == 0\n                      else \"Threshold probability to classify as positive\")\n\n        axis = self.plot.getAxis(\"left\")\n        axis.setLabel(Metrics[self.score].name)\n\n    def _initialize(self, results):\n        n = len(results.predicted)\n        names = getattr(results, \"learner_names\", None)\n        if names is None:\n            names = [\"#{}\".format(i + 1) for i in range(n)]\n\n        self.classifier_names = names\n        self.colors = colorpalettes.get_default_curve_colors(n)\n\n        for i in range(n):\n            item = self.classifiers_list_box.item(i)\n            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))\n\n        self.selected_classifiers = list(range(n))\n        self.target_cb.addItems(results.domain.class_var.values)\n        self.target_index = 0\n\n    def _rug(self, data, pen_args):\n        color = pen_args[\"pen\"].color()\n        rh = 0.025\n        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]\n        rug_x_true = rug_x[data.ytrue].ravel()\n        rug_x_false = rug_x[~data.ytrue].ravel()\n\n        rug_y_true = np.ones_like(rug_x_true)\n        rug_y_true[1::2] = 1 - rh\n        rug_y_false = np.zeros_like(rug_x_false)\n        rug_y_false[1::2] = rh\n\n        self.plot.plot(\n            rug_x_false, rug_y_false,\n            pen=color, connect=\"pairs\", antialias=True)\n        self.plot.plot(\n            rug_x_true, rug_y_true,\n            pen=color, connect=\"pairs\", antialias=True)\n\n    def plot_metrics(self, data, metrics, pen_args):\n        if metrics is None:\n            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)\n        ys = [metric(data) for metric in metrics]\n        for y in ys:\n            self.plot.plot(data.probs, y, **pen_args)\n        return data.probs, ys\n\n    def _prob_curve(self, ytrue, probs, pen_args):\n        xmin, xmax = probs.min(), probs.max()\n        x = np.linspace(xmin, xmax, 100)\n        if xmax != xmin:\n            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))\n            y = f(x)\n        else:\n            y = np.full(100, xmax)\n\n        self.plot.plot(x, y, symbol=\"+\", symbolSize=4, **pen_args)\n        return x, (y, )\n\n    def _setup_plot(self):\n        target = self.target_index\n        results = self.results\n        metrics = Metrics[self.score].functions\n        plot_folds = self.fold_curves and results.folds is not None\n        self.scores = []\n\n        if not self._check_class_presence(results.actual == target):\n            return\n\n        self.Warning.omitted_folds.clear()\n        self.Warning.omitted_nan_prob_points.clear()\n        no_valid_models = []\n        shadow_width = 4 + 4 * plot_folds\n        for clsf in self.selected_classifiers:\n            data = Curves.from_results(results, target, clsf)\n            if data.tot == 0:  # all probabilities are nan\n                no_valid_models.append(clsf)\n                continue\n            if data.tot != results.probabilities.shape[1]:  # some are nan\n                self.Warning.omitted_nan_prob_points()\n\n            color = self.colors[clsf]\n            pen_args = dict(\n                pen=pg.mkPen(color, width=1), antiAlias=True,\n                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))\n            self.scores.append(\n                (self.classifier_names[clsf],\n                 self.plot_metrics(data, metrics, pen_args)))\n\n            if self.display_rug:\n                self._rug(data, pen_args)\n\n            if plot_folds:\n                pen_args = dict(\n                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),\n                    antiAlias=True)\n                for fold in range(len(results.folds)):\n                    fold_results = results.get_fold(fold)\n                    fold_curve = Curves.from_results(fold_results, target, clsf)\n                    # Can't check this before: p and n can be 0 because of\n                    # nan probabilities\n                    if fold_curve.p * fold_curve.n == 0:\n                        self.Warning.omitted_folds()\n                    self.plot_metrics(fold_curve, metrics, pen_args)\n\n        if no_valid_models:\n            self.Warning.no_valid_data(\n                \", \".join(self.classifier_names[i] for i in no_valid_models))\n\n        if self.score == 0:\n            self.plot.plot([0, 1], [0, 1], antialias=True)\n        else:\n            self.line = pg.InfiniteLine(\n                pos=self.threshold, movable=True,\n                pen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=2),\n                hoverPen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=3),\n                bounds=(0, 1),\n            )\n            self.line.sigPositionChanged.connect(self.threshold_change)\n            self.line.sigPositionChangeFinished.connect(\n                self.threshold_change_done)\n            self.plot.addItem(self.line)\n\n    def _check_class_presence(self, ytrue):\n        self.Error.all_target_class.clear()\n        self.Error.no_target_class.clear()\n        if np.max(ytrue) == 0:\n            self.Error.no_target_class()\n            return False\n        if np.min(ytrue) == 1:\n            self.Error.all_target_class()\n            return False\n        return True\n\n    def _replot(self):\n        self.plot.clear()\n        if self.results is not None:\n            self._setup_plot()\n        self._update_info()\n\n    def _on_display_rug_changed(self):\n        self._replot()\n\n    def _on_selection_changed(self):\n        self._replot()\n        self.commit.deferred()\n\n    def threshold_change(self):\n        self.threshold = round(self.line.pos().x(), 2)\n        self.line.setPos(self.threshold)\n        self._update_info()\n\n    def get_info_text(self, short):\n        if short:\n            def elided(s):\n                return s[:17] + \"...\" if len(s) > 20 else s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold: p=</th>\n                                <td colspan='4'>{self.threshold:.2f}<br/></td>\n                            </tr>\"\"\"\n\n        else:\n            def elided(s):\n                return s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold:</th>\n                                <td colspan='4'>p = {self.threshold:.2f}<br/>\n                                </td>\n                                <tr/>\n                            </tr>\"\"\"\n\n        if self.scores is not None:\n            short_names = Metrics[self.score].short_names\n            if short_names:\n                text += f\"\"\"<tr>\n                                <th></th>\n                                {\"<td></td>\".join(f\"<td align='right'>{n}</td>\"\n                                                  for n in short_names)}\n                            </tr>\"\"\"\n            for name, (probs, curves) in self.scores:\n                ind = min(np.searchsorted(probs, self.threshold),\n                          len(probs) - 1)\n                text += f\"<tr><th align='right'>{elided(name)}:</th>\"\n                text += \"<td>/</td>\".join(f'<td>{curve[ind]:.3f}</td>'\n                                          for curve in curves)\n                text += \"</tr>\"\n            text += \"<table>\"\n            return text\n        return None\n\n    def _update_info(self):\n        self.info_label.setText(self.get_info_text(short=True))\n\n    def threshold_change_done(self):\n        self.commit.deferred()\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n                    (len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\n            if len(problems) == 1:\n                self.Information.no_output(problems[0])\n            elif problems:\n                self.Information.no_output(\n                    \"\".join(f\"\\n - {problem}\" for problem in problems))\n            else:\n                clsf_idx = self.selected_classifiers[0]\n                model = results.models[0, clsf_idx]\n                if self.score == 0:\n                    cal_learner = CalibratedLearner(\n                        None, self.output_calibration)\n                    wrapped = cal_learner.get_model(\n                        model, results.actual, results.probabilities[clsf_idx])\n                else:\n                    threshold = [1 - self.threshold,\n                                 self.threshold][self.target_index]\n                    wrapped = ThresholdClassifier(model, threshold)\n\n        self.Outputs.calibrated_model.send(wrapped)\n\n    def send_report(self):\n        if self.results is None:\n            return\n        self.report_items((\n            (\"Target class\", self.target_cb.currentText()),\n            (\"Output model calibration\",\n             self.score == 0\n             and (\"Sigmoid calibration\",\n                  \"Isotonic calibration\")[self.output_calibration])\n        ))\n        caption = report.list_legend(self.classifiers_list_box,\n                                     self.selected_classifiers)\n        self.report_plot()\n        self.report_caption(caption)\n        self.report_caption(self.controls.score.currentText())\n\n        if self.score != 0:\n            self.report_raw(self.get_info_text(short=False))\n\n    def set_visual_settings(self, key, value):\n        self.plot.parameter_setter.set_parameter(key, value)\n        self.visual_settings[key] = value\n\n\ndef gaussian_smoother(x, y, sigma=1.0):\n    x = np.asarray(x)\n    y = np.asarray(y)\n\n    gamma = 1. / (2 * sigma ** 2)\n    a = 1. / (sigma * np.sqrt(2 * np.pi))\n\n    if x.shape != y.shape:\n        raise ValueError\n\n    def smoother(xs):\n        W = a * np.exp(-gamma * ((xs - x) ** 2))\n        return np.average(y, weights=W)\n\n    return np.vectorize(smoother, otypes=[float])\n\n\nif __name__ == \"__main__\":  # pragma: no cover\n    WidgetPreview(OWCalibrationPlot).run(results_for_preview())\n",
            "new_file_content": "from collections import namedtuple\n\nimport numpy as np\n\nfrom AnyQt.QtCore import Qt, QSize\nfrom AnyQt.QtWidgets import QListWidget, QSizePolicy\n\nimport pyqtgraph as pg\n\nfrom orangewidget.utils.visual_settings_dlg import VisualSettingsDialog\n\nfrom Orange.base import Model\nfrom Orange.classification import ThresholdClassifier, CalibratedLearner\nfrom Orange.evaluation import Results\nfrom Orange.evaluation.performance_curves import Curves\nfrom Orange.widgets import widget, gui, settings\nfrom Orange.widgets.evaluate.contexthandlers import \\\n    EvaluationResultsContextHandler\nfrom Orange.widgets.evaluate.utils import results_for_preview\nfrom Orange.widgets.utils import colorpalettes\nfrom Orange.widgets.utils.widgetpreview import WidgetPreview\nfrom Orange.widgets.visualize.utils.customizableplot import \\\n    CommonParameterSetter\nfrom Orange.widgets.visualize.utils.plotutils import GraphicsView, PlotItem\nfrom Orange.widgets.widget import Input, Output, Msg\nfrom Orange.widgets import report\n\n\nMetricDefinition = namedtuple(\n    \"metric_definition\",\n    (\"name\", \"functions\", \"short_names\", \"explanation\"))\n\nMetrics = [MetricDefinition(*args) for args in (\n    (\"Calibration curve\", None, (), \"\"),\n    (\"Classification accuracy\", (Curves.ca, ), (), \"\"),\n    (\"F1\", (Curves.f1, ), (), \"\"),\n    (\"Sensitivity and specificity\",\n     (Curves.sensitivity, Curves.specificity),\n     (\"sens\", \"spec\"),\n     \"<p><b>Sensitivity</b> (falling) is the proportion of correctly \"\n     \"detected positive instances (TP&nbsp;/&nbsp;P).</p>\"\n     \"<p><b>Specificity</b> (rising) is the proportion of detected \"\n     \"negative instances (TN&nbsp;/&nbsp;N).</p>\"),\n    (\"Precision and recall\",\n     (Curves.precision, Curves.recall),\n     (\"prec\", \"recall\"),\n     \"<p><b>Precision</b> (rising) is the fraction of retrieved instances \"\n     \"that are relevant, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>\"\n     \"<p><b>Recall</b> (falling) is the proportion of discovered relevant \"\n     \"instances, TP&nbsp;/&nbsp;P.</p>\"),\n    (\"Pos and neg predictive value\",\n     (Curves.ppv, Curves.npv),\n     (\"PPV\", \"TPV\"),\n     \"<p><b>Positive predictive value</b> (rising) is the proportion of \"\n     \"correct positives, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>\"\n     \"<p><b>Negative predictive value</b> is the proportion of correct \"\n     \"negatives, TN&nbsp;/&nbsp;(TN&nbsp;+&nbsp;FN).</p>\"),\n    (\"True and false positive rate\",\n     (Curves.tpr, Curves.fpr),\n     (\"TPR\", \"FPR\"),\n     \"<p><b>True and false positive rate</b> are proportions of detected \"\n     \"and omitted positive instances</p>\"),\n)]\n\n\nclass ParameterSetter(CommonParameterSetter):\n\n    def __init__(self, master):\n        super().__init__()\n        self.master = master\n\n    def update_setters(self):\n        self.initial_settings = {\n            self.LABELS_BOX: {\n                self.FONT_FAMILY_LABEL: self.FONT_FAMILY_SETTING,\n                self.TITLE_LABEL: self.FONT_SETTING,\n                self.AXIS_TITLE_LABEL: self.FONT_SETTING,\n                self.AXIS_TICKS_LABEL: self.FONT_SETTING,\n            },\n            self.ANNOT_BOX: {\n                self.TITLE_LABEL: {self.TITLE_LABEL: (\"\", \"\")},\n            }\n        }\n\n    @property\n    def title_item(self):\n        return self.master.titleLabel\n\n    @property\n    def axis_items(self):\n        return [value[\"item\"] for value in self.master.axes.values()]\n\n\nclass OWCalibrationPlot(widget.OWWidget):\n    name = \"Calibration Plot\"\n    description = \"Calibration plot based on evaluation of classifiers.\"\n    icon = \"icons/CalibrationPlot.svg\"\n    priority = 1030\n    keywords = []\n\n    class Inputs:\n        evaluation_results = Input(\"Evaluation Results\", Results)\n\n    class Outputs:\n        calibrated_model = Output(\"Calibrated Model\", Model)\n\n    class Error(widget.OWWidget.Error):\n        non_discrete_target = Msg(\"Calibration plot requires a categorical \"\n                                  \"target variable.\")\n        empty_input = widget.Msg(\"Empty result on input. Nothing to display.\")\n        nan_classes = \\\n            widget.Msg(\"Remove test data instances with unknown classes.\")\n        all_target_class = widget.Msg(\n            \"All data instances belong to target class.\")\n        no_target_class = widget.Msg(\n            \"No data instances belong to target class.\")\n\n    class Warning(widget.OWWidget.Warning):\n        omitted_folds = widget.Msg(\n            \"Test folds where all data belongs to (non)-target are not shown.\")\n        omitted_nan_prob_points = widget.Msg(\n            \"Instance for which the model couldn't compute probabilities are\"\n            \"skipped.\")\n        no_valid_data = widget.Msg(\"No valid data for model(s) {}\")\n\n    class Information(widget.OWWidget.Information):\n        no_output = Msg(\"Can't output a model: {}\")\n\n    settingsHandler = EvaluationResultsContextHandler()\n    target_index = settings.ContextSetting(0)\n    selected_classifiers = settings.ContextSetting([])\n    score = settings.Setting(0)\n    output_calibration = settings.Setting(0)\n    fold_curves = settings.Setting(False)\n    display_rug = settings.Setting(True)\n    threshold = settings.Setting(0.5)\n    visual_settings = settings.Setting({}, schema_only=True)\n    auto_commit = settings.Setting(True)\n\n    graph_name = \"plot\"\n\n    def __init__(self):\n        super().__init__()\n\n        self.results = None\n        self.scores = None\n        self.classifier_names = []\n        self.colors = []\n        self.line = None\n\n        self._last_score_value = -1\n\n        box = gui.vBox(self.controlArea, box=\"Settings\")\n        self.target_cb = gui.comboBox(\n            box, self, \"target_index\", label=\"Target:\",\n            orientation=Qt.Horizontal, callback=self.target_index_changed,\n            contentsLength=8, searchable=True)\n        gui.checkBox(\n            box, self, \"display_rug\", \"Show rug\",\n            callback=self._on_display_rug_changed)\n        gui.checkBox(\n            box, self, \"fold_curves\", \"Curves for individual folds\",\n            callback=self._replot)\n\n        self.classifiers_list_box = gui.listBox(\n            self.controlArea, self, \"selected_classifiers\", \"classifier_names\",\n            box=\"Classifier\", selectionMode=QListWidget.ExtendedSelection,\n            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),\n            sizeHint=QSize(150, 40),\n            callback=self._on_selection_changed)\n\n        box = gui.vBox(self.controlArea, \"Metrics\")\n        combo = gui.comboBox(\n            box, self, \"score\", items=(metric.name for metric in Metrics),\n            callback=self.score_changed)\n\n        self.explanation = gui.widgetLabel(\n            box, wordWrap=True, fixedWidth=combo.sizeHint().width())\n        self.explanation.setContentsMargins(8, 8, 0, 0)\n        font = self.explanation.font()\n        font.setPointSizeF(0.85 * font.pointSizeF())\n        self.explanation.setFont(font)\n\n        gui.radioButtons(\n            box, self, value=\"output_calibration\",\n            btnLabels=(\"Sigmoid calibration\", \"Isotonic calibration\"),\n            label=\"Output model calibration\", callback=self.commit.deferred)\n\n        self.info_box = gui.widgetBox(self.controlArea, \"Info\")\n        self.info_label = gui.widgetLabel(self.info_box)\n\n        gui.auto_apply(self.buttonsArea, self, \"auto_commit\")\n\n        self.plotview = GraphicsView()\n        self.plot = PlotItem(enableMenu=False)\n        self.plot.parameter_setter = ParameterSetter(self.plot)\n        self.plot.setMouseEnabled(False, False)\n        self.plot.hideButtons()\n        for axis_name in (\"bottom\", \"left\"):\n            axis = self.plot.getAxis(axis_name)\n            # Remove the condition (that is, allow setting this for bottom\n            # axis) when pyqtgraph is fixed\n            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930\n            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932\n            if axis_name != \"bottom\":  # remove if when pyqtgraph is fixed\n                axis.setStyle(stopAxisAtTick=(True, True))\n\n        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)\n        self.plotview.setCentralItem(self.plot)\n\n        self.mainArea.layout().addWidget(self.plotview)\n        self._set_explanation()\n\n        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)\n\n    @Inputs.evaluation_results\n    def set_results(self, results):\n        self.closeContext()\n        self.clear()\n        self.Error.clear()\n        self.Information.clear()\n\n        self.results = None\n        if results is not None:\n            if not results.domain.has_discrete_class:\n                self.Error.non_discrete_target()\n            elif not results.actual.size:\n                self.Error.empty_input()\n            elif np.any(np.isnan(results.actual)):\n                self.Error.nan_classes()\n            else:\n                self.results = results\n                self._initialize(results)\n                class_var = self.results.domain.class_var\n                self.target_index = int(len(class_var.values) == 2)\n                self.openContext(class_var, self.classifier_names)\n                self._replot()\n\n        self.commit.now()\n\n    def clear(self):\n        self.plot.clear()\n        self.results = None\n        self.classifier_names = []\n        self.selected_classifiers = []\n        self.target_cb.clear()\n        self.colors = []\n\n    def target_index_changed(self):\n        if len(self.results.domain.class_var.values) == 2:\n            self.threshold = 1 - self.threshold\n        self._set_explanation()\n        self._replot()\n        self.commit.deferred()\n\n    def score_changed(self):\n        self._set_explanation()\n        self._replot()\n        if self._last_score_value != self.score:\n            self.commit.deferred()\n            self._last_score_value = self.score\n\n    def _set_explanation(self):\n        explanation = Metrics[self.score].explanation\n        if explanation:\n            self.explanation.setText(explanation)\n            self.explanation.show()\n        else:\n            self.explanation.hide()\n\n        if self.score == 0:\n            self.controls.output_calibration.show()\n            self.info_box.hide()\n        else:\n            self.controls.output_calibration.hide()\n            self.info_box.show()\n\n        axis = self.plot.getAxis(\"bottom\")\n        axis.setLabel(\"Predicted probability\" if self.score == 0\n                      else \"Threshold probability to classify as positive\")\n\n        axis = self.plot.getAxis(\"left\")\n        axis.setLabel(Metrics[self.score].name)\n\n    def _initialize(self, results):\n        n = len(results.predicted)\n        names = getattr(results, \"learner_names\", None)\n        if names is None:\n            names = [\"#{}\".format(i + 1) for i in range(n)]\n\n        self.classifier_names = names\n        self.colors = colorpalettes.get_default_curve_colors(n)\n\n        for i in range(n):\n            item = self.classifiers_list_box.item(i)\n            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))\n\n        self.selected_classifiers = list(range(n))\n        self.target_cb.addItems(results.domain.class_var.values)\n        self.target_index = 0\n\n    def _rug(self, data, pen_args):\n        color = pen_args[\"pen\"].color()\n        rh = 0.025\n        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]\n        rug_x_true = rug_x[data.ytrue].ravel()\n        rug_x_false = rug_x[~data.ytrue].ravel()\n\n        rug_y_true = np.ones_like(rug_x_true)\n        rug_y_true[1::2] = 1 - rh\n        rug_y_false = np.zeros_like(rug_x_false)\n        rug_y_false[1::2] = rh\n\n        self.plot.plot(\n            rug_x_false, rug_y_false,\n            pen=color, connect=\"pairs\", antialias=True)\n        self.plot.plot(\n            rug_x_true, rug_y_true,\n            pen=color, connect=\"pairs\", antialias=True)\n\n    def plot_metrics(self, data, metrics, pen_args):\n        if metrics is None:\n            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)\n        ys = [metric(data) for metric in metrics]\n        for y in ys:\n            self.plot.plot(data.probs, y, **pen_args)\n        return data.probs, ys\n\n    def _prob_curve(self, ytrue, probs, pen_args):\n        xmin, xmax = probs.min(), probs.max()\n        x = np.linspace(xmin, xmax, 100)\n        if xmax != xmin:\n            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))\n            y = f(x)\n        else:\n            y = np.full(100, xmax)\n\n        self.plot.plot(x, y, symbol=\"+\", symbolSize=4, **pen_args)\n        return x, (y, )\n\n    def _setup_plot(self):\n        target = self.target_index\n        results = self.results\n        metrics = Metrics[self.score].functions\n        plot_folds = self.fold_curves and results.folds is not None\n        self.scores = []\n\n        if not self._check_class_presence(results.actual == target):\n            return\n\n        self.Warning.omitted_folds.clear()\n        self.Warning.omitted_nan_prob_points.clear()\n        no_valid_models = []\n        shadow_width = 4 + 4 * plot_folds\n        for clsf in self.selected_classifiers:\n            data = Curves.from_results(results, target, clsf)\n            if data.tot == 0:  # all probabilities are nan\n                no_valid_models.append(clsf)\n                continue\n            if data.tot != results.probabilities.shape[1]:  # some are nan\n                self.Warning.omitted_nan_prob_points()\n\n            color = self.colors[clsf]\n            pen_args = dict(\n                pen=pg.mkPen(color, width=1), antiAlias=True,\n                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))\n            self.scores.append(\n                (self.classifier_names[clsf],\n                 self.plot_metrics(data, metrics, pen_args)))\n\n            if self.display_rug:\n                self._rug(data, pen_args)\n\n            if plot_folds:\n                pen_args = dict(\n                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),\n                    antiAlias=True)\n                for fold in range(len(results.folds)):\n                    fold_results = results.get_fold(fold)\n                    fold_curve = Curves.from_results(fold_results, target, clsf)\n                    # Can't check this before: p and n can be 0 because of\n                    # nan probabilities\n                    if fold_curve.p * fold_curve.n == 0:\n                        self.Warning.omitted_folds()\n                    self.plot_metrics(fold_curve, metrics, pen_args)\n\n        if no_valid_models:\n            self.Warning.no_valid_data(\n                \", \".join(self.classifier_names[i] for i in no_valid_models))\n\n        if self.score == 0:\n            self.plot.plot([0, 1], [0, 1], antialias=True)\n        else:\n            self.line = pg.InfiniteLine(\n                pos=self.threshold, movable=True,\n                pen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=2),\n                hoverPen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=3),\n                bounds=(0, 1),\n            )\n            self.line.sigPositionChanged.connect(self.threshold_change)\n            self.line.sigPositionChangeFinished.connect(\n                self.threshold_change_done)\n            self.plot.addItem(self.line)\n\n    def _check_class_presence(self, ytrue):\n        self.Error.all_target_class.clear()\n        self.Error.no_target_class.clear()\n        if np.max(ytrue) == 0:\n            self.Error.no_target_class()\n            return False\n        if np.min(ytrue) == 1:\n            self.Error.all_target_class()\n            return False\n        return True\n\n    def _replot(self):\n        self.plot.clear()\n        if self.results is not None:\n            self._setup_plot()\n        self._update_info()\n\n    def _on_display_rug_changed(self):\n        self._replot()\n\n    def _on_selection_changed(self):\n        self._replot()\n        self.commit.deferred()\n\n    def threshold_change(self):\n        self.threshold = round(self.line.pos().x(), 2)\n        self.line.setPos(self.threshold)\n        self._update_info()\n\n    def get_info_text(self, short):\n        if short:\n            def elided(s):\n                return s[:17] + \"...\" if len(s) > 20 else s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold: p=</th>\n                                <td colspan='4'>{self.threshold:.2f}<br/></td>\n                            </tr>\"\"\"\n\n        else:\n            def elided(s):\n                return s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold:</th>\n                                <td colspan='4'>p = {self.threshold:.2f}<br/>\n                                </td>\n                                <tr/>\n                            </tr>\"\"\"\n\n        if self.scores is not None:\n            short_names = Metrics[self.score].short_names\n            if short_names:\n                text += f\"\"\"<tr>\n                                <th></th>\n                                {\"<td></td>\".join(f\"<td align='right'>{n}</td>\"\n                                                  for n in short_names)}\n                            </tr>\"\"\"\n            for name, (probs, curves) in self.scores:\n                ind = min(np.searchsorted(probs, self.threshold),\n                          len(probs) - 1)\n                text += f\"<tr><th align='right'>{elided(name)}:</th>\"\n                text += \"<td>/</td>\".join(f'<td>{curve[ind]:.3f}</td>'\n                                          for curve in curves)\n                text += \"</tr>\"\n            text += \"<table>\"\n            return text\n        return None\n\n    def _update_info(self):\n        self.info_label.setText(self.get_info_text(short=True))\n\n    def threshold_change_done(self):\n        self.commit.deferred()\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n                    (results.folds is not None and len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\n            if len(problems) == 1:\n                self.Information.no_output(problems[0])\n            elif problems:\n                self.Information.no_output(\n                    \"\".join(f\"\\n - {problem}\" for problem in problems))\n            else:\n                clsf_idx = self.selected_classifiers[0]\n                model = results.models[0, clsf_idx]\n                if self.score == 0:\n                    cal_learner = CalibratedLearner(\n                        None, self.output_calibration)\n                    wrapped = cal_learner.get_model(\n                        model, results.actual, results.probabilities[clsf_idx])\n                else:\n                    threshold = [1 - self.threshold,\n                                 self.threshold][self.target_index]\n                    wrapped = ThresholdClassifier(model, threshold)\n\n        self.Outputs.calibrated_model.send(wrapped)\n\n    def send_report(self):\n        if self.results is None:\n            return\n        self.report_items((\n            (\"Target class\", self.target_cb.currentText()),\n            (\"Output model calibration\",\n             self.score == 0\n             and (\"Sigmoid calibration\",\n                  \"Isotonic calibration\")[self.output_calibration])\n        ))\n        caption = report.list_legend(self.classifiers_list_box,\n                                     self.selected_classifiers)\n        self.report_plot()\n        self.report_caption(caption)\n        self.report_caption(self.controls.score.currentText())\n\n        if self.score != 0:\n            self.report_raw(self.get_info_text(short=False))\n\n    def set_visual_settings(self, key, value):\n        self.plot.parameter_setter.set_parameter(key, value)\n        self.visual_settings[key] = value\n\n\ndef gaussian_smoother(x, y, sigma=1.0):\n    x = np.asarray(x)\n    y = np.asarray(y)\n\n    gamma = 1. / (2 * sigma ** 2)\n    a = 1. / (sigma * np.sqrt(2 * np.pi))\n\n    if x.shape != y.shape:\n        raise ValueError\n\n    def smoother(xs):\n        W = a * np.exp(-gamma * ((xs - x) ** 2))\n        return np.average(y, weights=W)\n\n    return np.vectorize(smoother, otypes=[float])\n\n\nif __name__ == \"__main__\":  # pragma: no cover\n    WidgetPreview(OWCalibrationPlot).run(results_for_preview())\n",
            "header": {
                "file": {
                    "path": "Orange/widgets/evaluate/owcalibrationplot.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "72c2822d8",
                "new_commit_hash": "853c7ae3c",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/Orange/widgets/evaluate/owcalibrationplot.py"
            },
            "plus_file": {
                "path": "b/Orange/widgets/evaluate/owcalibrationplot.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 487,
                            "length": 7
                        },
                        "new_range": {
                            "start": 487,
                            "length": 7
                        },
                        "section": "class OWCalibrationPlot(widget.OWWidget):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        if results is not None:",
                                "type": "context"
                            },
                            {
                                "content": "            problems = [",
                                "type": "context"
                            },
                            {
                                "content": "                msg for condition, msg in (",
                                "type": "context"
                            },
                            {
                                "content": "                    (len(results.folds) > 1,",
                                "type": "deleted"
                            },
                            {
                                "content": "                    (results.folds is not None and len(results.folds) > 1,",
                                "type": "added"
                            },
                            {
                                "content": "                     \"each training data sample produces a different model\"),",
                                "type": "context"
                            },
                            {
                                "content": "                    (results.models is None,",
                                "type": "context"
                            },
                            {
                                "content": "                     \"test results do not contain stored models - try testing \"",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/owcalibrationplot.py",
                            "type": "method",
                            "name": "OWCalibrationPlot.commit",
                            "content": "class OWCalibrationPlot(widget.OWWidget):\n    name = \"Calibration Plot\"\n    description = \"Calibration plot based on evaluation of classifiers.\"\n    icon = \"icons/CalibrationPlot.svg\"\n    priority = 1030\n    keywords = []\n\n    class Inputs:\n        evaluation_results = Input(\"Evaluation Results\", Results)\n\n    class Outputs:\n        calibrated_model = Output(\"Calibrated Model\", Model)\n\n    class Error(widget.OWWidget.Error):\n        non_discrete_target = Msg(\"Calibration plot requires a categorical \"\n                                  \"target variable.\")\n        empty_input = widget.Msg(\"Empty result on input. Nothing to display.\")\n        nan_classes = \\\n            widget.Msg(\"Remove test data instances with unknown classes.\")\n        all_target_class = widget.Msg(\n            \"All data instances belong to target class.\")\n        no_target_class = widget.Msg(\n            \"No data instances belong to target class.\")\n\n    class Warning(widget.OWWidget.Warning):\n        omitted_folds = widget.Msg(\n            \"Test folds where all data belongs to (non)-target are not shown.\")\n        omitted_nan_prob_points = widget.Msg(\n            \"Instance for which the model couldn't compute probabilities are\"\n            \"skipped.\")\n        no_valid_data = widget.Msg(\"No valid data for model(s) {}\")\n\n    class Information(widget.OWWidget.Information):\n        no_output = Msg(\"Can't output a model: {}\")\n\n    settingsHandler = EvaluationResultsContextHandler()\n    target_index = settings.ContextSetting(0)\n    selected_classifiers = settings.ContextSetting([])\n    score = settings.Setting(0)\n    output_calibration = settings.Setting(0)\n    fold_curves = settings.Setting(False)\n    display_rug = settings.Setting(True)\n    threshold = settings.Setting(0.5)\n    visual_settings = settings.Setting({}, schema_only=True)\n    auto_commit = settings.Setting(True)\n\n    graph_name = \"plot\"\n\n    def __init__(self):\n        super().__init__()\n\n        self.results = None\n        self.scores = None\n        self.classifier_names = []\n        self.colors = []\n        self.line = None\n\n        self._last_score_value = -1\n\n        box = gui.vBox(self.controlArea, box=\"Settings\")\n        self.target_cb = gui.comboBox(\n            box, self, \"target_index\", label=\"Target:\",\n            orientation=Qt.Horizontal, callback=self.target_index_changed,\n            contentsLength=8, searchable=True)\n        gui.checkBox(\n            box, self, \"display_rug\", \"Show rug\",\n            callback=self._on_display_rug_changed)\n        gui.checkBox(\n            box, self, \"fold_curves\", \"Curves for individual folds\",\n            callback=self._replot)\n\n        self.classifiers_list_box = gui.listBox(\n            self.controlArea, self, \"selected_classifiers\", \"classifier_names\",\n            box=\"Classifier\", selectionMode=QListWidget.ExtendedSelection,\n            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),\n            sizeHint=QSize(150, 40),\n            callback=self._on_selection_changed)\n\n        box = gui.vBox(self.controlArea, \"Metrics\")\n        combo = gui.comboBox(\n            box, self, \"score\", items=(metric.name for metric in Metrics),\n            callback=self.score_changed)\n\n        self.explanation = gui.widgetLabel(\n            box, wordWrap=True, fixedWidth=combo.sizeHint().width())\n        self.explanation.setContentsMargins(8, 8, 0, 0)\n        font = self.explanation.font()\n        font.setPointSizeF(0.85 * font.pointSizeF())\n        self.explanation.setFont(font)\n\n        gui.radioButtons(\n            box, self, value=\"output_calibration\",\n            btnLabels=(\"Sigmoid calibration\", \"Isotonic calibration\"),\n            label=\"Output model calibration\", callback=self.commit.deferred)\n\n        self.info_box = gui.widgetBox(self.controlArea, \"Info\")\n        self.info_label = gui.widgetLabel(self.info_box)\n\n        gui.auto_apply(self.buttonsArea, self, \"auto_commit\")\n\n        self.plotview = GraphicsView()\n        self.plot = PlotItem(enableMenu=False)\n        self.plot.parameter_setter = ParameterSetter(self.plot)\n        self.plot.setMouseEnabled(False, False)\n        self.plot.hideButtons()\n        for axis_name in (\"bottom\", \"left\"):\n            axis = self.plot.getAxis(axis_name)\n            # Remove the condition (that is, allow setting this for bottom\n            # axis) when pyqtgraph is fixed\n            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930\n            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932\n            if axis_name != \"bottom\":  # remove if when pyqtgraph is fixed\n                axis.setStyle(stopAxisAtTick=(True, True))\n\n        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)\n        self.plotview.setCentralItem(self.plot)\n\n        self.mainArea.layout().addWidget(self.plotview)\n        self._set_explanation()\n\n        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)\n\n    @Inputs.evaluation_results\n    def set_results(self, results):\n        self.closeContext()\n        self.clear()\n        self.Error.clear()\n        self.Information.clear()\n\n        self.results = None\n        if results is not None:\n            if not results.domain.has_discrete_class:\n                self.Error.non_discrete_target()\n            elif not results.actual.size:\n                self.Error.empty_input()\n            elif np.any(np.isnan(results.actual)):\n                self.Error.nan_classes()\n            else:\n                self.results = results\n                self._initialize(results)\n                class_var = self.results.domain.class_var\n                self.target_index = int(len(class_var.values) == 2)\n                self.openContext(class_var, self.classifier_names)\n                self._replot()\n\n        self.commit.now()\n\n    def clear(self):\n        self.plot.clear()\n        self.results = None\n        self.classifier_names = []\n        self.selected_classifiers = []\n        self.target_cb.clear()\n        self.colors = []\n\n    def target_index_changed(self):\n        if len(self.results.domain.class_var.values) == 2:\n            self.threshold = 1 - self.threshold\n        self._set_explanation()\n        self._replot()\n        self.commit.deferred()\n\n    def score_changed(self):\n        self._set_explanation()\n        self._replot()\n        if self._last_score_value != self.score:\n            self.commit.deferred()\n            self._last_score_value = self.score\n\n    def _set_explanation(self):\n        explanation = Metrics[self.score].explanation\n        if explanation:\n            self.explanation.setText(explanation)\n            self.explanation.show()\n        else:\n            self.explanation.hide()\n\n        if self.score == 0:\n            self.controls.output_calibration.show()\n            self.info_box.hide()\n        else:\n            self.controls.output_calibration.hide()\n            self.info_box.show()\n\n        axis = self.plot.getAxis(\"bottom\")\n        axis.setLabel(\"Predicted probability\" if self.score == 0\n                      else \"Threshold probability to classify as positive\")\n\n        axis = self.plot.getAxis(\"left\")\n        axis.setLabel(Metrics[self.score].name)\n\n    def _initialize(self, results):\n        n = len(results.predicted)\n        names = getattr(results, \"learner_names\", None)\n        if names is None:\n            names = [\"#{}\".format(i + 1) for i in range(n)]\n\n        self.classifier_names = names\n        self.colors = colorpalettes.get_default_curve_colors(n)\n\n        for i in range(n):\n            item = self.classifiers_list_box.item(i)\n            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))\n\n        self.selected_classifiers = list(range(n))\n        self.target_cb.addItems(results.domain.class_var.values)\n        self.target_index = 0\n\n    def _rug(self, data, pen_args):\n        color = pen_args[\"pen\"].color()\n        rh = 0.025\n        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]\n        rug_x_true = rug_x[data.ytrue].ravel()\n        rug_x_false = rug_x[~data.ytrue].ravel()\n\n        rug_y_true = np.ones_like(rug_x_true)\n        rug_y_true[1::2] = 1 - rh\n        rug_y_false = np.zeros_like(rug_x_false)\n        rug_y_false[1::2] = rh\n\n        self.plot.plot(\n            rug_x_false, rug_y_false,\n            pen=color, connect=\"pairs\", antialias=True)\n        self.plot.plot(\n            rug_x_true, rug_y_true,\n            pen=color, connect=\"pairs\", antialias=True)\n\n    def plot_metrics(self, data, metrics, pen_args):\n        if metrics is None:\n            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)\n        ys = [metric(data) for metric in metrics]\n        for y in ys:\n            self.plot.plot(data.probs, y, **pen_args)\n        return data.probs, ys\n\n    def _prob_curve(self, ytrue, probs, pen_args):\n        xmin, xmax = probs.min(), probs.max()\n        x = np.linspace(xmin, xmax, 100)\n        if xmax != xmin:\n            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))\n            y = f(x)\n        else:\n            y = np.full(100, xmax)\n\n        self.plot.plot(x, y, symbol=\"+\", symbolSize=4, **pen_args)\n        return x, (y, )\n\n    def _setup_plot(self):\n        target = self.target_index\n        results = self.results\n        metrics = Metrics[self.score].functions\n        plot_folds = self.fold_curves and results.folds is not None\n        self.scores = []\n\n        if not self._check_class_presence(results.actual == target):\n            return\n\n        self.Warning.omitted_folds.clear()\n        self.Warning.omitted_nan_prob_points.clear()\n        no_valid_models = []\n        shadow_width = 4 + 4 * plot_folds\n        for clsf in self.selected_classifiers:\n            data = Curves.from_results(results, target, clsf)\n            if data.tot == 0:  # all probabilities are nan\n                no_valid_models.append(clsf)\n                continue\n            if data.tot != results.probabilities.shape[1]:  # some are nan\n                self.Warning.omitted_nan_prob_points()\n\n            color = self.colors[clsf]\n            pen_args = dict(\n                pen=pg.mkPen(color, width=1), antiAlias=True,\n                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))\n            self.scores.append(\n                (self.classifier_names[clsf],\n                 self.plot_metrics(data, metrics, pen_args)))\n\n            if self.display_rug:\n                self._rug(data, pen_args)\n\n            if plot_folds:\n                pen_args = dict(\n                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),\n                    antiAlias=True)\n                for fold in range(len(results.folds)):\n                    fold_results = results.get_fold(fold)\n                    fold_curve = Curves.from_results(fold_results, target, clsf)\n                    # Can't check this before: p and n can be 0 because of\n                    # nan probabilities\n                    if fold_curve.p * fold_curve.n == 0:\n                        self.Warning.omitted_folds()\n                    self.plot_metrics(fold_curve, metrics, pen_args)\n\n        if no_valid_models:\n            self.Warning.no_valid_data(\n                \", \".join(self.classifier_names[i] for i in no_valid_models))\n\n        if self.score == 0:\n            self.plot.plot([0, 1], [0, 1], antialias=True)\n        else:\n            self.line = pg.InfiniteLine(\n                pos=self.threshold, movable=True,\n                pen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=2),\n                hoverPen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=3),\n                bounds=(0, 1),\n            )\n            self.line.sigPositionChanged.connect(self.threshold_change)\n            self.line.sigPositionChangeFinished.connect(\n                self.threshold_change_done)\n            self.plot.addItem(self.line)\n\n    def _check_class_presence(self, ytrue):\n        self.Error.all_target_class.clear()\n        self.Error.no_target_class.clear()\n        if np.max(ytrue) == 0:\n            self.Error.no_target_class()\n            return False\n        if np.min(ytrue) == 1:\n            self.Error.all_target_class()\n            return False\n        return True\n\n    def _replot(self):\n        self.plot.clear()\n        if self.results is not None:\n            self._setup_plot()\n        self._update_info()\n\n    def _on_display_rug_changed(self):\n        self._replot()\n\n    def _on_selection_changed(self):\n        self._replot()\n        self.commit.deferred()\n\n    def threshold_change(self):\n        self.threshold = round(self.line.pos().x(), 2)\n        self.line.setPos(self.threshold)\n        self._update_info()\n\n    def get_info_text(self, short):\n        if short:\n            def elided(s):\n                return s[:17] + \"...\" if len(s) > 20 else s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold: p=</th>\n                                <td colspan='4'>{self.threshold:.2f}<br/></td>\n                            </tr>\"\"\"\n\n        else:\n            def elided(s):\n                return s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold:</th>\n                                <td colspan='4'>p = {self.threshold:.2f}<br/>\n                                </td>\n                                <tr/>\n                            </tr>\"\"\"\n\n        if self.scores is not None:\n            short_names = Metrics[self.score].short_names\n            if short_names:\n                text += f\"\"\"<tr>\n                                <th></th>\n                                {\"<td></td>\".join(f\"<td align='right'>{n}</td>\"\n                                                  for n in short_names)}\n                            </tr>\"\"\"\n            for name, (probs, curves) in self.scores:\n                ind = min(np.searchsorted(probs, self.threshold),\n                          len(probs) - 1)\n                text += f\"<tr><th align='right'>{elided(name)}:</th>\"\n                text += \"<td>/</td>\".join(f'<td>{curve[ind]:.3f}</td>'\n                                          for curve in curves)\n                text += \"</tr>\"\n            text += \"<table>\"\n            return text\n        return None\n\n    def _update_info(self):\n        self.info_label.setText(self.get_info_text(short=True))\n\n    def threshold_change_done(self):\n        self.commit.deferred()\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n                    (len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\n            if len(problems) == 1:\n                self.Information.no_output(problems[0])\n            elif problems:\n                self.Information.no_output(\n                    \"\".join(f\"\\n - {problem}\" for problem in problems))\n            else:\n                clsf_idx = self.selected_classifiers[0]\n                model = results.models[0, clsf_idx]\n                if self.score == 0:\n                    cal_learner = CalibratedLearner(\n                        None, self.output_calibration)\n                    wrapped = cal_learner.get_model(\n                        model, results.actual, results.probabilities[clsf_idx])\n                else:\n                    threshold = [1 - self.threshold,\n                                 self.threshold][self.target_index]\n                    wrapped = ThresholdClassifier(model, threshold)\n\n        self.Outputs.calibrated_model.send(wrapped)\n\n    def send_report(self):\n        if self.results is None:\n            return\n        self.report_items((\n            (\"Target class\", self.target_cb.currentText()),\n            (\"Output model calibration\",\n             self.score == 0\n             and (\"Sigmoid calibration\",\n                  \"Isotonic calibration\")[self.output_calibration])\n        ))\n        caption = report.list_legend(self.classifiers_list_box,\n                                     self.selected_classifiers)\n        self.report_plot()\n        self.report_caption(caption)\n        self.report_caption(self.controls.score.currentText())\n\n        if self.score != 0:\n            self.report_raw(self.get_info_text(short=False))\n\n    def set_visual_settings(self, key, value):\n        self.plot.parameter_setter.set_parameter(key, value)\n        self.visual_settings[key] = value",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 483,
                            "end_lineno": 518,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/owcalibrationplot.py",
                                "type": "class",
                                "name": "OWCalibrationPlot",
                                "content": "class OWCalibrationPlot(widget.OWWidget):\n    name = \"Calibration Plot\"\n    description = \"Calibration plot based on evaluation of classifiers.\"\n    icon = \"icons/CalibrationPlot.svg\"\n    priority = 1030\n    keywords = []\n\n    class Inputs:\n        evaluation_results = Input(\"Evaluation Results\", Results)\n\n    class Outputs:\n        calibrated_model = Output(\"Calibrated Model\", Model)\n\n    class Error(widget.OWWidget.Error):\n        non_discrete_target = Msg(\"Calibration plot requires a categorical \"\n                                  \"target variable.\")\n        empty_input = widget.Msg(\"Empty result on input. Nothing to display.\")\n        nan_classes = \\\n            widget.Msg(\"Remove test data instances with unknown classes.\")\n        all_target_class = widget.Msg(\n            \"All data instances belong to target class.\")\n        no_target_class = widget.Msg(\n            \"No data instances belong to target class.\")\n\n    class Warning(widget.OWWidget.Warning):\n        omitted_folds = widget.Msg(\n            \"Test folds where all data belongs to (non)-target are not shown.\")\n        omitted_nan_prob_points = widget.Msg(\n            \"Instance for which the model couldn't compute probabilities are\"\n            \"skipped.\")\n        no_valid_data = widget.Msg(\"No valid data for model(s) {}\")\n\n    class Information(widget.OWWidget.Information):\n        no_output = Msg(\"Can't output a model: {}\")\n\n    settingsHandler = EvaluationResultsContextHandler()\n    target_index = settings.ContextSetting(0)\n    selected_classifiers = settings.ContextSetting([])\n    score = settings.Setting(0)\n    output_calibration = settings.Setting(0)\n    fold_curves = settings.Setting(False)\n    display_rug = settings.Setting(True)\n    threshold = settings.Setting(0.5)\n    visual_settings = settings.Setting({}, schema_only=True)\n    auto_commit = settings.Setting(True)\n\n    graph_name = \"plot\"\n\n    def __init__(self):\n        super().__init__()\n\n        self.results = None\n        self.scores = None\n        self.classifier_names = []\n        self.colors = []\n        self.line = None\n\n        self._last_score_value = -1\n\n        box = gui.vBox(self.controlArea, box=\"Settings\")\n        self.target_cb = gui.comboBox(\n            box, self, \"target_index\", label=\"Target:\",\n            orientation=Qt.Horizontal, callback=self.target_index_changed,\n            contentsLength=8, searchable=True)\n        gui.checkBox(\n            box, self, \"display_rug\", \"Show rug\",\n            callback=self._on_display_rug_changed)\n        gui.checkBox(\n            box, self, \"fold_curves\", \"Curves for individual folds\",\n            callback=self._replot)\n\n        self.classifiers_list_box = gui.listBox(\n            self.controlArea, self, \"selected_classifiers\", \"classifier_names\",\n            box=\"Classifier\", selectionMode=QListWidget.ExtendedSelection,\n            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),\n            sizeHint=QSize(150, 40),\n            callback=self._on_selection_changed)\n\n        box = gui.vBox(self.controlArea, \"Metrics\")\n        combo = gui.comboBox(\n            box, self, \"score\", items=(metric.name for metric in Metrics),\n            callback=self.score_changed)\n\n        self.explanation = gui.widgetLabel(\n            box, wordWrap=True, fixedWidth=combo.sizeHint().width())\n        self.explanation.setContentsMargins(8, 8, 0, 0)\n        font = self.explanation.font()\n        font.setPointSizeF(0.85 * font.pointSizeF())\n        self.explanation.setFont(font)\n\n        gui.radioButtons(\n            box, self, value=\"output_calibration\",\n            btnLabels=(\"Sigmoid calibration\", \"Isotonic calibration\"),\n            label=\"Output model calibration\", callback=self.commit.deferred)\n\n        self.info_box = gui.widgetBox(self.controlArea, \"Info\")\n        self.info_label = gui.widgetLabel(self.info_box)\n\n        gui.auto_apply(self.buttonsArea, self, \"auto_commit\")\n\n        self.plotview = GraphicsView()\n        self.plot = PlotItem(enableMenu=False)\n        self.plot.parameter_setter = ParameterSetter(self.plot)\n        self.plot.setMouseEnabled(False, False)\n        self.plot.hideButtons()\n        for axis_name in (\"bottom\", \"left\"):\n            axis = self.plot.getAxis(axis_name)\n            # Remove the condition (that is, allow setting this for bottom\n            # axis) when pyqtgraph is fixed\n            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930\n            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932\n            if axis_name != \"bottom\":  # remove if when pyqtgraph is fixed\n                axis.setStyle(stopAxisAtTick=(True, True))\n\n        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)\n        self.plotview.setCentralItem(self.plot)\n\n        self.mainArea.layout().addWidget(self.plotview)\n        self._set_explanation()\n\n        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)\n\n    @Inputs.evaluation_results\n    def set_results(self, results):\n        self.closeContext()\n        self.clear()\n        self.Error.clear()\n        self.Information.clear()\n\n        self.results = None\n        if results is not None:\n            if not results.domain.has_discrete_class:\n                self.Error.non_discrete_target()\n            elif not results.actual.size:\n                self.Error.empty_input()\n            elif np.any(np.isnan(results.actual)):\n                self.Error.nan_classes()\n            else:\n                self.results = results\n                self._initialize(results)\n                class_var = self.results.domain.class_var\n                self.target_index = int(len(class_var.values) == 2)\n                self.openContext(class_var, self.classifier_names)\n                self._replot()\n\n        self.commit.now()\n\n    def clear(self):\n        self.plot.clear()\n        self.results = None\n        self.classifier_names = []\n        self.selected_classifiers = []\n        self.target_cb.clear()\n        self.colors = []\n\n    def target_index_changed(self):\n        if len(self.results.domain.class_var.values) == 2:\n            self.threshold = 1 - self.threshold\n        self._set_explanation()\n        self._replot()\n        self.commit.deferred()\n\n    def score_changed(self):\n        self._set_explanation()\n        self._replot()\n        if self._last_score_value != self.score:\n            self.commit.deferred()\n            self._last_score_value = self.score\n\n    def _set_explanation(self):\n        explanation = Metrics[self.score].explanation\n        if explanation:\n            self.explanation.setText(explanation)\n            self.explanation.show()\n        else:\n            self.explanation.hide()\n\n        if self.score == 0:\n            self.controls.output_calibration.show()\n            self.info_box.hide()\n        else:\n            self.controls.output_calibration.hide()\n            self.info_box.show()\n\n        axis = self.plot.getAxis(\"bottom\")\n        axis.setLabel(\"Predicted probability\" if self.score == 0\n                      else \"Threshold probability to classify as positive\")\n\n        axis = self.plot.getAxis(\"left\")\n        axis.setLabel(Metrics[self.score].name)\n\n    def _initialize(self, results):\n        n = len(results.predicted)\n        names = getattr(results, \"learner_names\", None)\n        if names is None:\n            names = [\"#{}\".format(i + 1) for i in range(n)]\n\n        self.classifier_names = names\n        self.colors = colorpalettes.get_default_curve_colors(n)\n\n        for i in range(n):\n            item = self.classifiers_list_box.item(i)\n            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))\n\n        self.selected_classifiers = list(range(n))\n        self.target_cb.addItems(results.domain.class_var.values)\n        self.target_index = 0\n\n    def _rug(self, data, pen_args):\n        color = pen_args[\"pen\"].color()\n        rh = 0.025\n        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]\n        rug_x_true = rug_x[data.ytrue].ravel()\n        rug_x_false = rug_x[~data.ytrue].ravel()\n\n        rug_y_true = np.ones_like(rug_x_true)\n        rug_y_true[1::2] = 1 - rh\n        rug_y_false = np.zeros_like(rug_x_false)\n        rug_y_false[1::2] = rh\n\n        self.plot.plot(\n            rug_x_false, rug_y_false,\n            pen=color, connect=\"pairs\", antialias=True)\n        self.plot.plot(\n            rug_x_true, rug_y_true,\n            pen=color, connect=\"pairs\", antialias=True)\n\n    def plot_metrics(self, data, metrics, pen_args):\n        if metrics is None:\n            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)\n        ys = [metric(data) for metric in metrics]\n        for y in ys:\n            self.plot.plot(data.probs, y, **pen_args)\n        return data.probs, ys\n\n    def _prob_curve(self, ytrue, probs, pen_args):\n        xmin, xmax = probs.min(), probs.max()\n        x = np.linspace(xmin, xmax, 100)\n        if xmax != xmin:\n            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))\n            y = f(x)\n        else:\n            y = np.full(100, xmax)\n\n        self.plot.plot(x, y, symbol=\"+\", symbolSize=4, **pen_args)\n        return x, (y, )\n\n    def _setup_plot(self):\n        target = self.target_index\n        results = self.results\n        metrics = Metrics[self.score].functions\n        plot_folds = self.fold_curves and results.folds is not None\n        self.scores = []\n\n        if not self._check_class_presence(results.actual == target):\n            return\n\n        self.Warning.omitted_folds.clear()\n        self.Warning.omitted_nan_prob_points.clear()\n        no_valid_models = []\n        shadow_width = 4 + 4 * plot_folds\n        for clsf in self.selected_classifiers:\n            data = Curves.from_results(results, target, clsf)\n            if data.tot == 0:  # all probabilities are nan\n                no_valid_models.append(clsf)\n                continue\n            if data.tot != results.probabilities.shape[1]:  # some are nan\n                self.Warning.omitted_nan_prob_points()\n\n            color = self.colors[clsf]\n            pen_args = dict(\n                pen=pg.mkPen(color, width=1), antiAlias=True,\n                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))\n            self.scores.append(\n                (self.classifier_names[clsf],\n                 self.plot_metrics(data, metrics, pen_args)))\n\n            if self.display_rug:\n                self._rug(data, pen_args)\n\n            if plot_folds:\n                pen_args = dict(\n                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),\n                    antiAlias=True)\n                for fold in range(len(results.folds)):\n                    fold_results = results.get_fold(fold)\n                    fold_curve = Curves.from_results(fold_results, target, clsf)\n                    # Can't check this before: p and n can be 0 because of\n                    # nan probabilities\n                    if fold_curve.p * fold_curve.n == 0:\n                        self.Warning.omitted_folds()\n                    self.plot_metrics(fold_curve, metrics, pen_args)\n\n        if no_valid_models:\n            self.Warning.no_valid_data(\n                \", \".join(self.classifier_names[i] for i in no_valid_models))\n\n        if self.score == 0:\n            self.plot.plot([0, 1], [0, 1], antialias=True)\n        else:\n            self.line = pg.InfiniteLine(\n                pos=self.threshold, movable=True,\n                pen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=2),\n                hoverPen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=3),\n                bounds=(0, 1),\n            )\n            self.line.sigPositionChanged.connect(self.threshold_change)\n            self.line.sigPositionChangeFinished.connect(\n                self.threshold_change_done)\n            self.plot.addItem(self.line)\n\n    def _check_class_presence(self, ytrue):\n        self.Error.all_target_class.clear()\n        self.Error.no_target_class.clear()\n        if np.max(ytrue) == 0:\n            self.Error.no_target_class()\n            return False\n        if np.min(ytrue) == 1:\n            self.Error.all_target_class()\n            return False\n        return True\n\n    def _replot(self):\n        self.plot.clear()\n        if self.results is not None:\n            self._setup_plot()\n        self._update_info()\n\n    def _on_display_rug_changed(self):\n        self._replot()\n\n    def _on_selection_changed(self):\n        self._replot()\n        self.commit.deferred()\n\n    def threshold_change(self):\n        self.threshold = round(self.line.pos().x(), 2)\n        self.line.setPos(self.threshold)\n        self._update_info()\n\n    def get_info_text(self, short):\n        if short:\n            def elided(s):\n                return s[:17] + \"...\" if len(s) > 20 else s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold: p=</th>\n                                <td colspan='4'>{self.threshold:.2f}<br/></td>\n                            </tr>\"\"\"\n\n        else:\n            def elided(s):\n                return s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold:</th>\n                                <td colspan='4'>p = {self.threshold:.2f}<br/>\n                                </td>\n                                <tr/>\n                            </tr>\"\"\"\n\n        if self.scores is not None:\n            short_names = Metrics[self.score].short_names\n            if short_names:\n                text += f\"\"\"<tr>\n                                <th></th>\n                                {\"<td></td>\".join(f\"<td align='right'>{n}</td>\"\n                                                  for n in short_names)}\n                            </tr>\"\"\"\n            for name, (probs, curves) in self.scores:\n                ind = min(np.searchsorted(probs, self.threshold),\n                          len(probs) - 1)\n                text += f\"<tr><th align='right'>{elided(name)}:</th>\"\n                text += \"<td>/</td>\".join(f'<td>{curve[ind]:.3f}</td>'\n                                          for curve in curves)\n                text += \"</tr>\"\n            text += \"<table>\"\n            return text\n        return None\n\n    def _update_info(self):\n        self.info_label.setText(self.get_info_text(short=True))\n\n    def threshold_change_done(self):\n        self.commit.deferred()\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n                    (len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\n            if len(problems) == 1:\n                self.Information.no_output(problems[0])\n            elif problems:\n                self.Information.no_output(\n                    \"\".join(f\"\\n - {problem}\" for problem in problems))\n            else:\n                clsf_idx = self.selected_classifiers[0]\n                model = results.models[0, clsf_idx]\n                if self.score == 0:\n                    cal_learner = CalibratedLearner(\n                        None, self.output_calibration)\n                    wrapped = cal_learner.get_model(\n                        model, results.actual, results.probabilities[clsf_idx])\n                else:\n                    threshold = [1 - self.threshold,\n                                 self.threshold][self.target_index]\n                    wrapped = ThresholdClassifier(model, threshold)\n\n        self.Outputs.calibrated_model.send(wrapped)\n\n    def send_report(self):\n        if self.results is None:\n            return\n        self.report_items((\n            (\"Target class\", self.target_cb.currentText()),\n            (\"Output model calibration\",\n             self.score == 0\n             and (\"Sigmoid calibration\",\n                  \"Isotonic calibration\")[self.output_calibration])\n        ))\n        caption = report.list_legend(self.classifiers_list_box,\n                                     self.selected_classifiers)\n        self.report_plot()\n        self.report_caption(caption)\n        self.report_caption(self.controls.score.currentText())\n\n        if self.score != 0:\n            self.report_raw(self.get_info_text(short=False))\n\n    def set_visual_settings(self, key, value):\n        self.plot.parameter_setter.set_parameter(key, value)\n        self.visual_settings[key] = value",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 94,
                                "end_lineno": 541,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "Orange/widgets/evaluate/owcalibrationplot.py",
                            "type": "class",
                            "name": "OWCalibrationPlot",
                            "content": "class OWCalibrationPlot(widget.OWWidget):\n    name = \"Calibration Plot\"\n    description = \"Calibration plot based on evaluation of classifiers.\"\n    icon = \"icons/CalibrationPlot.svg\"\n    priority = 1030\n    keywords = []\n\n    class Inputs:\n        evaluation_results = Input(\"Evaluation Results\", Results)\n\n    class Outputs:\n        calibrated_model = Output(\"Calibrated Model\", Model)\n\n    class Error(widget.OWWidget.Error):\n        non_discrete_target = Msg(\"Calibration plot requires a categorical \"\n                                  \"target variable.\")\n        empty_input = widget.Msg(\"Empty result on input. Nothing to display.\")\n        nan_classes = \\\n            widget.Msg(\"Remove test data instances with unknown classes.\")\n        all_target_class = widget.Msg(\n            \"All data instances belong to target class.\")\n        no_target_class = widget.Msg(\n            \"No data instances belong to target class.\")\n\n    class Warning(widget.OWWidget.Warning):\n        omitted_folds = widget.Msg(\n            \"Test folds where all data belongs to (non)-target are not shown.\")\n        omitted_nan_prob_points = widget.Msg(\n            \"Instance for which the model couldn't compute probabilities are\"\n            \"skipped.\")\n        no_valid_data = widget.Msg(\"No valid data for model(s) {}\")\n\n    class Information(widget.OWWidget.Information):\n        no_output = Msg(\"Can't output a model: {}\")\n\n    settingsHandler = EvaluationResultsContextHandler()\n    target_index = settings.ContextSetting(0)\n    selected_classifiers = settings.ContextSetting([])\n    score = settings.Setting(0)\n    output_calibration = settings.Setting(0)\n    fold_curves = settings.Setting(False)\n    display_rug = settings.Setting(True)\n    threshold = settings.Setting(0.5)\n    visual_settings = settings.Setting({}, schema_only=True)\n    auto_commit = settings.Setting(True)\n\n    graph_name = \"plot\"\n\n    def __init__(self):\n        super().__init__()\n\n        self.results = None\n        self.scores = None\n        self.classifier_names = []\n        self.colors = []\n        self.line = None\n\n        self._last_score_value = -1\n\n        box = gui.vBox(self.controlArea, box=\"Settings\")\n        self.target_cb = gui.comboBox(\n            box, self, \"target_index\", label=\"Target:\",\n            orientation=Qt.Horizontal, callback=self.target_index_changed,\n            contentsLength=8, searchable=True)\n        gui.checkBox(\n            box, self, \"display_rug\", \"Show rug\",\n            callback=self._on_display_rug_changed)\n        gui.checkBox(\n            box, self, \"fold_curves\", \"Curves for individual folds\",\n            callback=self._replot)\n\n        self.classifiers_list_box = gui.listBox(\n            self.controlArea, self, \"selected_classifiers\", \"classifier_names\",\n            box=\"Classifier\", selectionMode=QListWidget.ExtendedSelection,\n            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),\n            sizeHint=QSize(150, 40),\n            callback=self._on_selection_changed)\n\n        box = gui.vBox(self.controlArea, \"Metrics\")\n        combo = gui.comboBox(\n            box, self, \"score\", items=(metric.name for metric in Metrics),\n            callback=self.score_changed)\n\n        self.explanation = gui.widgetLabel(\n            box, wordWrap=True, fixedWidth=combo.sizeHint().width())\n        self.explanation.setContentsMargins(8, 8, 0, 0)\n        font = self.explanation.font()\n        font.setPointSizeF(0.85 * font.pointSizeF())\n        self.explanation.setFont(font)\n\n        gui.radioButtons(\n            box, self, value=\"output_calibration\",\n            btnLabels=(\"Sigmoid calibration\", \"Isotonic calibration\"),\n            label=\"Output model calibration\", callback=self.commit.deferred)\n\n        self.info_box = gui.widgetBox(self.controlArea, \"Info\")\n        self.info_label = gui.widgetLabel(self.info_box)\n\n        gui.auto_apply(self.buttonsArea, self, \"auto_commit\")\n\n        self.plotview = GraphicsView()\n        self.plot = PlotItem(enableMenu=False)\n        self.plot.parameter_setter = ParameterSetter(self.plot)\n        self.plot.setMouseEnabled(False, False)\n        self.plot.hideButtons()\n        for axis_name in (\"bottom\", \"left\"):\n            axis = self.plot.getAxis(axis_name)\n            # Remove the condition (that is, allow setting this for bottom\n            # axis) when pyqtgraph is fixed\n            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930\n            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932\n            if axis_name != \"bottom\":  # remove if when pyqtgraph is fixed\n                axis.setStyle(stopAxisAtTick=(True, True))\n\n        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)\n        self.plotview.setCentralItem(self.plot)\n\n        self.mainArea.layout().addWidget(self.plotview)\n        self._set_explanation()\n\n        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)\n\n    @Inputs.evaluation_results\n    def set_results(self, results):\n        self.closeContext()\n        self.clear()\n        self.Error.clear()\n        self.Information.clear()\n\n        self.results = None\n        if results is not None:\n            if not results.domain.has_discrete_class:\n                self.Error.non_discrete_target()\n            elif not results.actual.size:\n                self.Error.empty_input()\n            elif np.any(np.isnan(results.actual)):\n                self.Error.nan_classes()\n            else:\n                self.results = results\n                self._initialize(results)\n                class_var = self.results.domain.class_var\n                self.target_index = int(len(class_var.values) == 2)\n                self.openContext(class_var, self.classifier_names)\n                self._replot()\n\n        self.commit.now()\n\n    def clear(self):\n        self.plot.clear()\n        self.results = None\n        self.classifier_names = []\n        self.selected_classifiers = []\n        self.target_cb.clear()\n        self.colors = []\n\n    def target_index_changed(self):\n        if len(self.results.domain.class_var.values) == 2:\n            self.threshold = 1 - self.threshold\n        self._set_explanation()\n        self._replot()\n        self.commit.deferred()\n\n    def score_changed(self):\n        self._set_explanation()\n        self._replot()\n        if self._last_score_value != self.score:\n            self.commit.deferred()\n            self._last_score_value = self.score\n\n    def _set_explanation(self):\n        explanation = Metrics[self.score].explanation\n        if explanation:\n            self.explanation.setText(explanation)\n            self.explanation.show()\n        else:\n            self.explanation.hide()\n\n        if self.score == 0:\n            self.controls.output_calibration.show()\n            self.info_box.hide()\n        else:\n            self.controls.output_calibration.hide()\n            self.info_box.show()\n\n        axis = self.plot.getAxis(\"bottom\")\n        axis.setLabel(\"Predicted probability\" if self.score == 0\n                      else \"Threshold probability to classify as positive\")\n\n        axis = self.plot.getAxis(\"left\")\n        axis.setLabel(Metrics[self.score].name)\n\n    def _initialize(self, results):\n        n = len(results.predicted)\n        names = getattr(results, \"learner_names\", None)\n        if names is None:\n            names = [\"#{}\".format(i + 1) for i in range(n)]\n\n        self.classifier_names = names\n        self.colors = colorpalettes.get_default_curve_colors(n)\n\n        for i in range(n):\n            item = self.classifiers_list_box.item(i)\n            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))\n\n        self.selected_classifiers = list(range(n))\n        self.target_cb.addItems(results.domain.class_var.values)\n        self.target_index = 0\n\n    def _rug(self, data, pen_args):\n        color = pen_args[\"pen\"].color()\n        rh = 0.025\n        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]\n        rug_x_true = rug_x[data.ytrue].ravel()\n        rug_x_false = rug_x[~data.ytrue].ravel()\n\n        rug_y_true = np.ones_like(rug_x_true)\n        rug_y_true[1::2] = 1 - rh\n        rug_y_false = np.zeros_like(rug_x_false)\n        rug_y_false[1::2] = rh\n\n        self.plot.plot(\n            rug_x_false, rug_y_false,\n            pen=color, connect=\"pairs\", antialias=True)\n        self.plot.plot(\n            rug_x_true, rug_y_true,\n            pen=color, connect=\"pairs\", antialias=True)\n\n    def plot_metrics(self, data, metrics, pen_args):\n        if metrics is None:\n            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)\n        ys = [metric(data) for metric in metrics]\n        for y in ys:\n            self.plot.plot(data.probs, y, **pen_args)\n        return data.probs, ys\n\n    def _prob_curve(self, ytrue, probs, pen_args):\n        xmin, xmax = probs.min(), probs.max()\n        x = np.linspace(xmin, xmax, 100)\n        if xmax != xmin:\n            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))\n            y = f(x)\n        else:\n            y = np.full(100, xmax)\n\n        self.plot.plot(x, y, symbol=\"+\", symbolSize=4, **pen_args)\n        return x, (y, )\n\n    def _setup_plot(self):\n        target = self.target_index\n        results = self.results\n        metrics = Metrics[self.score].functions\n        plot_folds = self.fold_curves and results.folds is not None\n        self.scores = []\n\n        if not self._check_class_presence(results.actual == target):\n            return\n\n        self.Warning.omitted_folds.clear()\n        self.Warning.omitted_nan_prob_points.clear()\n        no_valid_models = []\n        shadow_width = 4 + 4 * plot_folds\n        for clsf in self.selected_classifiers:\n            data = Curves.from_results(results, target, clsf)\n            if data.tot == 0:  # all probabilities are nan\n                no_valid_models.append(clsf)\n                continue\n            if data.tot != results.probabilities.shape[1]:  # some are nan\n                self.Warning.omitted_nan_prob_points()\n\n            color = self.colors[clsf]\n            pen_args = dict(\n                pen=pg.mkPen(color, width=1), antiAlias=True,\n                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))\n            self.scores.append(\n                (self.classifier_names[clsf],\n                 self.plot_metrics(data, metrics, pen_args)))\n\n            if self.display_rug:\n                self._rug(data, pen_args)\n\n            if plot_folds:\n                pen_args = dict(\n                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),\n                    antiAlias=True)\n                for fold in range(len(results.folds)):\n                    fold_results = results.get_fold(fold)\n                    fold_curve = Curves.from_results(fold_results, target, clsf)\n                    # Can't check this before: p and n can be 0 because of\n                    # nan probabilities\n                    if fold_curve.p * fold_curve.n == 0:\n                        self.Warning.omitted_folds()\n                    self.plot_metrics(fold_curve, metrics, pen_args)\n\n        if no_valid_models:\n            self.Warning.no_valid_data(\n                \", \".join(self.classifier_names[i] for i in no_valid_models))\n\n        if self.score == 0:\n            self.plot.plot([0, 1], [0, 1], antialias=True)\n        else:\n            self.line = pg.InfiniteLine(\n                pos=self.threshold, movable=True,\n                pen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=2),\n                hoverPen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=3),\n                bounds=(0, 1),\n            )\n            self.line.sigPositionChanged.connect(self.threshold_change)\n            self.line.sigPositionChangeFinished.connect(\n                self.threshold_change_done)\n            self.plot.addItem(self.line)\n\n    def _check_class_presence(self, ytrue):\n        self.Error.all_target_class.clear()\n        self.Error.no_target_class.clear()\n        if np.max(ytrue) == 0:\n            self.Error.no_target_class()\n            return False\n        if np.min(ytrue) == 1:\n            self.Error.all_target_class()\n            return False\n        return True\n\n    def _replot(self):\n        self.plot.clear()\n        if self.results is not None:\n            self._setup_plot()\n        self._update_info()\n\n    def _on_display_rug_changed(self):\n        self._replot()\n\n    def _on_selection_changed(self):\n        self._replot()\n        self.commit.deferred()\n\n    def threshold_change(self):\n        self.threshold = round(self.line.pos().x(), 2)\n        self.line.setPos(self.threshold)\n        self._update_info()\n\n    def get_info_text(self, short):\n        if short:\n            def elided(s):\n                return s[:17] + \"...\" if len(s) > 20 else s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold: p=</th>\n                                <td colspan='4'>{self.threshold:.2f}<br/></td>\n                            </tr>\"\"\"\n\n        else:\n            def elided(s):\n                return s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold:</th>\n                                <td colspan='4'>p = {self.threshold:.2f}<br/>\n                                </td>\n                                <tr/>\n                            </tr>\"\"\"\n\n        if self.scores is not None:\n            short_names = Metrics[self.score].short_names\n            if short_names:\n                text += f\"\"\"<tr>\n                                <th></th>\n                                {\"<td></td>\".join(f\"<td align='right'>{n}</td>\"\n                                                  for n in short_names)}\n                            </tr>\"\"\"\n            for name, (probs, curves) in self.scores:\n                ind = min(np.searchsorted(probs, self.threshold),\n                          len(probs) - 1)\n                text += f\"<tr><th align='right'>{elided(name)}:</th>\"\n                text += \"<td>/</td>\".join(f'<td>{curve[ind]:.3f}</td>'\n                                          for curve in curves)\n                text += \"</tr>\"\n            text += \"<table>\"\n            return text\n        return None\n\n    def _update_info(self):\n        self.info_label.setText(self.get_info_text(short=True))\n\n    def threshold_change_done(self):\n        self.commit.deferred()\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n                    (len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\n            if len(problems) == 1:\n                self.Information.no_output(problems[0])\n            elif problems:\n                self.Information.no_output(\n                    \"\".join(f\"\\n - {problem}\" for problem in problems))\n            else:\n                clsf_idx = self.selected_classifiers[0]\n                model = results.models[0, clsf_idx]\n                if self.score == 0:\n                    cal_learner = CalibratedLearner(\n                        None, self.output_calibration)\n                    wrapped = cal_learner.get_model(\n                        model, results.actual, results.probabilities[clsf_idx])\n                else:\n                    threshold = [1 - self.threshold,\n                                 self.threshold][self.target_index]\n                    wrapped = ThresholdClassifier(model, threshold)\n\n        self.Outputs.calibrated_model.send(wrapped)\n\n    def send_report(self):\n        if self.results is None:\n            return\n        self.report_items((\n            (\"Target class\", self.target_cb.currentText()),\n            (\"Output model calibration\",\n             self.score == 0\n             and (\"Sigmoid calibration\",\n                  \"Isotonic calibration\")[self.output_calibration])\n        ))\n        caption = report.list_legend(self.classifiers_list_box,\n                                     self.selected_classifiers)\n        self.report_plot()\n        self.report_caption(caption)\n        self.report_caption(self.controls.score.currentText())\n\n        if self.score != 0:\n            self.report_raw(self.get_info_text(short=False))\n\n    def set_visual_settings(self, key, value):\n        self.plot.parameter_setter.set_parameter(key, value)\n        self.visual_settings[key] = value",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 94,
                            "end_lineno": 541,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "import copy\nimport warnings\nfrom unittest.mock import Mock, patch\n\nimport numpy as np\nfrom AnyQt.QtCore import QItemSelection\nfrom pyqtgraph import InfiniteLine\n\nfrom sklearn.exceptions import ConvergenceWarning\n\nfrom Orange.data import Table, DiscreteVariable, Domain, ContinuousVariable\nimport Orange.evaluation\nimport Orange.classification\nfrom Orange.evaluation import Results\nfrom Orange.evaluation.performance_curves import Curves\n\nfrom Orange.widgets.evaluate.tests.base import EvaluateTest\nfrom Orange.widgets.evaluate.owcalibrationplot import OWCalibrationPlot\nfrom Orange.widgets.tests.base import WidgetTest\nfrom Orange.tests import test_filename\n\n\nclass TestOWCalibrationPlot(WidgetTest, EvaluateTest):\n    def setUp(self):\n        super().setUp()\n\n        n, p = (0, 1)\n        actual, probs = np.array([\n            (p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52),\n            (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37),\n            (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T\n        self.curves = Curves(actual, probs)\n        probs2 = (probs + 0.5) / 2 + 1\n        self.curves2 = Curves(actual, probs2)\n        pred = probs > 0.5\n        pred2 = probs2 > 0.5\n        probs = np.vstack((1 - probs, probs)).T\n        probs2 = np.vstack((1 - probs2, probs2)).T\n        domain = Domain([], DiscreteVariable(\"y\", values=(\"a\", \"b\")))\n        self.results = Results(\n            domain=domain,\n            actual=actual,\n            folds=np.array([Ellipsis]),\n            models=np.array([[Mock(), Mock()]]),\n            row_indices=np.arange(19),\n            predicted=np.array((pred, pred2)),\n            probabilities=np.array([probs, probs2]))\n\n        self.lenses = data = Table(test_filename(\"datasets/lenses.tab\"))\n        majority = Orange.classification.MajorityLearner()\n        majority.name = \"majority\"\n        knn3 = Orange.classification.KNNLearner(n_neighbors=3)\n        knn3.name = \"knn-3\"\n        knn1 = Orange.classification.KNNLearner(n_neighbors=1)\n        knn1.name = \"knn-1\"\n        self.lenses_results = Orange.evaluation.TestOnTestData(\n            store_data=True, store_models=True)(\n                data=data[::2], test_data=data[1::2],\n                learners=[majority, knn3, knn1])\n        self.lenses_results.learner_names = [\"majority\", \"knn-3\", \"knn-1\"]\n\n        self.widget = self.create_widget(OWCalibrationPlot)  # type: OWCalibrationPlot\n        warnings.filterwarnings(\"ignore\", \".*\", ConvergenceWarning)\n\n    def test_initialization(self):\n        \"\"\"Test initialization of lists and combos\"\"\"\n        def check_clsfr_names(names):\n            self.assertEqual(widget.classifier_names, names)\n            clsf_list = widget.controls.selected_classifiers\n            self.assertEqual(\n                [clsf_list.item(i).text() for i in range(clsf_list.count())],\n                names)\n\n        widget = self.widget\n        tcomb = widget.controls.target_index\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        check_clsfr_names([\"majority\", \"knn-3\", \"knn-1\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1, 2])\n        self.assertEqual(\n            tuple(tcomb.itemText(i) for i in range(tcomb.count())),\n            self.lenses.domain.class_var.values)\n        self.assertEqual(widget.target_index, 0)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        check_clsfr_names([\"#1\", \"#2\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1])\n        self.assertEqual(\n            [tcomb.itemText(i) for i in range(tcomb.count())], [\"a\", \"b\"])\n        self.assertEqual(widget.target_index, 1)\n\n        self.send_signal(widget.Inputs.evaluation_results, None)\n        check_clsfr_names([])\n        self.assertEqual(widget.selected_classifiers, [])\n        self.assertEqual(widget.controls.target_index.count(), 0)\n\n    def test_empty_input_error(self):\n        \"\"\"Show an error when data is present but empty\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.empty_input.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    def test_regression_input_error(self):\n        \"\"\"Show an error for regression data\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.domain = Domain([], ContinuousVariable(\"y\"))\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.non_discrete_target.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    @staticmethod\n    def _set_combo(combo, val):\n        combo.setCurrentIndex(val)\n        combo.activated[int].emit(val)\n        combo.activated[str].emit(combo.currentText())\n\n    @staticmethod\n    def _set_radio_buttons(radios, val):\n        radios.buttons[val].click()\n\n    @staticmethod\n    def _set_list_selection(listview, selection):\n        model = listview.model()\n        selectionmodel = listview.selectionModel()\n        itemselection = QItemSelection()\n        for item in selection:\n            itemselection.select(model.index(item, 0), model.index(item, 0))\n        selectionmodel.select(itemselection, selectionmodel.ClearAndSelect)\n\n    def _set_threshold(self, pos, done):\n        _, line = self._get_curves()\n        line.setPos(pos)\n        if done:\n            line.sigPositionChangeFinished.emit(line)\n        else:\n            line.sigPositionChanged.emit(line)\n\n    def _get_curves(self):\n        plot_items = self.widget.plot.items[:]\n        for i, item in enumerate(plot_items):\n            if isinstance(item, InfiniteLine):\n                del plot_items[i]\n                return plot_items, item\n        return plot_items, None\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_plotting_curves(self, *_):\n        \"\"\"Curve coordinates match those computed by `Curves`\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        combo = widget.controls.score\n\n        c = self.curves\n        combinations = ([c.ca()],\n                        [c.f1()],\n                        [c.sensitivity(), c.specificity()],\n                        [c.precision(), c.recall()],\n                        [c.ppv(), c.npv()],\n                        [c.tpr(), c.fpr()])\n        for idx, curves_data in enumerate(combinations, start=1):\n            self._set_combo(combo, idx)\n            curves, line = self._get_curves()\n            self.assertEqual(len(curves), len(curves_data))\n            self.assertIsNotNone(line)\n            for curve in curves:\n                x, y = curve.getData()\n                np.testing.assert_almost_equal(x, self.curves.probs)\n                for i, curve_data in enumerate(curves_data):\n                    if np.max(curve_data - y) < 1e-6:\n                        del curves_data[i]\n                        break\n                else:\n                    self.fail(f\"invalid curve for {combo.currentText()}\")\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_multiple_fold_curves(self, *_):\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(widget.controls.selected_classifiers, [0])\n        self._set_combo(widget.controls.score, 1)  # CA\n\n        self.results.folds = [slice(1, 5), slice(5, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 2)\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 3)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_change_target_class(self, *_):\n        \"\"\"Changing target combo changes the curves\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        score_combo = widget.controls.score\n        target_combo = widget.controls.target_index\n\n        self._set_combo(score_combo, 1)  # ca\n        self._set_combo(target_combo, 1)\n        (ca, ), _ = self._get_curves()\n        np.testing.assert_almost_equal(ca.getData()[1], self.curves.ca())\n\n        self._set_combo(target_combo, 0)\n        (ca, ), _ = self._get_curves()\n        curves = Curves(1 - self.curves.ytrue, 1 - self.curves.probs[:-1])\n        np.testing.assert_almost_equal(ca.getData()[1], curves.ca())\n\n    def test_changing_score_explanation(self):\n        \"\"\"Changing score hides/shows explanation and options for calibration\"\"\"\n        widget = self.widget\n        score_combo = widget.controls.score\n        explanation = widget.explanation\n        calibrations = widget.controls.output_calibration\n\n        self._set_combo(score_combo, 1)  # ca\n        self.assertTrue(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n        self._set_combo(score_combo, 0)  # calibration\n        self.assertTrue(explanation.isHidden())\n        self.assertFalse(calibrations.isHidden())\n\n        self._set_combo(score_combo, 3)  # sens/spec\n        self.assertFalse(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n    def test_rug(self):\n        \"\"\"Test rug appearance and positions\"\"\"\n        def get_rugs():\n            rugs = [None, None]\n            for item in widget.plot.items:\n                if item.curve.opts.get(\"connect\", \"\") == \"pairs\":\n                    x, y = item.getData()\n                    np.testing.assert_almost_equal(x[::2], x[1::2])\n                    rugs[int(y[0] == 1)] = x[::2]\n            return rugs\n\n        widget = self.widget\n        widget.display_rug = True\n        model_list = widget.controls.selected_classifiers\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n\n        self._set_list_selection(model_list, [0])\n        probs = self.curves.probs[:-1]\n        truex = probs[self.curves.ytrue == 1]\n        falsex = probs[self.curves.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex)\n        np.testing.assert_almost_equal(top, truex)\n\n        # Switching targets should switch rugs and takes other probabilities\n        self._set_combo(widget.controls.target_index, 0)\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, (1 - truex)[::-1])\n        np.testing.assert_almost_equal(top, (1 - falsex)[::-1])\n        self._set_combo(widget.controls.target_index, 1)\n\n        # Changing models gives a different rug\n        self._set_list_selection(model_list, [1])\n        probs2 = self.curves2.probs[:-1]\n        truex2 = probs2[self.curves2.ytrue == 1]\n        falsex2 = probs2[self.curves2.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex2)\n        np.testing.assert_almost_equal(top, truex2)\n\n        # Two models - two rugs - four rug items\n        self._set_list_selection(model_list, [0, 1])\n        self.assertEqual(sum(item.curve.opts.get(\"connect\", \"\") == \"pairs\"\n                             for item in widget.plot.items), 4)\n\n        # No models - no rugs\n        self._set_list_selection(model_list, [])\n        self.assertEqual(get_rugs(), [None, None])\n\n        # Bring the rug back\n        self._set_list_selection(model_list, [1])\n        self.assertIsNotNone(get_rugs()[0])\n\n        # Disable it with checkbox\n        widget.controls.display_rug.click()\n        self.assertEqual(get_rugs(), [None, None])\n\n    def test_calibration_curve(self):\n        \"\"\"Test the correct number of calibration curves\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        widget.display_rug = False\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertEqual(len(widget.plot.items), 3)  # 2 + diagonal\n\n        self._set_list_selection(model_list, [1])\n        self.assertEqual(len(widget.plot.items), 2)\n\n        self._set_list_selection(model_list, [])\n        self.assertEqual(len(widget.plot.items), 1)\n\n    def test_threshold_change_updates_info(self):\n        \"\"\"Changing the threshold updates info label\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n\n        original_text = widget.info_label.text()\n        self._set_threshold(0.3, False)\n        self.assertNotEqual(widget.info_label.text(), original_text)\n\n    def test_threshold_rounding(self):\n        \"\"\"Threshold is rounded to two decimals\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n        self._set_threshold(0.367, False)\n        self.assertAlmostEqual(widget.threshold, 0.37)\n\n    def test_threshold_flips_on_two_classes(self):\n        \"\"\"Threshold changes to 1 - threshold if *binary* class is switched\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.75)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.25)\n\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_apply_no_output(self, *_):\n        \"\"\"Test no output warnings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n\n        multiple_folds, multiple_selected, no_models, non_binary_class = \"abcd\"\n        messages = {\n            multiple_folds:\n                \"each training data sample produces a different model\",\n            no_models:\n                \"test results do not contain stored models - try testing on \"\n                \"separate data or on training data\",\n            multiple_selected:\n                \"select a single model - the widget can output only one\",\n            non_binary_class:\n                \"cannot calibrate non-binary classes\"}\n\n        def test_shown(shown):\n            widget_msg = widget.Information.no_output\n            output = self.get_output(widget.Outputs.calibrated_model)\n            if not shown:\n                self.assertFalse(widget_msg.is_shown())\n                self.assertIsNotNone(output)\n            else:\n                self.assertTrue(widget_msg.is_shown())\n                self.assertIsNone(output)\n                for msg_id in shown:\n                    msg = messages[msg_id]\n                    self.assertIn(msg, widget_msg.formatted,\n                                  f\"{msg} not included in the message\")\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        test_shown({multiple_selected})\n\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n        self._set_list_selection(model_list, [0, 1])\n\n        self.results.models = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, no_models})\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        test_shown({multiple_selected, non_binary_class})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({non_binary_class})\n\n        self.results.folds = [slice(0, 5), slice(5, 10), slice(10, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 3)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, multiple_folds})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({multiple_folds})\n\n        self._set_combo(widget.controls.score, 0)  # calibration\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 1])\n        test_shown({multiple_selected})\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    def test_output_threshold_classifier(self, threshold_classifier):\n        \"\"\"Test threshold classifier on output\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.results.models.ravel()\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        widget.target_index = 1\n\n        widget.threshold = 0.3\n        self._set_combo(widget.controls.score, 1)  # CA\n        model = self.get_output(widget.Outputs.calibrated_model)\n        threshold_classifier.assert_called_with(models[0], 0.3)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, False)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = False\n        self._set_threshold(0.35, True)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, True)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 0)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 1)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_list_selection(model_list, [1])\n        threshold_classifier.assert_called_with(models[1], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_output_calibrated_classifier(self, calibrated_learner):\n        \"\"\"Test calibrated classifier on output\"\"\"\n        calibrated_instance = calibrated_learner.return_value\n        get_model = calibrated_instance.get_model\n\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.lenses_results.models.ravel()\n        results = self.lenses_results\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 0)\n\n        self._set_list_selection(model_list, [1])\n\n        self._set_radio_buttons(widget.controls.output_calibration, 0)\n        calibrated_learner.assert_called_with(None, 0)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_list_selection(model_list, [0])\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[0])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[0])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n    def test_contexts(self):\n        \"\"\"Test storing and retrieving context settings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 2])\n        self._set_combo(target_combo, 2)\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        self._set_combo(target_combo, 0)\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self.assertEqual(widget.selected_classifiers, [0, 2])\n        self.assertEqual(widget.target_index, 2)\n\n    def test_report(self):\n        \"\"\"Test that report does not crash\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        widget.send_report()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class(self, *_):\n        \"\"\"Curves are not plotted if all data belongs to (non)-target\"\"\"\n        def check_error(shown):\n            for error in (errors.no_target_class, errors.all_target_class,\n                          errors.nan_classes):\n                self.assertEqual(error.is_shown(), error is shown,\n                                 f\"{error} is unexpectedly\"\n                                 f\"{'' if error.is_shown() else ' not'} shown\")\n            if shown is not None:\n                self.assertEqual(len(widget.plot.items), 0)\n            else:\n                self.assertGreater(len(widget.plot.items), 0)\n\n        widget = self.widget\n        errors = widget.Error\n        widget.display_rug = True\n        combo = widget.controls.score\n\n        original_actual = self.results.actual.copy()\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        for idx in range(combo.count()):\n            self._set_combo(combo, idx)\n            self.results.actual[:] = 0\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.no_target_class)\n\n            self.results.actual[:] = 1\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.all_target_class)\n\n            self.results.actual[:] = original_actual\n            self.results.actual[3] = np.nan\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.nan_classes)\n\n            self.results.actual[:] = original_actual\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(None)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class_folds(self, *_):\n        \"\"\"Curves for single-class folds are not plotted\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        results = self.lenses_results\n        results.folds = [slice(0, 5), slice(5, 19)]\n        results.models = results.models.repeat(2, axis=0)\n        results.actual = results.actual.copy()\n        results.actual[:3] = 0\n        results.probabilities[1, 3:5] = np.nan\n        # after this, model 1 has just negative instances in fold 0\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n        widget.controls.fold_curves.click()\n        self.assertTrue(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_warn_nan_probabilities(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        self.results.probabilities[1, 3] = np.nan\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())\n        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n",
            "new_file_content": "import copy\nimport warnings\nimport unittest\nfrom unittest.mock import Mock, patch\n\nimport numpy as np\nfrom AnyQt.QtCore import QItemSelection\nfrom pyqtgraph import InfiniteLine\n\nfrom sklearn.exceptions import ConvergenceWarning\n\nfrom Orange.data import Table, DiscreteVariable, Domain, ContinuousVariable\nimport Orange.evaluation\nimport Orange.classification\nfrom Orange.evaluation import Results\nfrom Orange.evaluation.performance_curves import Curves\n\nfrom Orange.widgets.evaluate.tests.base import EvaluateTest\nfrom Orange.widgets.evaluate.owcalibrationplot import OWCalibrationPlot\nfrom Orange.widgets.tests.base import WidgetTest\nfrom Orange.tests import test_filename\n\n\nclass TestOWCalibrationPlot(WidgetTest, EvaluateTest):\n    def setUp(self):\n        super().setUp()\n\n        n, p = (0, 1)\n        actual, probs = np.array([\n            (p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52),\n            (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37),\n            (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T\n        self.curves = Curves(actual, probs)\n        probs2 = (probs + 0.5) / 2 + 1\n        self.curves2 = Curves(actual, probs2)\n        pred = probs > 0.5\n        pred2 = probs2 > 0.5\n        probs = np.vstack((1 - probs, probs)).T\n        probs2 = np.vstack((1 - probs2, probs2)).T\n        domain = Domain([], DiscreteVariable(\"y\", values=(\"a\", \"b\")))\n        self.results = Results(\n            domain=domain,\n            actual=actual,\n            folds=np.array([Ellipsis]),\n            models=np.array([[Mock(), Mock()]]),\n            row_indices=np.arange(19),\n            predicted=np.array((pred, pred2)),\n            probabilities=np.array([probs, probs2]))\n\n        self.lenses = data = Table(test_filename(\"datasets/lenses.tab\"))\n        majority = Orange.classification.MajorityLearner()\n        majority.name = \"majority\"\n        knn3 = Orange.classification.KNNLearner(n_neighbors=3)\n        knn3.name = \"knn-3\"\n        knn1 = Orange.classification.KNNLearner(n_neighbors=1)\n        knn1.name = \"knn-1\"\n        self.lenses_results = Orange.evaluation.TestOnTestData(\n            store_data=True, store_models=True)(\n                data=data[::2], test_data=data[1::2],\n                learners=[majority, knn3, knn1])\n        self.lenses_results.learner_names = [\"majority\", \"knn-3\", \"knn-1\"]\n\n        self.widget = self.create_widget(OWCalibrationPlot)  # type: OWCalibrationPlot\n        warnings.filterwarnings(\"ignore\", \".*\", ConvergenceWarning)\n\n    def test_initialization(self):\n        \"\"\"Test initialization of lists and combos\"\"\"\n        def check_clsfr_names(names):\n            self.assertEqual(widget.classifier_names, names)\n            clsf_list = widget.controls.selected_classifiers\n            self.assertEqual(\n                [clsf_list.item(i).text() for i in range(clsf_list.count())],\n                names)\n\n        widget = self.widget\n        tcomb = widget.controls.target_index\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        check_clsfr_names([\"majority\", \"knn-3\", \"knn-1\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1, 2])\n        self.assertEqual(\n            tuple(tcomb.itemText(i) for i in range(tcomb.count())),\n            self.lenses.domain.class_var.values)\n        self.assertEqual(widget.target_index, 0)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        check_clsfr_names([\"#1\", \"#2\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1])\n        self.assertEqual(\n            [tcomb.itemText(i) for i in range(tcomb.count())], [\"a\", \"b\"])\n        self.assertEqual(widget.target_index, 1)\n\n        self.send_signal(widget.Inputs.evaluation_results, None)\n        check_clsfr_names([])\n        self.assertEqual(widget.selected_classifiers, [])\n        self.assertEqual(widget.controls.target_index.count(), 0)\n\n    def test_empty_input_error(self):\n        \"\"\"Show an error when data is present but empty\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.empty_input.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    def test_regression_input_error(self):\n        \"\"\"Show an error for regression data\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.domain = Domain([], ContinuousVariable(\"y\"))\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.non_discrete_target.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    @staticmethod\n    def _set_combo(combo, val):\n        combo.setCurrentIndex(val)\n        combo.activated[int].emit(val)\n        combo.activated[str].emit(combo.currentText())\n\n    @staticmethod\n    def _set_radio_buttons(radios, val):\n        radios.buttons[val].click()\n\n    @staticmethod\n    def _set_list_selection(listview, selection):\n        model = listview.model()\n        selectionmodel = listview.selectionModel()\n        itemselection = QItemSelection()\n        for item in selection:\n            itemselection.select(model.index(item, 0), model.index(item, 0))\n        selectionmodel.select(itemselection, selectionmodel.ClearAndSelect)\n\n    def _set_threshold(self, pos, done):\n        _, line = self._get_curves()\n        line.setPos(pos)\n        if done:\n            line.sigPositionChangeFinished.emit(line)\n        else:\n            line.sigPositionChanged.emit(line)\n\n    def _get_curves(self):\n        plot_items = self.widget.plot.items[:]\n        for i, item in enumerate(plot_items):\n            if isinstance(item, InfiniteLine):\n                del plot_items[i]\n                return plot_items, item\n        return plot_items, None\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_plotting_curves(self, *_):\n        \"\"\"Curve coordinates match those computed by `Curves`\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        combo = widget.controls.score\n\n        c = self.curves\n        combinations = ([c.ca()],\n                        [c.f1()],\n                        [c.sensitivity(), c.specificity()],\n                        [c.precision(), c.recall()],\n                        [c.ppv(), c.npv()],\n                        [c.tpr(), c.fpr()])\n        for idx, curves_data in enumerate(combinations, start=1):\n            self._set_combo(combo, idx)\n            curves, line = self._get_curves()\n            self.assertEqual(len(curves), len(curves_data))\n            self.assertIsNotNone(line)\n            for curve in curves:\n                x, y = curve.getData()\n                np.testing.assert_almost_equal(x, self.curves.probs)\n                for i, curve_data in enumerate(curves_data):\n                    if np.max(curve_data - y) < 1e-6:\n                        del curves_data[i]\n                        break\n                else:\n                    self.fail(f\"invalid curve for {combo.currentText()}\")\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_multiple_fold_curves(self, *_):\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(widget.controls.selected_classifiers, [0])\n        self._set_combo(widget.controls.score, 1)  # CA\n\n        self.results.folds = [slice(1, 5), slice(5, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 2)\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 3)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_change_target_class(self, *_):\n        \"\"\"Changing target combo changes the curves\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        score_combo = widget.controls.score\n        target_combo = widget.controls.target_index\n\n        self._set_combo(score_combo, 1)  # ca\n        self._set_combo(target_combo, 1)\n        (ca, ), _ = self._get_curves()\n        np.testing.assert_almost_equal(ca.getData()[1], self.curves.ca())\n\n        self._set_combo(target_combo, 0)\n        (ca, ), _ = self._get_curves()\n        curves = Curves(1 - self.curves.ytrue, 1 - self.curves.probs[:-1])\n        np.testing.assert_almost_equal(ca.getData()[1], curves.ca())\n\n    def test_changing_score_explanation(self):\n        \"\"\"Changing score hides/shows explanation and options for calibration\"\"\"\n        widget = self.widget\n        score_combo = widget.controls.score\n        explanation = widget.explanation\n        calibrations = widget.controls.output_calibration\n\n        self._set_combo(score_combo, 1)  # ca\n        self.assertTrue(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n        self._set_combo(score_combo, 0)  # calibration\n        self.assertTrue(explanation.isHidden())\n        self.assertFalse(calibrations.isHidden())\n\n        self._set_combo(score_combo, 3)  # sens/spec\n        self.assertFalse(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n    def test_rug(self):\n        \"\"\"Test rug appearance and positions\"\"\"\n        def get_rugs():\n            rugs = [None, None]\n            for item in widget.plot.items:\n                if item.curve.opts.get(\"connect\", \"\") == \"pairs\":\n                    x, y = item.getData()\n                    np.testing.assert_almost_equal(x[::2], x[1::2])\n                    rugs[int(y[0] == 1)] = x[::2]\n            return rugs\n\n        widget = self.widget\n        widget.display_rug = True\n        model_list = widget.controls.selected_classifiers\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n\n        self._set_list_selection(model_list, [0])\n        probs = self.curves.probs[:-1]\n        truex = probs[self.curves.ytrue == 1]\n        falsex = probs[self.curves.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex)\n        np.testing.assert_almost_equal(top, truex)\n\n        # Switching targets should switch rugs and takes other probabilities\n        self._set_combo(widget.controls.target_index, 0)\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, (1 - truex)[::-1])\n        np.testing.assert_almost_equal(top, (1 - falsex)[::-1])\n        self._set_combo(widget.controls.target_index, 1)\n\n        # Changing models gives a different rug\n        self._set_list_selection(model_list, [1])\n        probs2 = self.curves2.probs[:-1]\n        truex2 = probs2[self.curves2.ytrue == 1]\n        falsex2 = probs2[self.curves2.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex2)\n        np.testing.assert_almost_equal(top, truex2)\n\n        # Two models - two rugs - four rug items\n        self._set_list_selection(model_list, [0, 1])\n        self.assertEqual(sum(item.curve.opts.get(\"connect\", \"\") == \"pairs\"\n                             for item in widget.plot.items), 4)\n\n        # No models - no rugs\n        self._set_list_selection(model_list, [])\n        self.assertEqual(get_rugs(), [None, None])\n\n        # Bring the rug back\n        self._set_list_selection(model_list, [1])\n        self.assertIsNotNone(get_rugs()[0])\n\n        # Disable it with checkbox\n        widget.controls.display_rug.click()\n        self.assertEqual(get_rugs(), [None, None])\n\n    def test_calibration_curve(self):\n        \"\"\"Test the correct number of calibration curves\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        widget.display_rug = False\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertEqual(len(widget.plot.items), 3)  # 2 + diagonal\n\n        self._set_list_selection(model_list, [1])\n        self.assertEqual(len(widget.plot.items), 2)\n\n        self._set_list_selection(model_list, [])\n        self.assertEqual(len(widget.plot.items), 1)\n\n    def test_threshold_change_updates_info(self):\n        \"\"\"Changing the threshold updates info label\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n\n        original_text = widget.info_label.text()\n        self._set_threshold(0.3, False)\n        self.assertNotEqual(widget.info_label.text(), original_text)\n\n    def test_threshold_rounding(self):\n        \"\"\"Threshold is rounded to two decimals\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n        self._set_threshold(0.367, False)\n        self.assertAlmostEqual(widget.threshold, 0.37)\n\n    def test_threshold_flips_on_two_classes(self):\n        \"\"\"Threshold changes to 1 - threshold if *binary* class is switched\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.75)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.25)\n\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_apply_no_output(self, *_):\n        \"\"\"Test no output warnings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n\n        multiple_folds, multiple_selected, no_models, non_binary_class = \"abcd\"\n        messages = {\n            multiple_folds:\n                \"each training data sample produces a different model\",\n            no_models:\n                \"test results do not contain stored models - try testing on \"\n                \"separate data or on training data\",\n            multiple_selected:\n                \"select a single model - the widget can output only one\",\n            non_binary_class:\n                \"cannot calibrate non-binary classes\"}\n\n        def test_shown(shown):\n            widget_msg = widget.Information.no_output\n            output = self.get_output(widget.Outputs.calibrated_model)\n            if not shown:\n                self.assertFalse(widget_msg.is_shown())\n                self.assertIsNotNone(output)\n            else:\n                self.assertTrue(widget_msg.is_shown())\n                self.assertIsNone(output)\n                for msg_id in shown:\n                    msg = messages[msg_id]\n                    self.assertIn(msg, widget_msg.formatted,\n                                  f\"{msg} not included in the message\")\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        test_shown({multiple_selected})\n\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n        self._set_list_selection(model_list, [0, 1])\n\n        self.results.models = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, no_models})\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        test_shown({multiple_selected, non_binary_class})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({non_binary_class})\n\n        self.results.folds = [slice(0, 5), slice(5, 10), slice(10, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 3)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, multiple_folds})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({multiple_folds})\n\n        self._set_combo(widget.controls.score, 0)  # calibration\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 1])\n        test_shown({multiple_selected})\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    def test_output_threshold_classifier(self, threshold_classifier):\n        \"\"\"Test threshold classifier on output\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.results.models.ravel()\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        widget.target_index = 1\n\n        widget.threshold = 0.3\n        self._set_combo(widget.controls.score, 1)  # CA\n        model = self.get_output(widget.Outputs.calibrated_model)\n        threshold_classifier.assert_called_with(models[0], 0.3)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, False)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = False\n        self._set_threshold(0.35, True)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, True)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 0)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 1)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_list_selection(model_list, [1])\n        threshold_classifier.assert_called_with(models[1], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_output_calibrated_classifier(self, calibrated_learner):\n        \"\"\"Test calibrated classifier on output\"\"\"\n        calibrated_instance = calibrated_learner.return_value\n        get_model = calibrated_instance.get_model\n\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.lenses_results.models.ravel()\n        results = self.lenses_results\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 0)\n\n        self._set_list_selection(model_list, [1])\n\n        self._set_radio_buttons(widget.controls.output_calibration, 0)\n        calibrated_learner.assert_called_with(None, 0)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_list_selection(model_list, [0])\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[0])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[0])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n    def test_contexts(self):\n        \"\"\"Test storing and retrieving context settings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 2])\n        self._set_combo(target_combo, 2)\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        self._set_combo(target_combo, 0)\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self.assertEqual(widget.selected_classifiers, [0, 2])\n        self.assertEqual(widget.target_index, 2)\n\n    def test_report(self):\n        \"\"\"Test that report does not crash\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        widget.send_report()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class(self, *_):\n        \"\"\"Curves are not plotted if all data belongs to (non)-target\"\"\"\n        def check_error(shown):\n            for error in (errors.no_target_class, errors.all_target_class,\n                          errors.nan_classes):\n                self.assertEqual(error.is_shown(), error is shown,\n                                 f\"{error} is unexpectedly\"\n                                 f\"{'' if error.is_shown() else ' not'} shown\")\n            if shown is not None:\n                self.assertEqual(len(widget.plot.items), 0)\n            else:\n                self.assertGreater(len(widget.plot.items), 0)\n\n        widget = self.widget\n        errors = widget.Error\n        widget.display_rug = True\n        combo = widget.controls.score\n\n        original_actual = self.results.actual.copy()\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        for idx in range(combo.count()):\n            self._set_combo(combo, idx)\n            self.results.actual[:] = 0\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.no_target_class)\n\n            self.results.actual[:] = 1\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.all_target_class)\n\n            self.results.actual[:] = original_actual\n            self.results.actual[3] = np.nan\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.nan_classes)\n\n            self.results.actual[:] = original_actual\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(None)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class_folds(self, *_):\n        \"\"\"Curves for single-class folds are not plotted\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        results = self.lenses_results\n        results.folds = [slice(0, 5), slice(5, 19)]\n        results.models = results.models.repeat(2, axis=0)\n        results.actual = results.actual.copy()\n        results.actual[:3] = 0\n        results.probabilities[1, 3:5] = np.nan\n        # after this, model 1 has just negative instances in fold 0\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n        widget.controls.fold_curves.click()\n        self.assertTrue(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_warn_nan_probabilities(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        self.results.probabilities[1, 3] = np.nan\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())\n        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_no_folds(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n\n        self.results.folds = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        widget.commit.now()\n        self.assertIsNotNone(self.get_output(widget.Outputs.calibrated_model))\n\n\nif __name__ == \"__main__\":\n    unittest.main()\n",
            "header": {
                "file": {
                    "path": "Orange/widgets/evaluate/tests/test_owcalibrationplot.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "87f1fdc9e",
                "new_commit_hash": "bfcb9cd91",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/Orange/widgets/evaluate/tests/test_owcalibrationplot.py"
            },
            "plus_file": {
                "path": "b/Orange/widgets/evaluate/tests/test_owcalibrationplot.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1,
                            "length": 5
                        },
                        "new_range": {
                            "start": 1,
                            "length": 6
                        },
                        "section": ""
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "import copy",
                                "type": "context"
                            },
                            {
                                "content": "import warnings",
                                "type": "context"
                            },
                            {
                                "content": "import unittest",
                                "type": "added"
                            },
                            {
                                "content": "from unittest.mock import Mock, patch",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "import numpy as np",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owcalibrationplot.py",
                            "type": "import",
                            "name": "unittest",
                            "content": "import unittest",
                            "ast_type_str": "Import",
                            "start_lineno": 3,
                            "end_lineno": 3,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 637,
                            "length": 3
                        },
                        "new_range": {
                            "start": 638,
                            "length": 19
                        },
                        "section": "class TestOWCalibrationPlot(WidgetTest, EvaluateTest):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())",
                                "type": "context"
                            },
                            {
                                "content": "        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])",
                                "type": "context"
                            },
                            {
                                "content": "        self.assertFalse(widget.Warning.omitted_folds.is_shown())",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")",
                                "type": "added"
                            },
                            {
                                "content": "    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")",
                                "type": "added"
                            },
                            {
                                "content": "    def test_no_folds(self, *_):",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\"Warn about omitted points with nan probabiities\"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "        widget = self.widget",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        self.results.folds = None",
                                "type": "added"
                            },
                            {
                                "content": "        self.send_signal(widget.Inputs.evaluation_results, self.results)",
                                "type": "added"
                            },
                            {
                                "content": "        widget.selected_classifiers = [0]",
                                "type": "added"
                            },
                            {
                                "content": "        widget.commit.now()",
                                "type": "added"
                            },
                            {
                                "content": "        self.assertIsNotNone(self.get_output(widget.Outputs.calibrated_model))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "if __name__ == \"__main__\":",
                                "type": "added"
                            },
                            {
                                "content": "    unittest.main()",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owcalibrationplot.py",
                            "type": "class",
                            "name": "TestOWCalibrationPlot",
                            "content": "class TestOWCalibrationPlot(WidgetTest, EvaluateTest):\n    def setUp(self):\n        super().setUp()\n\n        n, p = (0, 1)\n        actual, probs = np.array([\n            (p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52),\n            (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37),\n            (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T\n        self.curves = Curves(actual, probs)\n        probs2 = (probs + 0.5) / 2 + 1\n        self.curves2 = Curves(actual, probs2)\n        pred = probs > 0.5\n        pred2 = probs2 > 0.5\n        probs = np.vstack((1 - probs, probs)).T\n        probs2 = np.vstack((1 - probs2, probs2)).T\n        domain = Domain([], DiscreteVariable(\"y\", values=(\"a\", \"b\")))\n        self.results = Results(\n            domain=domain,\n            actual=actual,\n            folds=np.array([Ellipsis]),\n            models=np.array([[Mock(), Mock()]]),\n            row_indices=np.arange(19),\n            predicted=np.array((pred, pred2)),\n            probabilities=np.array([probs, probs2]))\n\n        self.lenses = data = Table(test_filename(\"datasets/lenses.tab\"))\n        majority = Orange.classification.MajorityLearner()\n        majority.name = \"majority\"\n        knn3 = Orange.classification.KNNLearner(n_neighbors=3)\n        knn3.name = \"knn-3\"\n        knn1 = Orange.classification.KNNLearner(n_neighbors=1)\n        knn1.name = \"knn-1\"\n        self.lenses_results = Orange.evaluation.TestOnTestData(\n            store_data=True, store_models=True)(\n                data=data[::2], test_data=data[1::2],\n                learners=[majority, knn3, knn1])\n        self.lenses_results.learner_names = [\"majority\", \"knn-3\", \"knn-1\"]\n\n        self.widget = self.create_widget(OWCalibrationPlot)  # type: OWCalibrationPlot\n        warnings.filterwarnings(\"ignore\", \".*\", ConvergenceWarning)\n\n    def test_initialization(self):\n        \"\"\"Test initialization of lists and combos\"\"\"\n        def check_clsfr_names(names):\n            self.assertEqual(widget.classifier_names, names)\n            clsf_list = widget.controls.selected_classifiers\n            self.assertEqual(\n                [clsf_list.item(i).text() for i in range(clsf_list.count())],\n                names)\n\n        widget = self.widget\n        tcomb = widget.controls.target_index\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        check_clsfr_names([\"majority\", \"knn-3\", \"knn-1\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1, 2])\n        self.assertEqual(\n            tuple(tcomb.itemText(i) for i in range(tcomb.count())),\n            self.lenses.domain.class_var.values)\n        self.assertEqual(widget.target_index, 0)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        check_clsfr_names([\"#1\", \"#2\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1])\n        self.assertEqual(\n            [tcomb.itemText(i) for i in range(tcomb.count())], [\"a\", \"b\"])\n        self.assertEqual(widget.target_index, 1)\n\n        self.send_signal(widget.Inputs.evaluation_results, None)\n        check_clsfr_names([])\n        self.assertEqual(widget.selected_classifiers, [])\n        self.assertEqual(widget.controls.target_index.count(), 0)\n\n    def test_empty_input_error(self):\n        \"\"\"Show an error when data is present but empty\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.empty_input.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    def test_regression_input_error(self):\n        \"\"\"Show an error for regression data\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.domain = Domain([], ContinuousVariable(\"y\"))\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.non_discrete_target.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    @staticmethod\n    def _set_combo(combo, val):\n        combo.setCurrentIndex(val)\n        combo.activated[int].emit(val)\n        combo.activated[str].emit(combo.currentText())\n\n    @staticmethod\n    def _set_radio_buttons(radios, val):\n        radios.buttons[val].click()\n\n    @staticmethod\n    def _set_list_selection(listview, selection):\n        model = listview.model()\n        selectionmodel = listview.selectionModel()\n        itemselection = QItemSelection()\n        for item in selection:\n            itemselection.select(model.index(item, 0), model.index(item, 0))\n        selectionmodel.select(itemselection, selectionmodel.ClearAndSelect)\n\n    def _set_threshold(self, pos, done):\n        _, line = self._get_curves()\n        line.setPos(pos)\n        if done:\n            line.sigPositionChangeFinished.emit(line)\n        else:\n            line.sigPositionChanged.emit(line)\n\n    def _get_curves(self):\n        plot_items = self.widget.plot.items[:]\n        for i, item in enumerate(plot_items):\n            if isinstance(item, InfiniteLine):\n                del plot_items[i]\n                return plot_items, item\n        return plot_items, None\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_plotting_curves(self, *_):\n        \"\"\"Curve coordinates match those computed by `Curves`\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        combo = widget.controls.score\n\n        c = self.curves\n        combinations = ([c.ca()],\n                        [c.f1()],\n                        [c.sensitivity(), c.specificity()],\n                        [c.precision(), c.recall()],\n                        [c.ppv(), c.npv()],\n                        [c.tpr(), c.fpr()])\n        for idx, curves_data in enumerate(combinations, start=1):\n            self._set_combo(combo, idx)\n            curves, line = self._get_curves()\n            self.assertEqual(len(curves), len(curves_data))\n            self.assertIsNotNone(line)\n            for curve in curves:\n                x, y = curve.getData()\n                np.testing.assert_almost_equal(x, self.curves.probs)\n                for i, curve_data in enumerate(curves_data):\n                    if np.max(curve_data - y) < 1e-6:\n                        del curves_data[i]\n                        break\n                else:\n                    self.fail(f\"invalid curve for {combo.currentText()}\")\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_multiple_fold_curves(self, *_):\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(widget.controls.selected_classifiers, [0])\n        self._set_combo(widget.controls.score, 1)  # CA\n\n        self.results.folds = [slice(1, 5), slice(5, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 2)\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 3)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_change_target_class(self, *_):\n        \"\"\"Changing target combo changes the curves\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        score_combo = widget.controls.score\n        target_combo = widget.controls.target_index\n\n        self._set_combo(score_combo, 1)  # ca\n        self._set_combo(target_combo, 1)\n        (ca, ), _ = self._get_curves()\n        np.testing.assert_almost_equal(ca.getData()[1], self.curves.ca())\n\n        self._set_combo(target_combo, 0)\n        (ca, ), _ = self._get_curves()\n        curves = Curves(1 - self.curves.ytrue, 1 - self.curves.probs[:-1])\n        np.testing.assert_almost_equal(ca.getData()[1], curves.ca())\n\n    def test_changing_score_explanation(self):\n        \"\"\"Changing score hides/shows explanation and options for calibration\"\"\"\n        widget = self.widget\n        score_combo = widget.controls.score\n        explanation = widget.explanation\n        calibrations = widget.controls.output_calibration\n\n        self._set_combo(score_combo, 1)  # ca\n        self.assertTrue(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n        self._set_combo(score_combo, 0)  # calibration\n        self.assertTrue(explanation.isHidden())\n        self.assertFalse(calibrations.isHidden())\n\n        self._set_combo(score_combo, 3)  # sens/spec\n        self.assertFalse(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n    def test_rug(self):\n        \"\"\"Test rug appearance and positions\"\"\"\n        def get_rugs():\n            rugs = [None, None]\n            for item in widget.plot.items:\n                if item.curve.opts.get(\"connect\", \"\") == \"pairs\":\n                    x, y = item.getData()\n                    np.testing.assert_almost_equal(x[::2], x[1::2])\n                    rugs[int(y[0] == 1)] = x[::2]\n            return rugs\n\n        widget = self.widget\n        widget.display_rug = True\n        model_list = widget.controls.selected_classifiers\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n\n        self._set_list_selection(model_list, [0])\n        probs = self.curves.probs[:-1]\n        truex = probs[self.curves.ytrue == 1]\n        falsex = probs[self.curves.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex)\n        np.testing.assert_almost_equal(top, truex)\n\n        # Switching targets should switch rugs and takes other probabilities\n        self._set_combo(widget.controls.target_index, 0)\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, (1 - truex)[::-1])\n        np.testing.assert_almost_equal(top, (1 - falsex)[::-1])\n        self._set_combo(widget.controls.target_index, 1)\n\n        # Changing models gives a different rug\n        self._set_list_selection(model_list, [1])\n        probs2 = self.curves2.probs[:-1]\n        truex2 = probs2[self.curves2.ytrue == 1]\n        falsex2 = probs2[self.curves2.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex2)\n        np.testing.assert_almost_equal(top, truex2)\n\n        # Two models - two rugs - four rug items\n        self._set_list_selection(model_list, [0, 1])\n        self.assertEqual(sum(item.curve.opts.get(\"connect\", \"\") == \"pairs\"\n                             for item in widget.plot.items), 4)\n\n        # No models - no rugs\n        self._set_list_selection(model_list, [])\n        self.assertEqual(get_rugs(), [None, None])\n\n        # Bring the rug back\n        self._set_list_selection(model_list, [1])\n        self.assertIsNotNone(get_rugs()[0])\n\n        # Disable it with checkbox\n        widget.controls.display_rug.click()\n        self.assertEqual(get_rugs(), [None, None])\n\n    def test_calibration_curve(self):\n        \"\"\"Test the correct number of calibration curves\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        widget.display_rug = False\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertEqual(len(widget.plot.items), 3)  # 2 + diagonal\n\n        self._set_list_selection(model_list, [1])\n        self.assertEqual(len(widget.plot.items), 2)\n\n        self._set_list_selection(model_list, [])\n        self.assertEqual(len(widget.plot.items), 1)\n\n    def test_threshold_change_updates_info(self):\n        \"\"\"Changing the threshold updates info label\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n\n        original_text = widget.info_label.text()\n        self._set_threshold(0.3, False)\n        self.assertNotEqual(widget.info_label.text(), original_text)\n\n    def test_threshold_rounding(self):\n        \"\"\"Threshold is rounded to two decimals\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n        self._set_threshold(0.367, False)\n        self.assertAlmostEqual(widget.threshold, 0.37)\n\n    def test_threshold_flips_on_two_classes(self):\n        \"\"\"Threshold changes to 1 - threshold if *binary* class is switched\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.75)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.25)\n\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_apply_no_output(self, *_):\n        \"\"\"Test no output warnings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n\n        multiple_folds, multiple_selected, no_models, non_binary_class = \"abcd\"\n        messages = {\n            multiple_folds:\n                \"each training data sample produces a different model\",\n            no_models:\n                \"test results do not contain stored models - try testing on \"\n                \"separate data or on training data\",\n            multiple_selected:\n                \"select a single model - the widget can output only one\",\n            non_binary_class:\n                \"cannot calibrate non-binary classes\"}\n\n        def test_shown(shown):\n            widget_msg = widget.Information.no_output\n            output = self.get_output(widget.Outputs.calibrated_model)\n            if not shown:\n                self.assertFalse(widget_msg.is_shown())\n                self.assertIsNotNone(output)\n            else:\n                self.assertTrue(widget_msg.is_shown())\n                self.assertIsNone(output)\n                for msg_id in shown:\n                    msg = messages[msg_id]\n                    self.assertIn(msg, widget_msg.formatted,\n                                  f\"{msg} not included in the message\")\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        test_shown({multiple_selected})\n\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n        self._set_list_selection(model_list, [0, 1])\n\n        self.results.models = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, no_models})\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        test_shown({multiple_selected, non_binary_class})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({non_binary_class})\n\n        self.results.folds = [slice(0, 5), slice(5, 10), slice(10, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 3)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, multiple_folds})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({multiple_folds})\n\n        self._set_combo(widget.controls.score, 0)  # calibration\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 1])\n        test_shown({multiple_selected})\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    def test_output_threshold_classifier(self, threshold_classifier):\n        \"\"\"Test threshold classifier on output\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.results.models.ravel()\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        widget.target_index = 1\n\n        widget.threshold = 0.3\n        self._set_combo(widget.controls.score, 1)  # CA\n        model = self.get_output(widget.Outputs.calibrated_model)\n        threshold_classifier.assert_called_with(models[0], 0.3)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, False)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = False\n        self._set_threshold(0.35, True)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, True)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 0)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 1)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_list_selection(model_list, [1])\n        threshold_classifier.assert_called_with(models[1], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_output_calibrated_classifier(self, calibrated_learner):\n        \"\"\"Test calibrated classifier on output\"\"\"\n        calibrated_instance = calibrated_learner.return_value\n        get_model = calibrated_instance.get_model\n\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.lenses_results.models.ravel()\n        results = self.lenses_results\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 0)\n\n        self._set_list_selection(model_list, [1])\n\n        self._set_radio_buttons(widget.controls.output_calibration, 0)\n        calibrated_learner.assert_called_with(None, 0)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_list_selection(model_list, [0])\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[0])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[0])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n    def test_contexts(self):\n        \"\"\"Test storing and retrieving context settings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 2])\n        self._set_combo(target_combo, 2)\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        self._set_combo(target_combo, 0)\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self.assertEqual(widget.selected_classifiers, [0, 2])\n        self.assertEqual(widget.target_index, 2)\n\n    def test_report(self):\n        \"\"\"Test that report does not crash\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        widget.send_report()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class(self, *_):\n        \"\"\"Curves are not plotted if all data belongs to (non)-target\"\"\"\n        def check_error(shown):\n            for error in (errors.no_target_class, errors.all_target_class,\n                          errors.nan_classes):\n                self.assertEqual(error.is_shown(), error is shown,\n                                 f\"{error} is unexpectedly\"\n                                 f\"{'' if error.is_shown() else ' not'} shown\")\n            if shown is not None:\n                self.assertEqual(len(widget.plot.items), 0)\n            else:\n                self.assertGreater(len(widget.plot.items), 0)\n\n        widget = self.widget\n        errors = widget.Error\n        widget.display_rug = True\n        combo = widget.controls.score\n\n        original_actual = self.results.actual.copy()\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        for idx in range(combo.count()):\n            self._set_combo(combo, idx)\n            self.results.actual[:] = 0\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.no_target_class)\n\n            self.results.actual[:] = 1\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.all_target_class)\n\n            self.results.actual[:] = original_actual\n            self.results.actual[3] = np.nan\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.nan_classes)\n\n            self.results.actual[:] = original_actual\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(None)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class_folds(self, *_):\n        \"\"\"Curves for single-class folds are not plotted\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        results = self.lenses_results\n        results.folds = [slice(0, 5), slice(5, 19)]\n        results.models = results.models.repeat(2, axis=0)\n        results.actual = results.actual.copy()\n        results.actual[:3] = 0\n        results.probabilities[1, 3:5] = np.nan\n        # after this, model 1 has just negative instances in fold 0\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n        widget.controls.fold_curves.click()\n        self.assertTrue(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_warn_nan_probabilities(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        self.results.probabilities[1, 3] = np.nan\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())\n        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_no_folds(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n\n        self.results.folds = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        widget.commit.now()\n        self.assertIsNotNone(self.get_output(widget.Outputs.calibrated_model))",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 24,
                            "end_lineno": 652,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owcalibrationplot.py",
                            "type": "statement",
                            "name": "",
                            "content": "if __name__ == \"__main__\":\n    unittest.main()",
                            "ast_type_str": "If",
                            "start_lineno": 655,
                            "end_lineno": 656,
                            "parent": null
                        },
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owcalibrationplot.py",
                            "type": "method",
                            "name": "TestOWCalibrationPlot.test_no_folds",
                            "content": "class TestOWCalibrationPlot(WidgetTest, EvaluateTest):\n    def setUp(self):\n        super().setUp()\n\n        n, p = (0, 1)\n        actual, probs = np.array([\n            (p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52),\n            (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37),\n            (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T\n        self.curves = Curves(actual, probs)\n        probs2 = (probs + 0.5) / 2 + 1\n        self.curves2 = Curves(actual, probs2)\n        pred = probs > 0.5\n        pred2 = probs2 > 0.5\n        probs = np.vstack((1 - probs, probs)).T\n        probs2 = np.vstack((1 - probs2, probs2)).T\n        domain = Domain([], DiscreteVariable(\"y\", values=(\"a\", \"b\")))\n        self.results = Results(\n            domain=domain,\n            actual=actual,\n            folds=np.array([Ellipsis]),\n            models=np.array([[Mock(), Mock()]]),\n            row_indices=np.arange(19),\n            predicted=np.array((pred, pred2)),\n            probabilities=np.array([probs, probs2]))\n\n        self.lenses = data = Table(test_filename(\"datasets/lenses.tab\"))\n        majority = Orange.classification.MajorityLearner()\n        majority.name = \"majority\"\n        knn3 = Orange.classification.KNNLearner(n_neighbors=3)\n        knn3.name = \"knn-3\"\n        knn1 = Orange.classification.KNNLearner(n_neighbors=1)\n        knn1.name = \"knn-1\"\n        self.lenses_results = Orange.evaluation.TestOnTestData(\n            store_data=True, store_models=True)(\n                data=data[::2], test_data=data[1::2],\n                learners=[majority, knn3, knn1])\n        self.lenses_results.learner_names = [\"majority\", \"knn-3\", \"knn-1\"]\n\n        self.widget = self.create_widget(OWCalibrationPlot)  # type: OWCalibrationPlot\n        warnings.filterwarnings(\"ignore\", \".*\", ConvergenceWarning)\n\n    def test_initialization(self):\n        \"\"\"Test initialization of lists and combos\"\"\"\n        def check_clsfr_names(names):\n            self.assertEqual(widget.classifier_names, names)\n            clsf_list = widget.controls.selected_classifiers\n            self.assertEqual(\n                [clsf_list.item(i).text() for i in range(clsf_list.count())],\n                names)\n\n        widget = self.widget\n        tcomb = widget.controls.target_index\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        check_clsfr_names([\"majority\", \"knn-3\", \"knn-1\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1, 2])\n        self.assertEqual(\n            tuple(tcomb.itemText(i) for i in range(tcomb.count())),\n            self.lenses.domain.class_var.values)\n        self.assertEqual(widget.target_index, 0)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        check_clsfr_names([\"#1\", \"#2\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1])\n        self.assertEqual(\n            [tcomb.itemText(i) for i in range(tcomb.count())], [\"a\", \"b\"])\n        self.assertEqual(widget.target_index, 1)\n\n        self.send_signal(widget.Inputs.evaluation_results, None)\n        check_clsfr_names([])\n        self.assertEqual(widget.selected_classifiers, [])\n        self.assertEqual(widget.controls.target_index.count(), 0)\n\n    def test_empty_input_error(self):\n        \"\"\"Show an error when data is present but empty\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.empty_input.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    def test_regression_input_error(self):\n        \"\"\"Show an error for regression data\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.domain = Domain([], ContinuousVariable(\"y\"))\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.non_discrete_target.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    @staticmethod\n    def _set_combo(combo, val):\n        combo.setCurrentIndex(val)\n        combo.activated[int].emit(val)\n        combo.activated[str].emit(combo.currentText())\n\n    @staticmethod\n    def _set_radio_buttons(radios, val):\n        radios.buttons[val].click()\n\n    @staticmethod\n    def _set_list_selection(listview, selection):\n        model = listview.model()\n        selectionmodel = listview.selectionModel()\n        itemselection = QItemSelection()\n        for item in selection:\n            itemselection.select(model.index(item, 0), model.index(item, 0))\n        selectionmodel.select(itemselection, selectionmodel.ClearAndSelect)\n\n    def _set_threshold(self, pos, done):\n        _, line = self._get_curves()\n        line.setPos(pos)\n        if done:\n            line.sigPositionChangeFinished.emit(line)\n        else:\n            line.sigPositionChanged.emit(line)\n\n    def _get_curves(self):\n        plot_items = self.widget.plot.items[:]\n        for i, item in enumerate(plot_items):\n            if isinstance(item, InfiniteLine):\n                del plot_items[i]\n                return plot_items, item\n        return plot_items, None\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_plotting_curves(self, *_):\n        \"\"\"Curve coordinates match those computed by `Curves`\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        combo = widget.controls.score\n\n        c = self.curves\n        combinations = ([c.ca()],\n                        [c.f1()],\n                        [c.sensitivity(), c.specificity()],\n                        [c.precision(), c.recall()],\n                        [c.ppv(), c.npv()],\n                        [c.tpr(), c.fpr()])\n        for idx, curves_data in enumerate(combinations, start=1):\n            self._set_combo(combo, idx)\n            curves, line = self._get_curves()\n            self.assertEqual(len(curves), len(curves_data))\n            self.assertIsNotNone(line)\n            for curve in curves:\n                x, y = curve.getData()\n                np.testing.assert_almost_equal(x, self.curves.probs)\n                for i, curve_data in enumerate(curves_data):\n                    if np.max(curve_data - y) < 1e-6:\n                        del curves_data[i]\n                        break\n                else:\n                    self.fail(f\"invalid curve for {combo.currentText()}\")\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_multiple_fold_curves(self, *_):\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(widget.controls.selected_classifiers, [0])\n        self._set_combo(widget.controls.score, 1)  # CA\n\n        self.results.folds = [slice(1, 5), slice(5, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 2)\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 3)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_change_target_class(self, *_):\n        \"\"\"Changing target combo changes the curves\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        score_combo = widget.controls.score\n        target_combo = widget.controls.target_index\n\n        self._set_combo(score_combo, 1)  # ca\n        self._set_combo(target_combo, 1)\n        (ca, ), _ = self._get_curves()\n        np.testing.assert_almost_equal(ca.getData()[1], self.curves.ca())\n\n        self._set_combo(target_combo, 0)\n        (ca, ), _ = self._get_curves()\n        curves = Curves(1 - self.curves.ytrue, 1 - self.curves.probs[:-1])\n        np.testing.assert_almost_equal(ca.getData()[1], curves.ca())\n\n    def test_changing_score_explanation(self):\n        \"\"\"Changing score hides/shows explanation and options for calibration\"\"\"\n        widget = self.widget\n        score_combo = widget.controls.score\n        explanation = widget.explanation\n        calibrations = widget.controls.output_calibration\n\n        self._set_combo(score_combo, 1)  # ca\n        self.assertTrue(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n        self._set_combo(score_combo, 0)  # calibration\n        self.assertTrue(explanation.isHidden())\n        self.assertFalse(calibrations.isHidden())\n\n        self._set_combo(score_combo, 3)  # sens/spec\n        self.assertFalse(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n    def test_rug(self):\n        \"\"\"Test rug appearance and positions\"\"\"\n        def get_rugs():\n            rugs = [None, None]\n            for item in widget.plot.items:\n                if item.curve.opts.get(\"connect\", \"\") == \"pairs\":\n                    x, y = item.getData()\n                    np.testing.assert_almost_equal(x[::2], x[1::2])\n                    rugs[int(y[0] == 1)] = x[::2]\n            return rugs\n\n        widget = self.widget\n        widget.display_rug = True\n        model_list = widget.controls.selected_classifiers\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n\n        self._set_list_selection(model_list, [0])\n        probs = self.curves.probs[:-1]\n        truex = probs[self.curves.ytrue == 1]\n        falsex = probs[self.curves.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex)\n        np.testing.assert_almost_equal(top, truex)\n\n        # Switching targets should switch rugs and takes other probabilities\n        self._set_combo(widget.controls.target_index, 0)\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, (1 - truex)[::-1])\n        np.testing.assert_almost_equal(top, (1 - falsex)[::-1])\n        self._set_combo(widget.controls.target_index, 1)\n\n        # Changing models gives a different rug\n        self._set_list_selection(model_list, [1])\n        probs2 = self.curves2.probs[:-1]\n        truex2 = probs2[self.curves2.ytrue == 1]\n        falsex2 = probs2[self.curves2.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex2)\n        np.testing.assert_almost_equal(top, truex2)\n\n        # Two models - two rugs - four rug items\n        self._set_list_selection(model_list, [0, 1])\n        self.assertEqual(sum(item.curve.opts.get(\"connect\", \"\") == \"pairs\"\n                             for item in widget.plot.items), 4)\n\n        # No models - no rugs\n        self._set_list_selection(model_list, [])\n        self.assertEqual(get_rugs(), [None, None])\n\n        # Bring the rug back\n        self._set_list_selection(model_list, [1])\n        self.assertIsNotNone(get_rugs()[0])\n\n        # Disable it with checkbox\n        widget.controls.display_rug.click()\n        self.assertEqual(get_rugs(), [None, None])\n\n    def test_calibration_curve(self):\n        \"\"\"Test the correct number of calibration curves\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        widget.display_rug = False\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertEqual(len(widget.plot.items), 3)  # 2 + diagonal\n\n        self._set_list_selection(model_list, [1])\n        self.assertEqual(len(widget.plot.items), 2)\n\n        self._set_list_selection(model_list, [])\n        self.assertEqual(len(widget.plot.items), 1)\n\n    def test_threshold_change_updates_info(self):\n        \"\"\"Changing the threshold updates info label\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n\n        original_text = widget.info_label.text()\n        self._set_threshold(0.3, False)\n        self.assertNotEqual(widget.info_label.text(), original_text)\n\n    def test_threshold_rounding(self):\n        \"\"\"Threshold is rounded to two decimals\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n        self._set_threshold(0.367, False)\n        self.assertAlmostEqual(widget.threshold, 0.37)\n\n    def test_threshold_flips_on_two_classes(self):\n        \"\"\"Threshold changes to 1 - threshold if *binary* class is switched\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.75)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.25)\n\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_apply_no_output(self, *_):\n        \"\"\"Test no output warnings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n\n        multiple_folds, multiple_selected, no_models, non_binary_class = \"abcd\"\n        messages = {\n            multiple_folds:\n                \"each training data sample produces a different model\",\n            no_models:\n                \"test results do not contain stored models - try testing on \"\n                \"separate data or on training data\",\n            multiple_selected:\n                \"select a single model - the widget can output only one\",\n            non_binary_class:\n                \"cannot calibrate non-binary classes\"}\n\n        def test_shown(shown):\n            widget_msg = widget.Information.no_output\n            output = self.get_output(widget.Outputs.calibrated_model)\n            if not shown:\n                self.assertFalse(widget_msg.is_shown())\n                self.assertIsNotNone(output)\n            else:\n                self.assertTrue(widget_msg.is_shown())\n                self.assertIsNone(output)\n                for msg_id in shown:\n                    msg = messages[msg_id]\n                    self.assertIn(msg, widget_msg.formatted,\n                                  f\"{msg} not included in the message\")\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        test_shown({multiple_selected})\n\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n        self._set_list_selection(model_list, [0, 1])\n\n        self.results.models = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, no_models})\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        test_shown({multiple_selected, non_binary_class})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({non_binary_class})\n\n        self.results.folds = [slice(0, 5), slice(5, 10), slice(10, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 3)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, multiple_folds})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({multiple_folds})\n\n        self._set_combo(widget.controls.score, 0)  # calibration\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 1])\n        test_shown({multiple_selected})\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    def test_output_threshold_classifier(self, threshold_classifier):\n        \"\"\"Test threshold classifier on output\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.results.models.ravel()\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        widget.target_index = 1\n\n        widget.threshold = 0.3\n        self._set_combo(widget.controls.score, 1)  # CA\n        model = self.get_output(widget.Outputs.calibrated_model)\n        threshold_classifier.assert_called_with(models[0], 0.3)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, False)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = False\n        self._set_threshold(0.35, True)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, True)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 0)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 1)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_list_selection(model_list, [1])\n        threshold_classifier.assert_called_with(models[1], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_output_calibrated_classifier(self, calibrated_learner):\n        \"\"\"Test calibrated classifier on output\"\"\"\n        calibrated_instance = calibrated_learner.return_value\n        get_model = calibrated_instance.get_model\n\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.lenses_results.models.ravel()\n        results = self.lenses_results\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 0)\n\n        self._set_list_selection(model_list, [1])\n\n        self._set_radio_buttons(widget.controls.output_calibration, 0)\n        calibrated_learner.assert_called_with(None, 0)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_list_selection(model_list, [0])\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[0])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[0])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n    def test_contexts(self):\n        \"\"\"Test storing and retrieving context settings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 2])\n        self._set_combo(target_combo, 2)\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        self._set_combo(target_combo, 0)\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self.assertEqual(widget.selected_classifiers, [0, 2])\n        self.assertEqual(widget.target_index, 2)\n\n    def test_report(self):\n        \"\"\"Test that report does not crash\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        widget.send_report()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class(self, *_):\n        \"\"\"Curves are not plotted if all data belongs to (non)-target\"\"\"\n        def check_error(shown):\n            for error in (errors.no_target_class, errors.all_target_class,\n                          errors.nan_classes):\n                self.assertEqual(error.is_shown(), error is shown,\n                                 f\"{error} is unexpectedly\"\n                                 f\"{'' if error.is_shown() else ' not'} shown\")\n            if shown is not None:\n                self.assertEqual(len(widget.plot.items), 0)\n            else:\n                self.assertGreater(len(widget.plot.items), 0)\n\n        widget = self.widget\n        errors = widget.Error\n        widget.display_rug = True\n        combo = widget.controls.score\n\n        original_actual = self.results.actual.copy()\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        for idx in range(combo.count()):\n            self._set_combo(combo, idx)\n            self.results.actual[:] = 0\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.no_target_class)\n\n            self.results.actual[:] = 1\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.all_target_class)\n\n            self.results.actual[:] = original_actual\n            self.results.actual[3] = np.nan\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.nan_classes)\n\n            self.results.actual[:] = original_actual\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(None)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class_folds(self, *_):\n        \"\"\"Curves for single-class folds are not plotted\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        results = self.lenses_results\n        results.folds = [slice(0, 5), slice(5, 19)]\n        results.models = results.models.repeat(2, axis=0)\n        results.actual = results.actual.copy()\n        results.actual[:3] = 0\n        results.probabilities[1, 3:5] = np.nan\n        # after this, model 1 has just negative instances in fold 0\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n        widget.controls.fold_curves.click()\n        self.assertTrue(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_warn_nan_probabilities(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        self.results.probabilities[1, 3] = np.nan\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())\n        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_no_folds(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n\n        self.results.folds = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        widget.commit.now()\n        self.assertIsNotNone(self.get_output(widget.Outputs.calibrated_model))",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 644,
                            "end_lineno": 652,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/tests/test_owcalibrationplot.py",
                                "type": "class",
                                "name": "TestOWCalibrationPlot",
                                "content": "class TestOWCalibrationPlot(WidgetTest, EvaluateTest):\n    def setUp(self):\n        super().setUp()\n\n        n, p = (0, 1)\n        actual, probs = np.array([\n            (p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52),\n            (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37),\n            (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T\n        self.curves = Curves(actual, probs)\n        probs2 = (probs + 0.5) / 2 + 1\n        self.curves2 = Curves(actual, probs2)\n        pred = probs > 0.5\n        pred2 = probs2 > 0.5\n        probs = np.vstack((1 - probs, probs)).T\n        probs2 = np.vstack((1 - probs2, probs2)).T\n        domain = Domain([], DiscreteVariable(\"y\", values=(\"a\", \"b\")))\n        self.results = Results(\n            domain=domain,\n            actual=actual,\n            folds=np.array([Ellipsis]),\n            models=np.array([[Mock(), Mock()]]),\n            row_indices=np.arange(19),\n            predicted=np.array((pred, pred2)),\n            probabilities=np.array([probs, probs2]))\n\n        self.lenses = data = Table(test_filename(\"datasets/lenses.tab\"))\n        majority = Orange.classification.MajorityLearner()\n        majority.name = \"majority\"\n        knn3 = Orange.classification.KNNLearner(n_neighbors=3)\n        knn3.name = \"knn-3\"\n        knn1 = Orange.classification.KNNLearner(n_neighbors=1)\n        knn1.name = \"knn-1\"\n        self.lenses_results = Orange.evaluation.TestOnTestData(\n            store_data=True, store_models=True)(\n                data=data[::2], test_data=data[1::2],\n                learners=[majority, knn3, knn1])\n        self.lenses_results.learner_names = [\"majority\", \"knn-3\", \"knn-1\"]\n\n        self.widget = self.create_widget(OWCalibrationPlot)  # type: OWCalibrationPlot\n        warnings.filterwarnings(\"ignore\", \".*\", ConvergenceWarning)\n\n    def test_initialization(self):\n        \"\"\"Test initialization of lists and combos\"\"\"\n        def check_clsfr_names(names):\n            self.assertEqual(widget.classifier_names, names)\n            clsf_list = widget.controls.selected_classifiers\n            self.assertEqual(\n                [clsf_list.item(i).text() for i in range(clsf_list.count())],\n                names)\n\n        widget = self.widget\n        tcomb = widget.controls.target_index\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        check_clsfr_names([\"majority\", \"knn-3\", \"knn-1\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1, 2])\n        self.assertEqual(\n            tuple(tcomb.itemText(i) for i in range(tcomb.count())),\n            self.lenses.domain.class_var.values)\n        self.assertEqual(widget.target_index, 0)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        check_clsfr_names([\"#1\", \"#2\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1])\n        self.assertEqual(\n            [tcomb.itemText(i) for i in range(tcomb.count())], [\"a\", \"b\"])\n        self.assertEqual(widget.target_index, 1)\n\n        self.send_signal(widget.Inputs.evaluation_results, None)\n        check_clsfr_names([])\n        self.assertEqual(widget.selected_classifiers, [])\n        self.assertEqual(widget.controls.target_index.count(), 0)\n\n    def test_empty_input_error(self):\n        \"\"\"Show an error when data is present but empty\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.empty_input.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    def test_regression_input_error(self):\n        \"\"\"Show an error for regression data\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.domain = Domain([], ContinuousVariable(\"y\"))\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.non_discrete_target.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    @staticmethod\n    def _set_combo(combo, val):\n        combo.setCurrentIndex(val)\n        combo.activated[int].emit(val)\n        combo.activated[str].emit(combo.currentText())\n\n    @staticmethod\n    def _set_radio_buttons(radios, val):\n        radios.buttons[val].click()\n\n    @staticmethod\n    def _set_list_selection(listview, selection):\n        model = listview.model()\n        selectionmodel = listview.selectionModel()\n        itemselection = QItemSelection()\n        for item in selection:\n            itemselection.select(model.index(item, 0), model.index(item, 0))\n        selectionmodel.select(itemselection, selectionmodel.ClearAndSelect)\n\n    def _set_threshold(self, pos, done):\n        _, line = self._get_curves()\n        line.setPos(pos)\n        if done:\n            line.sigPositionChangeFinished.emit(line)\n        else:\n            line.sigPositionChanged.emit(line)\n\n    def _get_curves(self):\n        plot_items = self.widget.plot.items[:]\n        for i, item in enumerate(plot_items):\n            if isinstance(item, InfiniteLine):\n                del plot_items[i]\n                return plot_items, item\n        return plot_items, None\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_plotting_curves(self, *_):\n        \"\"\"Curve coordinates match those computed by `Curves`\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        combo = widget.controls.score\n\n        c = self.curves\n        combinations = ([c.ca()],\n                        [c.f1()],\n                        [c.sensitivity(), c.specificity()],\n                        [c.precision(), c.recall()],\n                        [c.ppv(), c.npv()],\n                        [c.tpr(), c.fpr()])\n        for idx, curves_data in enumerate(combinations, start=1):\n            self._set_combo(combo, idx)\n            curves, line = self._get_curves()\n            self.assertEqual(len(curves), len(curves_data))\n            self.assertIsNotNone(line)\n            for curve in curves:\n                x, y = curve.getData()\n                np.testing.assert_almost_equal(x, self.curves.probs)\n                for i, curve_data in enumerate(curves_data):\n                    if np.max(curve_data - y) < 1e-6:\n                        del curves_data[i]\n                        break\n                else:\n                    self.fail(f\"invalid curve for {combo.currentText()}\")\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_multiple_fold_curves(self, *_):\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(widget.controls.selected_classifiers, [0])\n        self._set_combo(widget.controls.score, 1)  # CA\n\n        self.results.folds = [slice(1, 5), slice(5, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 2)\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 3)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_change_target_class(self, *_):\n        \"\"\"Changing target combo changes the curves\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        score_combo = widget.controls.score\n        target_combo = widget.controls.target_index\n\n        self._set_combo(score_combo, 1)  # ca\n        self._set_combo(target_combo, 1)\n        (ca, ), _ = self._get_curves()\n        np.testing.assert_almost_equal(ca.getData()[1], self.curves.ca())\n\n        self._set_combo(target_combo, 0)\n        (ca, ), _ = self._get_curves()\n        curves = Curves(1 - self.curves.ytrue, 1 - self.curves.probs[:-1])\n        np.testing.assert_almost_equal(ca.getData()[1], curves.ca())\n\n    def test_changing_score_explanation(self):\n        \"\"\"Changing score hides/shows explanation and options for calibration\"\"\"\n        widget = self.widget\n        score_combo = widget.controls.score\n        explanation = widget.explanation\n        calibrations = widget.controls.output_calibration\n\n        self._set_combo(score_combo, 1)  # ca\n        self.assertTrue(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n        self._set_combo(score_combo, 0)  # calibration\n        self.assertTrue(explanation.isHidden())\n        self.assertFalse(calibrations.isHidden())\n\n        self._set_combo(score_combo, 3)  # sens/spec\n        self.assertFalse(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n    def test_rug(self):\n        \"\"\"Test rug appearance and positions\"\"\"\n        def get_rugs():\n            rugs = [None, None]\n            for item in widget.plot.items:\n                if item.curve.opts.get(\"connect\", \"\") == \"pairs\":\n                    x, y = item.getData()\n                    np.testing.assert_almost_equal(x[::2], x[1::2])\n                    rugs[int(y[0] == 1)] = x[::2]\n            return rugs\n\n        widget = self.widget\n        widget.display_rug = True\n        model_list = widget.controls.selected_classifiers\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n\n        self._set_list_selection(model_list, [0])\n        probs = self.curves.probs[:-1]\n        truex = probs[self.curves.ytrue == 1]\n        falsex = probs[self.curves.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex)\n        np.testing.assert_almost_equal(top, truex)\n\n        # Switching targets should switch rugs and takes other probabilities\n        self._set_combo(widget.controls.target_index, 0)\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, (1 - truex)[::-1])\n        np.testing.assert_almost_equal(top, (1 - falsex)[::-1])\n        self._set_combo(widget.controls.target_index, 1)\n\n        # Changing models gives a different rug\n        self._set_list_selection(model_list, [1])\n        probs2 = self.curves2.probs[:-1]\n        truex2 = probs2[self.curves2.ytrue == 1]\n        falsex2 = probs2[self.curves2.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex2)\n        np.testing.assert_almost_equal(top, truex2)\n\n        # Two models - two rugs - four rug items\n        self._set_list_selection(model_list, [0, 1])\n        self.assertEqual(sum(item.curve.opts.get(\"connect\", \"\") == \"pairs\"\n                             for item in widget.plot.items), 4)\n\n        # No models - no rugs\n        self._set_list_selection(model_list, [])\n        self.assertEqual(get_rugs(), [None, None])\n\n        # Bring the rug back\n        self._set_list_selection(model_list, [1])\n        self.assertIsNotNone(get_rugs()[0])\n\n        # Disable it with checkbox\n        widget.controls.display_rug.click()\n        self.assertEqual(get_rugs(), [None, None])\n\n    def test_calibration_curve(self):\n        \"\"\"Test the correct number of calibration curves\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        widget.display_rug = False\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertEqual(len(widget.plot.items), 3)  # 2 + diagonal\n\n        self._set_list_selection(model_list, [1])\n        self.assertEqual(len(widget.plot.items), 2)\n\n        self._set_list_selection(model_list, [])\n        self.assertEqual(len(widget.plot.items), 1)\n\n    def test_threshold_change_updates_info(self):\n        \"\"\"Changing the threshold updates info label\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n\n        original_text = widget.info_label.text()\n        self._set_threshold(0.3, False)\n        self.assertNotEqual(widget.info_label.text(), original_text)\n\n    def test_threshold_rounding(self):\n        \"\"\"Threshold is rounded to two decimals\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n        self._set_threshold(0.367, False)\n        self.assertAlmostEqual(widget.threshold, 0.37)\n\n    def test_threshold_flips_on_two_classes(self):\n        \"\"\"Threshold changes to 1 - threshold if *binary* class is switched\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.75)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.25)\n\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_apply_no_output(self, *_):\n        \"\"\"Test no output warnings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n\n        multiple_folds, multiple_selected, no_models, non_binary_class = \"abcd\"\n        messages = {\n            multiple_folds:\n                \"each training data sample produces a different model\",\n            no_models:\n                \"test results do not contain stored models - try testing on \"\n                \"separate data or on training data\",\n            multiple_selected:\n                \"select a single model - the widget can output only one\",\n            non_binary_class:\n                \"cannot calibrate non-binary classes\"}\n\n        def test_shown(shown):\n            widget_msg = widget.Information.no_output\n            output = self.get_output(widget.Outputs.calibrated_model)\n            if not shown:\n                self.assertFalse(widget_msg.is_shown())\n                self.assertIsNotNone(output)\n            else:\n                self.assertTrue(widget_msg.is_shown())\n                self.assertIsNone(output)\n                for msg_id in shown:\n                    msg = messages[msg_id]\n                    self.assertIn(msg, widget_msg.formatted,\n                                  f\"{msg} not included in the message\")\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        test_shown({multiple_selected})\n\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n        self._set_list_selection(model_list, [0, 1])\n\n        self.results.models = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, no_models})\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        test_shown({multiple_selected, non_binary_class})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({non_binary_class})\n\n        self.results.folds = [slice(0, 5), slice(5, 10), slice(10, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 3)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, multiple_folds})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({multiple_folds})\n\n        self._set_combo(widget.controls.score, 0)  # calibration\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 1])\n        test_shown({multiple_selected})\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    def test_output_threshold_classifier(self, threshold_classifier):\n        \"\"\"Test threshold classifier on output\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.results.models.ravel()\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        widget.target_index = 1\n\n        widget.threshold = 0.3\n        self._set_combo(widget.controls.score, 1)  # CA\n        model = self.get_output(widget.Outputs.calibrated_model)\n        threshold_classifier.assert_called_with(models[0], 0.3)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, False)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = False\n        self._set_threshold(0.35, True)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, True)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 0)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 1)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_list_selection(model_list, [1])\n        threshold_classifier.assert_called_with(models[1], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_output_calibrated_classifier(self, calibrated_learner):\n        \"\"\"Test calibrated classifier on output\"\"\"\n        calibrated_instance = calibrated_learner.return_value\n        get_model = calibrated_instance.get_model\n\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.lenses_results.models.ravel()\n        results = self.lenses_results\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 0)\n\n        self._set_list_selection(model_list, [1])\n\n        self._set_radio_buttons(widget.controls.output_calibration, 0)\n        calibrated_learner.assert_called_with(None, 0)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_list_selection(model_list, [0])\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[0])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[0])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n    def test_contexts(self):\n        \"\"\"Test storing and retrieving context settings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 2])\n        self._set_combo(target_combo, 2)\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        self._set_combo(target_combo, 0)\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self.assertEqual(widget.selected_classifiers, [0, 2])\n        self.assertEqual(widget.target_index, 2)\n\n    def test_report(self):\n        \"\"\"Test that report does not crash\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        widget.send_report()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class(self, *_):\n        \"\"\"Curves are not plotted if all data belongs to (non)-target\"\"\"\n        def check_error(shown):\n            for error in (errors.no_target_class, errors.all_target_class,\n                          errors.nan_classes):\n                self.assertEqual(error.is_shown(), error is shown,\n                                 f\"{error} is unexpectedly\"\n                                 f\"{'' if error.is_shown() else ' not'} shown\")\n            if shown is not None:\n                self.assertEqual(len(widget.plot.items), 0)\n            else:\n                self.assertGreater(len(widget.plot.items), 0)\n\n        widget = self.widget\n        errors = widget.Error\n        widget.display_rug = True\n        combo = widget.controls.score\n\n        original_actual = self.results.actual.copy()\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        for idx in range(combo.count()):\n            self._set_combo(combo, idx)\n            self.results.actual[:] = 0\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.no_target_class)\n\n            self.results.actual[:] = 1\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.all_target_class)\n\n            self.results.actual[:] = original_actual\n            self.results.actual[3] = np.nan\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.nan_classes)\n\n            self.results.actual[:] = original_actual\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(None)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class_folds(self, *_):\n        \"\"\"Curves for single-class folds are not plotted\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        results = self.lenses_results\n        results.folds = [slice(0, 5), slice(5, 19)]\n        results.models = results.models.repeat(2, axis=0)\n        results.actual = results.actual.copy()\n        results.actual[:3] = 0\n        results.probabilities[1, 3:5] = np.nan\n        # after this, model 1 has just negative instances in fold 0\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n        widget.controls.fold_curves.click()\n        self.assertTrue(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_warn_nan_probabilities(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        self.results.probabilities[1, 3] = np.nan\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())\n        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_no_folds(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n\n        self.results.folds = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        widget.commit.now()\n        self.assertIsNotNone(self.get_output(widget.Outputs.calibrated_model))",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 24,
                                "end_lineno": 652,
                                "parent": null
                            }
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd^",
    "new_commit_hash": "0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd",
    "commit_message": "Calibration plot: Fix crash for Results without folds",
    "commit_date": "2022-02-25T11:29:17+01:00",
    "metadata": {}
}
{
    "repo_name": "orange3",
    "new_commit_hash": "0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd",
    "test_file_codes": [
        "import copy\nimport warnings\nimport unittest\nfrom unittest.mock import Mock, patch\n\nimport numpy as np\nfrom AnyQt.QtCore import QItemSelection\nfrom pyqtgraph import InfiniteLine\n\nfrom sklearn.exceptions import ConvergenceWarning\n\nfrom Orange.data import Table, DiscreteVariable, Domain, ContinuousVariable\nimport Orange.evaluation\nimport Orange.classification\nfrom Orange.evaluation import Results\nfrom Orange.evaluation.performance_curves import Curves\n\nfrom Orange.widgets.evaluate.tests.base import EvaluateTest\nfrom Orange.widgets.evaluate.owcalibrationplot import OWCalibrationPlot\nfrom Orange.widgets.tests.base import WidgetTest\nfrom Orange.tests import test_filename\n\n\nclass TestOWCalibrationPlot(WidgetTest, EvaluateTest):\n    def setUp(self):\n        super().setUp()\n\n        n, p = (0, 1)\n        actual, probs = np.array([\n            (p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52),\n            (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37),\n            (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T\n        self.curves = Curves(actual, probs)\n        probs2 = (probs + 0.5) / 2 + 1\n        self.curves2 = Curves(actual, probs2)\n        pred = probs > 0.5\n        pred2 = probs2 > 0.5\n        probs = np.vstack((1 - probs, probs)).T\n        probs2 = np.vstack((1 - probs2, probs2)).T\n        domain = Domain([], DiscreteVariable(\"y\", values=(\"a\", \"b\")))\n        self.results = Results(\n            domain=domain,\n            actual=actual,\n            folds=np.array([Ellipsis]),\n            models=np.array([[Mock(), Mock()]]),\n            row_indices=np.arange(19),\n            predicted=np.array((pred, pred2)),\n            probabilities=np.array([probs, probs2]))\n\n        self.lenses = data = Table(test_filename(\"datasets/lenses.tab\"))\n        majority = Orange.classification.MajorityLearner()\n        majority.name = \"majority\"\n        knn3 = Orange.classification.KNNLearner(n_neighbors=3)\n        knn3.name = \"knn-3\"\n        knn1 = Orange.classification.KNNLearner(n_neighbors=1)\n        knn1.name = \"knn-1\"\n        self.lenses_results = Orange.evaluation.TestOnTestData(\n            store_data=True, store_models=True)(\n                data=data[::2], test_data=data[1::2],\n                learners=[majority, knn3, knn1])\n        self.lenses_results.learner_names = [\"majority\", \"knn-3\", \"knn-1\"]\n\n        self.widget = self.create_widget(OWCalibrationPlot)  # type: OWCalibrationPlot\n        warnings.filterwarnings(\"ignore\", \".*\", ConvergenceWarning)\n\n    def test_initialization(self):\n        \"\"\"Test initialization of lists and combos\"\"\"\n        def check_clsfr_names(names):\n            self.assertEqual(widget.classifier_names, names)\n            clsf_list = widget.controls.selected_classifiers\n            self.assertEqual(\n                [clsf_list.item(i).text() for i in range(clsf_list.count())],\n                names)\n\n        widget = self.widget\n        tcomb = widget.controls.target_index\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        check_clsfr_names([\"majority\", \"knn-3\", \"knn-1\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1, 2])\n        self.assertEqual(\n            tuple(tcomb.itemText(i) for i in range(tcomb.count())),\n            self.lenses.domain.class_var.values)\n        self.assertEqual(widget.target_index, 0)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        check_clsfr_names([\"#1\", \"#2\"])\n        self.assertEqual(widget.selected_classifiers, [0, 1])\n        self.assertEqual(\n            [tcomb.itemText(i) for i in range(tcomb.count())], [\"a\", \"b\"])\n        self.assertEqual(widget.target_index, 1)\n\n        self.send_signal(widget.Inputs.evaluation_results, None)\n        check_clsfr_names([])\n        self.assertEqual(widget.selected_classifiers, [])\n        self.assertEqual(widget.controls.target_index.count(), 0)\n\n    def test_empty_input_error(self):\n        \"\"\"Show an error when data is present but empty\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.empty_input.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.empty_input.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    def test_regression_input_error(self):\n        \"\"\"Show an error for regression data\"\"\"\n        widget = self.widget\n\n        res = copy.copy(self.results)\n        res.domain = Domain([], ContinuousVariable(\"y\"))\n        res.row_indices = res.row_indices[:0]\n        res.actual = res.actual[:0]\n        res.probabilities = res.probabilities[:, :0, :]\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, res)\n        self.assertTrue(widget.Error.non_discrete_target.is_shown())\n        self.assertIsNone(widget.results)\n        self.assertFalse(bool(widget.plot.items))\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertFalse(widget.Error.non_discrete_target.is_shown())\n        self.assertTrue(bool(widget.plot.items))\n\n    @staticmethod\n    def _set_combo(combo, val):\n        combo.setCurrentIndex(val)\n        combo.activated[int].emit(val)\n        combo.activated[str].emit(combo.currentText())\n\n    @staticmethod\n    def _set_radio_buttons(radios, val):\n        radios.buttons[val].click()\n\n    @staticmethod\n    def _set_list_selection(listview, selection):\n        model = listview.model()\n        selectionmodel = listview.selectionModel()\n        itemselection = QItemSelection()\n        for item in selection:\n            itemselection.select(model.index(item, 0), model.index(item, 0))\n        selectionmodel.select(itemselection, selectionmodel.ClearAndSelect)\n\n    def _set_threshold(self, pos, done):\n        _, line = self._get_curves()\n        line.setPos(pos)\n        if done:\n            line.sigPositionChangeFinished.emit(line)\n        else:\n            line.sigPositionChanged.emit(line)\n\n    def _get_curves(self):\n        plot_items = self.widget.plot.items[:]\n        for i, item in enumerate(plot_items):\n            if isinstance(item, InfiniteLine):\n                del plot_items[i]\n                return plot_items, item\n        return plot_items, None\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_plotting_curves(self, *_):\n        \"\"\"Curve coordinates match those computed by `Curves`\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        combo = widget.controls.score\n\n        c = self.curves\n        combinations = ([c.ca()],\n                        [c.f1()],\n                        [c.sensitivity(), c.specificity()],\n                        [c.precision(), c.recall()],\n                        [c.ppv(), c.npv()],\n                        [c.tpr(), c.fpr()])\n        for idx, curves_data in enumerate(combinations, start=1):\n            self._set_combo(combo, idx)\n            curves, line = self._get_curves()\n            self.assertEqual(len(curves), len(curves_data))\n            self.assertIsNotNone(line)\n            for curve in curves:\n                x, y = curve.getData()\n                np.testing.assert_almost_equal(x, self.curves.probs)\n                for i, curve_data in enumerate(curves_data):\n                    if np.max(curve_data - y) < 1e-6:\n                        del curves_data[i]\n                        break\n                else:\n                    self.fail(f\"invalid curve for {combo.currentText()}\")\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_multiple_fold_curves(self, *_):\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(widget.controls.selected_classifiers, [0])\n        self._set_combo(widget.controls.score, 1)  # CA\n\n        self.results.folds = [slice(1, 5), slice(5, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 2)\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 3)\n\n        widget.controls.fold_curves.click()\n        curves, _ = self._get_curves()\n        self.assertEqual(len(curves), 1)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_change_target_class(self, *_):\n        \"\"\"Changing target combo changes the curves\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        score_combo = widget.controls.score\n        target_combo = widget.controls.target_index\n\n        self._set_combo(score_combo, 1)  # ca\n        self._set_combo(target_combo, 1)\n        (ca, ), _ = self._get_curves()\n        np.testing.assert_almost_equal(ca.getData()[1], self.curves.ca())\n\n        self._set_combo(target_combo, 0)\n        (ca, ), _ = self._get_curves()\n        curves = Curves(1 - self.curves.ytrue, 1 - self.curves.probs[:-1])\n        np.testing.assert_almost_equal(ca.getData()[1], curves.ca())\n\n    def test_changing_score_explanation(self):\n        \"\"\"Changing score hides/shows explanation and options for calibration\"\"\"\n        widget = self.widget\n        score_combo = widget.controls.score\n        explanation = widget.explanation\n        calibrations = widget.controls.output_calibration\n\n        self._set_combo(score_combo, 1)  # ca\n        self.assertTrue(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n        self._set_combo(score_combo, 0)  # calibration\n        self.assertTrue(explanation.isHidden())\n        self.assertFalse(calibrations.isHidden())\n\n        self._set_combo(score_combo, 3)  # sens/spec\n        self.assertFalse(explanation.isHidden())\n        self.assertTrue(calibrations.isHidden())\n\n    def test_rug(self):\n        \"\"\"Test rug appearance and positions\"\"\"\n        def get_rugs():\n            rugs = [None, None]\n            for item in widget.plot.items:\n                if item.curve.opts.get(\"connect\", \"\") == \"pairs\":\n                    x, y = item.getData()\n                    np.testing.assert_almost_equal(x[::2], x[1::2])\n                    rugs[int(y[0] == 1)] = x[::2]\n            return rugs\n\n        widget = self.widget\n        widget.display_rug = True\n        model_list = widget.controls.selected_classifiers\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n\n        self._set_list_selection(model_list, [0])\n        probs = self.curves.probs[:-1]\n        truex = probs[self.curves.ytrue == 1]\n        falsex = probs[self.curves.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex)\n        np.testing.assert_almost_equal(top, truex)\n\n        # Switching targets should switch rugs and takes other probabilities\n        self._set_combo(widget.controls.target_index, 0)\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, (1 - truex)[::-1])\n        np.testing.assert_almost_equal(top, (1 - falsex)[::-1])\n        self._set_combo(widget.controls.target_index, 1)\n\n        # Changing models gives a different rug\n        self._set_list_selection(model_list, [1])\n        probs2 = self.curves2.probs[:-1]\n        truex2 = probs2[self.curves2.ytrue == 1]\n        falsex2 = probs2[self.curves2.ytrue == 0]\n        bottom, top = get_rugs()\n        np.testing.assert_almost_equal(bottom, falsex2)\n        np.testing.assert_almost_equal(top, truex2)\n\n        # Two models - two rugs - four rug items\n        self._set_list_selection(model_list, [0, 1])\n        self.assertEqual(sum(item.curve.opts.get(\"connect\", \"\") == \"pairs\"\n                             for item in widget.plot.items), 4)\n\n        # No models - no rugs\n        self._set_list_selection(model_list, [])\n        self.assertEqual(get_rugs(), [None, None])\n\n        # Bring the rug back\n        self._set_list_selection(model_list, [1])\n        self.assertIsNotNone(get_rugs()[0])\n\n        # Disable it with checkbox\n        widget.controls.display_rug.click()\n        self.assertEqual(get_rugs(), [None, None])\n\n    def test_calibration_curve(self):\n        \"\"\"Test the correct number of calibration curves\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        widget.display_rug = False\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertEqual(len(widget.plot.items), 3)  # 2 + diagonal\n\n        self._set_list_selection(model_list, [1])\n        self.assertEqual(len(widget.plot.items), 2)\n\n        self._set_list_selection(model_list, [])\n        self.assertEqual(len(widget.plot.items), 1)\n\n    def test_threshold_change_updates_info(self):\n        \"\"\"Changing the threshold updates info label\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n\n        original_text = widget.info_label.text()\n        self._set_threshold(0.3, False)\n        self.assertNotEqual(widget.info_label.text(), original_text)\n\n    def test_threshold_rounding(self):\n        \"\"\"Threshold is rounded to two decimals\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)\n        self._set_threshold(0.367, False)\n        self.assertAlmostEqual(widget.threshold, 0.37)\n\n    def test_threshold_flips_on_two_classes(self):\n        \"\"\"Threshold changes to 1 - threshold if *binary* class is switched\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.75)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_combo(widget.controls.target_index, 0)\n        self._set_combo(widget.controls.score, 1) # CA\n        self._set_threshold(0.25, False)\n        self.assertEqual(widget.threshold, 0.25)\n        self._set_combo(widget.controls.target_index, 1)\n        self.assertEqual(widget.threshold, 0.25)\n\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_apply_no_output(self, *_):\n        \"\"\"Test no output warnings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n\n        multiple_folds, multiple_selected, no_models, non_binary_class = \"abcd\"\n        messages = {\n            multiple_folds:\n                \"each training data sample produces a different model\",\n            no_models:\n                \"test results do not contain stored models - try testing on \"\n                \"separate data or on training data\",\n            multiple_selected:\n                \"select a single model - the widget can output only one\",\n            non_binary_class:\n                \"cannot calibrate non-binary classes\"}\n\n        def test_shown(shown):\n            widget_msg = widget.Information.no_output\n            output = self.get_output(widget.Outputs.calibrated_model)\n            if not shown:\n                self.assertFalse(widget_msg.is_shown())\n                self.assertIsNotNone(output)\n            else:\n                self.assertTrue(widget_msg.is_shown())\n                self.assertIsNone(output)\n                for msg_id in shown:\n                    msg = messages[msg_id]\n                    self.assertIn(msg, widget_msg.formatted,\n                                  f\"{msg} not included in the message\")\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        test_shown({multiple_selected})\n\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n        self._set_list_selection(model_list, [0, 1])\n\n        self.results.models = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, no_models})\n\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        test_shown({multiple_selected, non_binary_class})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({non_binary_class})\n\n        self.results.folds = [slice(0, 5), slice(5, 10), slice(10, 19)]\n        self.results.models = np.array([[Mock(), Mock()]] * 3)\n\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        test_shown({multiple_selected, multiple_folds})\n\n        self._set_list_selection(model_list, [0])\n        test_shown({multiple_folds})\n\n        self._set_combo(widget.controls.score, 0)  # calibration\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 1])\n        test_shown({multiple_selected})\n        self._set_list_selection(model_list, [0])\n        test_shown(())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    def test_output_threshold_classifier(self, threshold_classifier):\n        \"\"\"Test threshold classifier on output\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.results.models.ravel()\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        widget.target_index = 1\n\n        widget.threshold = 0.3\n        self._set_combo(widget.controls.score, 1)  # CA\n        model = self.get_output(widget.Outputs.calibrated_model)\n        threshold_classifier.assert_called_with(models[0], 0.3)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, False)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = False\n        self._set_threshold(0.35, True)\n        threshold_classifier.assert_not_called()\n\n        widget.auto_commit = True\n        self._set_threshold(0.4, True)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 0)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_combo(target_combo, 1)\n        threshold_classifier.assert_called_with(models[0], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n        self._set_list_selection(model_list, [1])\n        threshold_classifier.assert_called_with(models[1], 0.4)\n        self.assertIs(model, threshold_classifier.return_value)\n        threshold_classifier.reset_mock()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_output_calibrated_classifier(self, calibrated_learner):\n        \"\"\"Test calibrated classifier on output\"\"\"\n        calibrated_instance = calibrated_learner.return_value\n        get_model = calibrated_instance.get_model\n\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        models = self.lenses_results.models.ravel()\n        results = self.lenses_results\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 0)\n\n        self._set_list_selection(model_list, [1])\n\n        self._set_radio_buttons(widget.controls.output_calibration, 0)\n        calibrated_learner.assert_called_with(None, 0)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[1])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[1])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n        self._set_list_selection(model_list, [0])\n        self._set_radio_buttons(widget.controls.output_calibration, 1)\n        calibrated_learner.assert_called_with(None, 1)\n        model, actual, probabilities = get_model.call_args[0]\n        self.assertIs(model, models[0])\n        np.testing.assert_equal(actual, results.actual)\n        np.testing.assert_equal(probabilities, results.probabilities[0])\n        self.assertIs(self.get_output(widget.Outputs.calibrated_model),\n                      get_model.return_value)\n        calibrated_learner.reset_mock()\n        get_model.reset_mock()\n\n    def test_contexts(self):\n        \"\"\"Test storing and retrieving context settings\"\"\"\n        widget = self.widget\n        model_list = widget.controls.selected_classifiers\n        target_combo = widget.controls.target_index\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self._set_list_selection(model_list, [0, 2])\n        self._set_combo(target_combo, 2)\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self._set_list_selection(model_list, [0])\n        self._set_combo(target_combo, 0)\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        self.assertEqual(widget.selected_classifiers, [0, 2])\n        self.assertEqual(widget.target_index, 2)\n\n    def test_report(self):\n        \"\"\"Test that report does not crash\"\"\"\n        widget = self.widget\n        self.send_signal(widget.Inputs.evaluation_results, self.lenses_results)\n        widget.send_report()\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class(self, *_):\n        \"\"\"Curves are not plotted if all data belongs to (non)-target\"\"\"\n        def check_error(shown):\n            for error in (errors.no_target_class, errors.all_target_class,\n                          errors.nan_classes):\n                self.assertEqual(error.is_shown(), error is shown,\n                                 f\"{error} is unexpectedly\"\n                                 f\"{'' if error.is_shown() else ' not'} shown\")\n            if shown is not None:\n                self.assertEqual(len(widget.plot.items), 0)\n            else:\n                self.assertGreater(len(widget.plot.items), 0)\n\n        widget = self.widget\n        errors = widget.Error\n        widget.display_rug = True\n        combo = widget.controls.score\n\n        original_actual = self.results.actual.copy()\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        for idx in range(combo.count()):\n            self._set_combo(combo, idx)\n            self.results.actual[:] = 0\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.no_target_class)\n\n            self.results.actual[:] = 1\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.all_target_class)\n\n            self.results.actual[:] = original_actual\n            self.results.actual[3] = np.nan\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(errors.nan_classes)\n\n            self.results.actual[:] = original_actual\n            self.send_signal(widget.Inputs.evaluation_results, self.results)\n            check_error(None)\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_single_class_folds(self, *_):\n        \"\"\"Curves for single-class folds are not plotted\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        results = self.lenses_results\n        results.folds = [slice(0, 5), slice(5, 19)]\n        results.models = results.models.repeat(2, axis=0)\n        results.actual = results.actual.copy()\n        results.actual[:3] = 0\n        results.probabilities[1, 3:5] = np.nan\n        # after this, model 1 has just negative instances in fold 0\n        self.send_signal(widget.Inputs.evaluation_results, results)\n        self._set_combo(widget.controls.score, 1)  # CA\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n        widget.controls.fold_curves.click()\n        self.assertTrue(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_warn_nan_probabilities(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n        widget.display_rug = False\n        widget.fold_curves = False\n\n        self.results.probabilities[1, 3] = np.nan\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        self.assertTrue(widget.Warning.omitted_nan_prob_points.is_shown())\n        self._set_list_selection(widget.controls.selected_classifiers, [0, 2])\n        self.assertFalse(widget.Warning.omitted_folds.is_shown())\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_no_folds(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n\n        self.results.folds = None\n        self.send_signal(widget.Inputs.evaluation_results, self.results)\n        widget.selected_classifiers = [0]\n        widget.commit.now()\n        self.assertIsNotNone(self.get_output(widget.Outputs.calibrated_model))\n\n\nif __name__ == \"__main__\":\n    unittest.main()\n"
    ],
    "test_file_names": [
        "test_1.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Orange installation attempts...\nAttempting installation with Python 3.7...\nrunning build_ext\nCompiling Orange/classification/_tree_scorers.pyx because it changed.\nCompiling Orange/data/_contingency.pyx because it changed.\nCompiling Orange/data/_io.pyx because it changed.\nCompiling Orange/data/_valuecount.pyx because it changed.\nCompiling Orange/data/_variable.pyx because it changed.\nCompiling Orange/distance/_distance.pyx because it changed.\nCompiling Orange/preprocess/_discretize.pyx because it changed.\nCompiling Orange/preprocess/_relieff.pyx because it changed.\nCompiling Orange/projection/_som.pyx because it changed.\n[1/9] Cythonizing Orange/classification/_tree_scorers.pyx\n[2/9] Cythonizing Orange/data/_contingency.pyx\n[3/9] Cythonizing Orange/data/_io.pyx\n[4/9] Cythonizing Orange/data/_valuecount.pyx\n[5/9] Cythonizing Orange/data/_variable.pyx\n[6/9] Cythonizing Orange/distance/_distance.pyx\n[7/9] Cythonizing Orange/preprocess/_discretize.pyx\n[8/9] Cythonizing Orange/preprocess/_relieff.pyx\n[9/9] Cythonizing Orange/projection/_som.pyx\nbuilding 'Orange.classification._tree_scorers' extension\ncreating build\ncreating build/temp.linux-x86_64-3.7\ncreating build/temp.linux-x86_64-3.7/Orange\ncreating build/temp.linux-x86_64-3.7/Orange/classification\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/classification/_tree_scorers.c -o build/temp.linux-x86_64-3.7/Orange/classification/_tree_scorers.o\ncreating build/lib.linux-x86_64-3.7\ncreating build/lib.linux-x86_64-3.7/Orange\ncreating build/lib.linux-x86_64-3.7/Orange/classification\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/classification/_tree_scorers.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/classification/_tree_scorers.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._contingency' extension\ncreating build/temp.linux-x86_64-3.7/Orange/data\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/data/_contingency.c -o build/temp.linux-x86_64-3.7/Orange/data/_contingency.o\ncreating build/lib.linux-x86_64-3.7/Orange/data\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_contingency.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/data/_contingency.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._io' extension\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/data/_io.c -o build/temp.linux-x86_64-3.7/Orange/data/_io.o\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_io.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/data/_io.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._valuecount' extension\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/data/_valuecount.c -o build/temp.linux-x86_64-3.7/Orange/data/_valuecount.o\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_valuecount.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/data/_valuecount.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._variable' extension\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/data/_variable.c -o build/temp.linux-x86_64-3.7/Orange/data/_variable.o\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_variable.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/data/_variable.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.distance._distance' extension\ncreating build/temp.linux-x86_64-3.7/Orange/distance\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/distance/_distance.c -o build/temp.linux-x86_64-3.7/Orange/distance/_distance.o\ncreating build/lib.linux-x86_64-3.7/Orange/distance\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/distance/_distance.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/distance/_distance.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.preprocess._discretize' extension\ncreating build/temp.linux-x86_64-3.7/Orange/preprocess\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/preprocess/_discretize.c -o build/temp.linux-x86_64-3.7/Orange/preprocess/_discretize.o\ncreating build/lib.linux-x86_64-3.7/Orange/preprocess\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/preprocess/_discretize.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/preprocess/_discretize.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.preprocess._relieff' extension\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/preprocess/_relieff.cpp -o build/temp.linux-x86_64-3.7/Orange/preprocess/_relieff.o\nc++ -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/preprocess/_relieff.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/preprocess/_relieff.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.projection._som' extension\ncreating build/temp.linux-x86_64-3.7/Orange/projection\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/projection/_som.c -o build/temp.linux-x86_64-3.7/Orange/projection/_som.o\ncreating build/lib.linux-x86_64-3.7/Orange/projection\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/projection/_som.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/projection/_som.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.classification._simple_tree' extension\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/classification/_simple_tree.c -o build/temp.linux-x86_64-3.7/Orange/classification/_simple_tree.o\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/classification/_simple_tree.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/classification/_simple_tree.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.widgets.utils._grid_density' extension\ncreating build/temp.linux-x86_64-3.7/Orange/widgets\ncreating build/temp.linux-x86_64-3.7/Orange/widgets/utils\ncc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c Orange/widgets/utils/_grid_density.cpp -o build/temp.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.o\ncreating build/lib.linux-x86_64-3.7/Orange/widgets\ncreating build/lib.linux-x86_64-3.7/Orange/widgets/utils\nc++ -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o build/lib.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.cpython-37m-x86_64-linux-gnu.so\ncopying build/lib.linux-x86_64-3.7/Orange/classification/_tree_scorers.cpython-37m-x86_64-linux-gnu.so -> Orange/classification\ncopying build/lib.linux-x86_64-3.7/Orange/data/_contingency.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/data/_io.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/data/_valuecount.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/data/_variable.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/distance/_distance.cpython-37m-x86_64-linux-gnu.so -> Orange/distance\ncopying build/lib.linux-x86_64-3.7/Orange/preprocess/_discretize.cpython-37m-x86_64-linux-gnu.so -> Orange/preprocess\ncopying build/lib.linux-x86_64-3.7/Orange/preprocess/_relieff.cpython-37m-x86_64-linux-gnu.so -> Orange/preprocess\ncopying build/lib.linux-x86_64-3.7/Orange/projection/_som.cpython-37m-x86_64-linux-gnu.so -> Orange/projection\ncopying build/lib.linux-x86_64-3.7/Orange/classification/_simple_tree.cpython-37m-x86_64-linux-gnu.so -> Orange/classification\ncopying build/lib.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.cpython-37m-x86_64-linux-gnu.so -> Orange/widgets/utils\nrunning develop\nrunning egg_info\ncreating Orange3.egg-info\nwriting Orange3.egg-info/PKG-INFO\nwriting dependency_links to Orange3.egg-info/dependency_links.txt\nwriting entry points to Orange3.egg-info/entry_points.txt\nwriting requirements to Orange3.egg-info/requires.txt\nwriting top-level names to Orange3.egg-info/top_level.txt\nwriting manifest file 'Orange3.egg-info/SOURCES.txt'\nreading manifest file 'Orange3.egg-info/SOURCES.txt'\nreading manifest template 'MANIFEST.in'\nadding license file 'LICENSE'\nwriting manifest file 'Orange3.egg-info/SOURCES.txt'\nrunning build_ext\ncopying build/lib.linux-x86_64-3.7/Orange/classification/_tree_scorers.cpython-37m-x86_64-linux-gnu.so -> Orange/classification\ncopying build/lib.linux-x86_64-3.7/Orange/data/_contingency.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/data/_io.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/data/_valuecount.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/data/_variable.cpython-37m-x86_64-linux-gnu.so -> Orange/data\ncopying build/lib.linux-x86_64-3.7/Orange/distance/_distance.cpython-37m-x86_64-linux-gnu.so -> Orange/distance\ncopying build/lib.linux-x86_64-3.7/Orange/preprocess/_discretize.cpython-37m-x86_64-linux-gnu.so -> Orange/preprocess\ncopying build/lib.linux-x86_64-3.7/Orange/preprocess/_relieff.cpython-37m-x86_64-linux-gnu.so -> Orange/preprocess\ncopying build/lib.linux-x86_64-3.7/Orange/projection/_som.cpython-37m-x86_64-linux-gnu.so -> Orange/projection\ncopying build/lib.linux-x86_64-3.7/Orange/classification/_simple_tree.cpython-37m-x86_64-linux-gnu.so -> Orange/classification\ncopying build/lib.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.cpython-37m-x86_64-linux-gnu.so -> Orange/widgets/utils\nCreating /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/Orange3.egg-link (link to .)\nAdding Orange3 3.32.0.dev0+0cefc17 to easy-install.pth file\nInstalling orange-canvas script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nInstalled /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd\nProcessing dependencies for Orange3==3.32.0.dev0+0cefc17\nSearching for XlsxWriter==3.2.0\nBest match: XlsxWriter 3.2.0\nAdding XlsxWriter 3.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for xlrd==2.0.1\nBest match: xlrd 2.0.1\nAdding xlrd 2.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for setuptools==59.8.0\nBest match: setuptools 59.8.0\nAdding setuptools 59.8.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for serverfiles==0.3.1\nBest match: serverfiles 0.3.1\nAdding serverfiles 0.3.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for scipy==1.7.3\nBest match: scipy 1.7.3\nAdding scipy 1.7.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for scikit-learn==1.0.2\nBest match: scikit-learn 1.0.2\nAdding scikit-learn 1.0.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for requests==2.31.0\nBest match: requests 2.31.0\nAdding requests 2.31.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for qtconsole==5.4.4\nBest match: qtconsole 5.4.4\nAdding qtconsole 5.4.4 to easy-install.pth file\nInstalling jupyter-qtconsole script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for PyYAML==6.0.1\nBest match: PyYAML 6.0.1\nAdding PyYAML 6.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for python-louvain==0.16\nBest match: python-louvain 0.16\nAdding python-louvain 0.16 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pyqtgraph==0.12.4\nBest match: pyqtgraph 0.12.4\nAdding pyqtgraph 0.12.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pygments==2.17.2\nBest match: pygments 2.17.2\nAdding pygments 2.17.2 to easy-install.pth file\nInstalling pygmentize script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pip==24.0\nBest match: pip 24.0\nAdding pip 24.0 to easy-install.pth file\nInstalling pip script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling pip3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling pip3.10 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pandas==1.3.5\nBest match: pandas 1.3.5\nAdding pandas 1.3.5 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for orange-widget-base==4.22.0\nBest match: orange-widget-base 4.22.0\nAdding orange-widget-base 4.22.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for orange-canvas-core==0.1.35\nBest match: orange-canvas-core 0.1.35\nAdding orange-canvas-core 0.1.35 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for openpyxl==3.1.3\nBest match: openpyxl 3.1.3\nAdding openpyxl 3.1.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for openTSNE==1.0.1\nBest match: openTSNE 1.0.1\nAdding openTSNE 1.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for numpy==1.17.5\nBest match: numpy 1.17.5\nAdding numpy 1.17.5 to easy-install.pth file\nInstalling f2py script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling f2py3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling f2py3.7 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for networkx==2.6.3\nBest match: networkx 2.6.3\nAdding networkx 2.6.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for matplotlib==3.5.3\nBest match: matplotlib 3.5.3\nAdding matplotlib 3.5.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for keyrings.alt==4.2.0\nBest match: keyrings.alt 4.2.0\nAdding keyrings.alt 4.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for keyring==24.1.1\nBest match: keyring 24.1.1\nAdding keyring 24.1.1 to easy-install.pth file\nInstalling keyring script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for joblib==1.3.2\nBest match: joblib 1.3.2\nAdding joblib 1.3.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for httpx==0.19.0\nBest match: httpx 0.19.0\nAdding httpx 0.19.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for chardet==5.2.0\nBest match: chardet 5.2.0\nAdding chardet 5.2.0 to easy-install.pth file\nInstalling chardetect script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for Bottleneck==1.4.0\nBest match: Bottleneck 1.4.0\nAdding Bottleneck 1.4.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for baycomp==1.0.3\nBest match: baycomp 1.0.3\nAdding baycomp 1.0.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for PyQtWebEngine==5.15.6\nBest match: PyQtWebEngine 5.15.6\nAdding PyQtWebEngine 5.15.6 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for PyQt5==5.15.10\nBest match: PyQt5 5.15.10\nAdding PyQt5 5.15.10 to easy-install.pth file\nInstalling pylupdate5 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling pyrcc5 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling pyuic5 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for AnyQt==0.2.0\nBest match: AnyQt 0.2.0\nAdding AnyQt 0.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for threadpoolctl==3.1.0\nBest match: threadpoolctl 3.1.0\nAdding threadpoolctl 3.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for certifi==2024.12.14\nBest match: certifi 2024.12.14\nAdding certifi 2024.12.14 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for urllib3==2.0.7\nBest match: urllib3 2.0.7\nAdding urllib3 2.0.7 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for charset-normalizer==3.4.1\nBest match: charset-normalizer 3.4.1\nAdding charset-normalizer 3.4.1 to easy-install.pth file\nInstalling normalizer script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for idna==3.10\nBest match: idna 3.10\nAdding idna 3.10 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for ipykernel==6.16.2\nBest match: ipykernel 6.16.2\nAdding ipykernel 6.16.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pyzmq==26.2.0\nBest match: pyzmq 26.2.0\nAdding pyzmq 26.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for ipython-genutils==0.2.0\nBest match: ipython-genutils 0.2.0\nAdding ipython-genutils 0.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for jupyter-client==7.4.9\nBest match: jupyter-client 7.4.9\nAdding jupyter-client 7.4.9 to easy-install.pth file\nInstalling jupyter-kernel script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling jupyter-kernelspec script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling jupyter-run script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for QtPy==2.4.2\nBest match: QtPy 2.4.2\nAdding QtPy 2.4.2 to easy-install.pth file\nInstalling qtpy script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for jupyter-core==4.12.0\nBest match: jupyter-core 4.12.0\nAdding jupyter-core 4.12.0 to easy-install.pth file\nInstalling jupyter script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling jupyter-migrate script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling jupyter-troubleshoot script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for traitlets==5.9.0\nBest match: traitlets 5.9.0\nAdding traitlets 5.9.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for packaging==24.0\nBest match: packaging 24.0\nAdding packaging 24.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pytz==2024.2\nBest match: pytz 2024.2\nAdding pytz 2024.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for python-dateutil==2.9.0.post0\nBest match: python-dateutil 2.9.0.post0\nAdding python-dateutil 2.9.0.post0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for typing-extensions==4.7.1\nBest match: typing-extensions 4.7.1\nAdding typing-extensions 4.7.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for requests-cache==1.1.1\nBest match: requests-cache 1.1.1\nAdding requests-cache 1.1.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for importlib-metadata==6.7.0\nBest match: importlib-metadata 6.7.0\nAdding importlib-metadata 6.7.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for dictdiffer==0.9.0\nBest match: dictdiffer 0.9.0\nAdding dictdiffer 0.9.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for qasync==0.25.0\nBest match: qasync 0.25.0\nAdding qasync 0.25.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for docutils==0.20.1\nBest match: docutils 0.20.1\nAdding docutils 0.20.1 to easy-install.pth file\nInstalling docutils script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for commonmark==0.9.1\nBest match: commonmark 0.9.1\nAdding commonmark 0.9.1 to easy-install.pth file\nInstalling cmark script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for et-xmlfile==1.1.0\nBest match: et-xmlfile 1.1.0\nAdding et-xmlfile 1.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pyparsing==3.1.4\nBest match: pyparsing 3.1.4\nAdding pyparsing 3.1.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for kiwisolver==1.4.5\nBest match: kiwisolver 1.4.5\nAdding kiwisolver 1.4.5 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for fonttools==4.38.0\nBest match: fonttools 4.38.0\nAdding fonttools 4.38.0 to easy-install.pth file\nInstalling fonttools script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling pyftmerge script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling pyftsubset script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling ttx script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for cycler==0.11.0\nBest match: cycler 0.11.0\nAdding cycler 0.11.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for Pillow==9.5.0\nBest match: Pillow 9.5.0\nAdding Pillow 9.5.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for jaraco.classes==3.2.3\nBest match: jaraco.classes 3.2.3\nAdding jaraco.classes 3.2.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for importlib-resources==5.12.0\nBest match: importlib-resources 5.12.0\nAdding importlib-resources 5.12.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for SecretStorage==3.3.3\nBest match: SecretStorage 3.3.3\nAdding SecretStorage 3.3.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for jeepney==0.8.0\nBest match: jeepney 0.8.0\nAdding jeepney 0.8.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for sniffio==1.3.1\nBest match: sniffio 1.3.1\nAdding sniffio 1.3.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for rfc3986==1.5.0\nBest match: rfc3986 1.5.0\nAdding rfc3986 1.5.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for httpcore==0.13.7\nBest match: httpcore 0.13.7\nAdding httpcore 0.13.7 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for PyQt5-sip==12.13.0\nBest match: PyQt5-sip 12.13.0\nAdding PyQt5-sip 12.13.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for PyQtWebEngine-Qt5==5.15.16\nBest match: PyQtWebEngine-Qt5 5.15.16\nAdding PyQtWebEngine-Qt5 5.15.16 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for PyQt5-Qt5==5.15.16\nBest match: PyQt5-Qt5 5.15.16\nAdding PyQt5-Qt5 5.15.16 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for matplotlib-inline==0.1.6\nBest match: matplotlib-inline 0.1.6\nAdding matplotlib-inline 0.1.6 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for tornado==6.2\nBest match: tornado 6.2\nAdding tornado 6.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for debugpy==1.7.0\nBest match: debugpy 1.7.0\nAdding debugpy 1.7.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for nest-asyncio==1.6.0\nBest match: nest-asyncio 1.6.0\nAdding nest-asyncio 1.6.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for psutil==6.1.1\nBest match: psutil 6.1.1\nAdding psutil 6.1.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for ipython==7.34.0\nBest match: ipython 7.34.0\nAdding ipython 7.34.0 to easy-install.pth file\nInstalling iptest script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling iptest3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling ipython script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\nInstalling ipython3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for entrypoints==0.4\nBest match: entrypoints 0.4\nAdding entrypoints 0.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for six==1.17.0\nBest match: six 1.17.0\nAdding six 1.17.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for url-normalize==1.4.3\nBest match: url-normalize 1.4.3\nAdding url-normalize 1.4.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for platformdirs==4.0.0\nBest match: platformdirs 4.0.0\nAdding platformdirs 4.0.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for attrs==24.2.0\nBest match: attrs 24.2.0\nAdding attrs 24.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for cattrs==23.1.2\nBest match: cattrs 23.1.2\nAdding cattrs 23.1.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for zipp==3.15.0\nBest match: zipp 3.15.0\nAdding zipp 3.15.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for more-itertools==9.1.0\nBest match: more-itertools 9.1.0\nAdding more-itertools 9.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for cryptography==44.0.0\nBest match: cryptography 44.0.0\nAdding cryptography 44.0.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for anyio==3.7.1\nBest match: anyio 3.7.1\nAdding anyio 3.7.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for h11==0.12.0\nBest match: h11 0.12.0\nAdding h11 0.12.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pickleshare==0.7.5\nBest match: pickleshare 0.7.5\nAdding pickleshare 0.7.5 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for jedi==0.19.2\nBest match: jedi 0.19.2\nAdding jedi 0.19.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for decorator==5.1.1\nBest match: decorator 5.1.1\nAdding decorator 5.1.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pexpect==4.9.0\nBest match: pexpect 4.9.0\nAdding pexpect 4.9.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for prompt-toolkit==3.0.48\nBest match: prompt-toolkit 3.0.48\nAdding prompt-toolkit 3.0.48 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for backcall==0.2.0\nBest match: backcall 0.2.0\nAdding backcall 0.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for exceptiongroup==1.2.2\nBest match: exceptiongroup 1.2.2\nAdding exceptiongroup 1.2.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for cffi==1.15.1\nBest match: cffi 1.15.1\nAdding cffi 1.15.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for parso==0.8.4\nBest match: parso 0.8.4\nAdding parso 0.8.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for ptyprocess==0.7.0\nBest match: ptyprocess 0.7.0\nAdding ptyprocess 0.7.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for wcwidth==0.2.13\nBest match: wcwidth 0.2.13\nAdding wcwidth 0.2.13 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nSearching for pycparser==2.21\nBest match: pycparser 2.21\nAdding pycparser 2.21 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages\nFinished processing dependencies for Orange3==3.32.0.dev0+0cefc17\nVerifying Orange installation...\n✅ Orange installation successful!\nSuccessfully installed orange using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 18 packages in 136ms\nPrepared 18 packages in 2ms\nInstalled 18 packages in 123ms\n + cython==0.29.37\n + exceptiongroup==1.2.2\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + numpy==1.17.5\n + packaging==24.0\n + pluggy==1.2.0\n + pyqt5==5.15.10\n + pyqt5-qt5==5.15.16\n + pyqt5-sip==12.13.0\n + pyqtwebengine==5.15.6\n + pyqtwebengine-qt5==5.15.16\n + pytest==7.4.4\n + setuptools==59.8.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + wheel==0.42.0\n + zipp==3.15.0\nResolved 55 packages in 330ms\nInstalled 48 packages in 528ms\n + anyio==3.7.1\n + baycomp==1.0.3\n + bottleneck==1.4.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + cycler==0.11.0\n + et-xmlfile==1.1.0\n + fonttools==4.38.0\n + h11==0.12.0\n + httpcore==0.13.7\n + httpx==0.19.0\n + idna==3.10\n + importlib-resources==5.12.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + joblib==1.3.2\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + kiwisolver==1.4.5\n + matplotlib==3.5.3\n + more-itertools==9.1.0\n + networkx==2.6.3\n + openpyxl==3.1.3\n + opentsne==1.0.1\n + pandas==1.3.5\n + pillow==9.5.0\n + pip==24.0\n + pycparser==2.21\n + pyparsing==3.1.4\n + python-dateutil==2.9.0.post0\n + python-louvain==0.16\n + pytz==2024.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + rfc3986==1.5.0\n + scikit-learn==1.0.2\n + scipy==1.7.3\n + secretstorage==3.3.3\n + serverfiles==0.3.1\n + six==1.17.0\n + sniffio==1.3.1\n + threadpoolctl==3.1.0\n + urllib3==2.0.7\n + xlrd==2.0.1\n + xlsxwriter==3.2.0\nResolved 59 packages in 235ms\nInstalled 38 packages in 667ms\n + anyqt==0.2.0\n + attrs==24.2.0\n + backcall==0.2.0\n + cattrs==23.1.2\n + commonmark==0.9.1\n + debugpy==1.7.0\n + decorator==5.1.1\n + dictdiffer==0.9.0\n + docutils==0.20.1\n + entrypoints==0.4\n + ipykernel==6.16.2\n + ipython==7.34.0\n + ipython-genutils==0.2.0\n + jedi==0.19.2\n + jupyter-client==7.4.9\n + jupyter-core==4.12.0\n + matplotlib-inline==0.1.6\n + nest-asyncio==1.6.0\n + orange-canvas-core==0.1.35\n + orange-widget-base==4.22.0\n + parso==0.8.4\n + pexpect==4.9.0\n + pickleshare==0.7.5\n + platformdirs==4.0.0\n + prompt-toolkit==3.0.48\n + psutil==6.1.1\n + ptyprocess==0.7.0\n + pygments==2.17.2\n + pyqtgraph==0.12.4\n + pyzmq==26.2.0\n + qasync==0.25.0\n + qtconsole==5.4.4\n + qtpy==2.4.2\n + requests-cache==1.1.1\n + tornado==6.2\n + traitlets==5.9.0\n + url-normalize==1.4.3\n + wcwidth==0.2.13\nResolved 2 packages in 5ms\nInstalled 2 packages in 16ms\n + psycopg2==2.9.9\n + pymssql==2.3.2\nResolved 19 packages in 48ms\nInstalled 5 packages in 981ms\n + catboost==1.2.7\n + graphviz==0.20.1\n + plotly==5.18.0\n + tenacity==8.2.3\n + xgboost==1.6.2\nAudited 2 packages in 58ms\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/classification/_tree_scorers.c:775:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_2find_threshold_entropy’:\nOrange/classification/_tree_scorers.c:3427:27: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 3427 |   __pyx_t_7 = ((__pyx_v_N <= __pyx_v_min_leaf) != 0);\n      |                           ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_6find_threshold_MSE’:\nOrange/classification/_tree_scorers.c:4883:27: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 4883 |   __pyx_t_1 = ((__pyx_v_N <= __pyx_v_min_leaf) != 0);\n      |                           ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_8find_binarization_MSE’:\nOrange/classification/_tree_scorers.c:5629:43: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5629 |           for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) {\n      |                                           ^\nOrange/classification/_tree_scorers.c:5750:39: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5750 |           __pyx_t_20 = ((__pyx_v_left >= __pyx_v_min_leaf) != 0);\n      |                                       ^~\nOrange/classification/_tree_scorers.c:5756:53: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5756 |           __pyx_t_20 = (((__pyx_v_N - __pyx_v_left) >= __pyx_v_min_leaf) != 0);\n      |                                                     ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_14compute_predictions_csr’:\nOrange/classification/_tree_scorers.c:7217:41: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 7217 |         for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_11; __pyx_t_12+=1) {\n      |                                         ^\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_16compute_predictions_csc’:\nOrange/classification/_tree_scorers.c:7874:41: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 7874 |         for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_11; __pyx_t_12+=1) {\n      |                                         ^\nOrange/classification/_tree_scorers.c:7934:150: warning: comparison of integer expressions of different signedness: ‘__pyx_t_5numpy_int32_t’ {aka ‘int’} and ‘unsigned int’ [-Wsign-compare]\n 7934 |               __pyx_t_16 = (((*((__pyx_t_5numpy_int32_t const  *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_15 * __pyx_v_indices.strides[0]) ))) != __pyx_v_i) != 0);\n      |                                                                                                                                                      ^~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/data/_contingency.c:775:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/data/_io.c: In function ‘__pyx_pf_6Orange_4data_3_io_sparse_read_float.constprop’:\nOrange/data/_io.c:4120:43: warning: ‘__pyx_v_decs’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4120 |           __pyx_v_value = ((__pyx_v_value * __pyx_v_decs) + (__pyx_v_c & 0xf));\n      |                            ~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/data/_io.c:4318:24: warning: ‘__pyx_v_value’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4318 |           __pyx_t_10 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 223, __pyx_L5_error)\n      |                        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:683:40: warning: ‘__pyx_v_attr_index’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n  683 |   #define PyInt_FromLong               PyLong_FromLong\n      |                                        ^~~~~~~~~~~~~~~\nOrange/data/_io.c:1590:7: note: ‘__pyx_v_attr_index’ was declared here\n 1590 |   int __pyx_v_attr_index;\n      |       ^~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:3183:25: warning: ‘__pyx_v_atomp’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3183 |           __pyx_v_atomp = (__pyx_v_atomp + 1);\n      |           ~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/data/_valuecount.c:775:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/distance/_distance.c:775:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/preprocess/_discretize.c:774:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/preprocess/_relieff.cpp:790:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/projection/_som.c:775:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_5update_hex’:\nOrange/projection/_som.c:3863:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3863 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:3862:7: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3862 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pf_6Orange_10projection_4_som_8update_sparse.constprop’:\nOrange/projection/_som.c:4934:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4934 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:4933:7: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4933 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_7get_winners_sparse’:\nOrange/projection/_som.c:4732:223: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4732 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_26, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_27, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_y;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:4307:7: note: ‘__pyx_v_win_y’ was declared here\n 4307 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:4721:223: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4721 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_27, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_26, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_x;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:4306:7: note: ‘__pyx_v_win_x’ was declared here\n 4306 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pf_6Orange_10projection_4_som_10update_sparse_hex.constprop’:\nOrange/projection/_som.c:5682:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 5682 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:5681:7: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 5681 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_1get_winners’:\nOrange/projection/_som.c:3184:223: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3184 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_22, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_23, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_y;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:2887:7: note: ‘__pyx_v_win_y’ was declared here\n 2887 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:3173:223: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3173 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_23, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_22, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_x;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:2886:7: note: ‘__pyx_v_win_x’ was declared here\n 2886 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_3update’:\nOrange/projection/_som.c:3357:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3357 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/classification/_simple_tree.c: In function ‘gain_ratio_c’:\nOrange/classification/_simple_tree.c:63:9: warning: implicit declaration of function ‘qsort_r’; did you mean ‘qsort’? [-Wimplicit-function-declaration]\n   63 |         qsort_r(base, nel, size, compar, thunk)\n      |         ^~~~~~~\nOrange/classification/_simple_tree.c:147:9: note: in expansion of macro ‘QSORT_R_FUNC’\n  147 |         QSORT_R_FUNC(examples, size, sizeof(struct Example), (void*) &compar_attr, compar_examples);\n      |         ^~~~~~~~~~~~\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/setuptools/command/easy_install.py:159: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools.\n  EasyInstallDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/.venv/lib/python3.7/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n  setuptools.SetuptoolsDeprecationWarning,\nwarning: no files found matching 'README*' under directory 'Orange'\nwarning: no files found matching 'LICENSE' under directory 'Orange'\nwarning: no files found matching '*.basket' under directory 'Orange/datasets'\nwarning: no files found matching '*.svg' under directory 'Orange/canvas'\nwarning: no files found matching '*.js' under directory 'Orange/widgets'\nwarning: no files found matching '*.css' under directory 'Orange/widgets'\nwarning: no files found matching '*.html' under directory 'Orange/widgets'\nwarning: no files found matching '*.svg' under directory 'distribute'\nwarning: no files found matching 'COPYING'\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd\nplugins: anyio-3.7.1\ncollected 29 items\n\nr2e_tests/test_1.py sssE.........................                        [100%]\n\n==================================== ERRORS ====================================\n_______________________ ERROR at setup of test_filename ________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/Orange/tests/__init__.py, line 68\n  def test_filename(path):\nE       fixture 'path' not found\n>       available fixtures: anyio_backend, anyio_backend_name, anyio_backend_options, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/Orange/tests/__init__.py:68\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_apply_no_output\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_calibration_curve\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_change_target_class\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_changing_score_explanation\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_contexts\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_empty_input_error\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_image_export\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_initialization\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_many_evaluation_results\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_minimum_size\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_msg_base_class\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_multiple_fold_curves\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_no_folds\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_output_calibrated_classifier\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_output_threshold_classifier\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_plotting_curves\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_regression_input_error\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_report\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_rug\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_single_class\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_single_class_folds\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_threshold_change_updates_info\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_threshold_flips_on_two_classes\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_threshold_rounding\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_warn_nan_probabilities\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:244: image exporting not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:238: minimum size not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:250: msg base class not tested as .widget was not set\nERROR r2e_tests/test_1.py::test_filename\n==================== 25 passed, 3 skipped, 1 error in 9.64s ====================\nWidgets left: 0    Max widgets: 4275 \n\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd\nplugins: anyio-3.7.1\ncollected 29 items\n\nr2e_tests/test_1.py sssE............F............                        [100%]\n\n==================================== ERRORS ====================================\n_______________________ ERROR at setup of test_filename ________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/Orange/tests/__init__.py, line 68\n  def test_filename(path):\nE       fixture 'path' not found\n>       available fixtures: anyio_backend, anyio_backend_name, anyio_backend_options, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_0cefc175f4e7facac38eba0bfd6b4c6ef1a29ddd/Orange/tests/__init__.py:68\n=================================== FAILURES ===================================\n_____________________ TestOWCalibrationPlot.test_no_folds ______________________\n\nself = <r2e_tests.test_1.TestOWCalibrationPlot testMethod=test_no_folds>\n_ = (<MagicMock name='CalibratedLearner' id='138790387322448'>, <MagicMock name='ThresholdClassifier' id='138790387322576'>)\nwidget = <Orange.widgets.evaluate.owcalibrationplot.OWCalibrationPlot object at 0x7e3aa7b46eb0>\n\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.ThresholdClassifier\")\n    @patch(\"Orange.widgets.evaluate.owcalibrationplot.CalibratedLearner\")\n    def test_no_folds(self, *_):\n        \"\"\"Warn about omitted points with nan probabiities\"\"\"\n        widget = self.widget\n    \n        self.results.folds = None\n>       self.send_signal(widget.Inputs.evaluation_results, self.results)\n\nr2e_tests/test_1.py:649: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n.venv/lib/python3.7/site-packages/orangewidget/tests/base.py:446: in send_signal\n    widget=widget, wait=wait)\n.venv/lib/python3.7/site-packages/orangewidget/tests/base.py:471: in send_signals\n    self._send_signal(widget, input, value, *args)\n.venv/lib/python3.7/site-packages/orangewidget/tests/base.py:490: in _send_signal\n    notify_input_helper(input, widget, value, *args, **kwargs)\n../../../../.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/functools.py:840: in wrapper\n    return dispatch(args[0].__class__)(*args, **kw)\n.venv/lib/python3.7/site-packages/orangewidget/utils/signals.py:735: in set_input_helper\n    handler(*args)\n.venv/lib/python3.7/site-packages/orangewidget/utils/signals.py:208: in summarize_wrapper\n    method(widget, value)\nOrange/widgets/evaluate/owcalibrationplot.py:239: in set_results\n    self.commit.now()\n.venv/lib/python3.7/site-packages/orangewidget/gui.py:2007: in do_commit\n    commit.call()\n.venv/lib/python3.7/site-packages/orangewidget/gui.py:1872: in call\n    acting_func(instance)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <Orange.widgets.evaluate.owcalibrationplot.OWCalibrationPlot object at 0x7e3aa7b46eb0>\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n>                   (len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\nE           TypeError: object of type 'NoneType' has no len()\n\nOrange/widgets/evaluate/owcalibrationplot.py:490: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_apply_no_output\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_calibration_curve\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_change_target_class\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_changing_score_explanation\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_contexts\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_empty_input_error\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_image_export\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_initialization\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_many_evaluation_results\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_minimum_size\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_msg_base_class\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_multiple_fold_curves\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_output_calibrated_classifier\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_output_threshold_classifier\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_plotting_curves\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_regression_input_error\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_report\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_rug\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_single_class\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_single_class_folds\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_threshold_change_updates_info\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_threshold_flips_on_two_classes\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_threshold_rounding\nPASSED r2e_tests/test_1.py::TestOWCalibrationPlot::test_warn_nan_probabilities\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:244: image exporting not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:238: minimum size not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:250: msg base class not tested as .widget was not set\nERROR r2e_tests/test_1.py::test_filename\nFAILED r2e_tests/test_1.py::TestOWCalibrationPlot::test_no_folds - TypeError:...\n=============== 1 failed, 24 passed, 3 skipped, 1 error in 7.35s ===============\nWidgets left: 0    Max widgets: 4275 \n\n",
    "old_commit_res_stderr": ""
}
Orange/widgets/evaluate/owcalibrationplot.py Orange/widgets/evaluate/tests/test_owcalibrationplot.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Calibration Plot crashes when Results have no folds

**Description:**
When the Calibration Plot widget processes evaluation results that do not include any folds (i.e., `results.folds` is `None`), it crashes with a `TypeError`. This happens because the widget attempts to determine the number of folds using `len(results.folds)` without first verifying if `folds` is `None`.

**Example Buggy Code:**
```python
results = EvaluationResults(folds=None, models=...)
widget = OWCalibrationPlot()
widget.set_results(results)
widget.commit.now()
```

**Expected Behavior:**
The Calibration Plot should handle cases where `results.folds` is `None` gracefully. It should either skip fold-related processing or notify the user that no fold data is available, avoiding any crashes.

**Actual Behavior:**
A `TypeError` is raised with the message: "object of type 'NoneType' has no len()", causing the Calibration Plot widget to crash.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### Orange/widgets/evaluate/owcalibrationplot.py
from collections import namedtuple

import numpy as np

from AnyQt.QtCore import Qt, QSize
from AnyQt.QtWidgets import QListWidget, QSizePolicy

import pyqtgraph as pg

from orangewidget.utils.visual_settings_dlg import VisualSettingsDialog

from Orange.base import Model
from Orange.classification import ThresholdClassifier, CalibratedLearner
from Orange.evaluation import Results
from Orange.evaluation.performance_curves import Curves
from Orange.widgets import widget, gui, settings
from Orange.widgets.evaluate.contexthandlers import \
    EvaluationResultsContextHandler
from Orange.widgets.evaluate.utils import results_for_preview
from Orange.widgets.utils import colorpalettes
from Orange.widgets.utils.widgetpreview import WidgetPreview
from Orange.widgets.visualize.utils.customizableplot import \
    CommonParameterSetter
from Orange.widgets.visualize.utils.plotutils import GraphicsView, PlotItem
from Orange.widgets.widget import Input, Output, Msg
from Orange.widgets import report


MetricDefinition = namedtuple(
    "metric_definition",
    ("name", "functions", "short_names", "explanation"))

Metrics = [MetricDefinition(*args) for args in (
    ("Calibration curve", None, (), ""),
    ("Classification accuracy", (Curves.ca, ), (), ""),
    ("F1", (Curves.f1, ), (), ""),
    ("Sensitivity and specificity",
     (Curves.sensitivity, Curves.specificity),
     ("sens", "spec"),
     "<p><b>Sensitivity</b> (falling) is the proportion of correctly "
     "detected positive instances (TP&nbsp;/&nbsp;P).</p>"
     "<p><b>Specificity</b> (rising) is the proportion of detected "
     "negative instances (TN&nbsp;/&nbsp;N).</p>"),
    ("Precision and recall",
     (Curves.precision, Curves.recall),
     ("prec", "recall"),
     "<p><b>Precision</b> (rising) is the fraction of retrieved instances "
     "that are relevant, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>"
     "<p><b>Recall</b> (falling) is the proportion of discovered relevant "
     "instances, TP&nbsp;/&nbsp;P.</p>"),
    ("Pos and neg predictive value",
     (Curves.ppv, Curves.npv),
     ("PPV", "TPV"),
     "<p><b>Positive predictive value</b> (rising) is the proportion of "
     "correct positives, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>"
     "<p><b>Negative predictive value</b> is the proportion of correct "
     "negatives, TN&nbsp;/&nbsp;(TN&nbsp;+&nbsp;FN).</p>"),
    ("True and false positive rate",
     (Curves.tpr, Curves.fpr),
     ("TPR", "FPR"),
     "<p><b>True and false positive rate</b> are proportions of detected "
     "and omitted positive instances</p>"),
)]


class ParameterSetter(CommonParameterSetter):

    def __init__(self, master):
        super().__init__()
        self.master = master

    def update_setters(self):
        self.initial_settings = {
            self.LABELS_BOX: {
                self.FONT_FAMILY_LABEL: self.FONT_FAMILY_SETTING,
                self.TITLE_LABEL: self.FONT_SETTING,
                self.AXIS_TITLE_LABEL: self.FONT_SETTING,
                self.AXIS_TICKS_LABEL: self.FONT_SETTING,
            },
            self.ANNOT_BOX: {
                self.TITLE_LABEL: {self.TITLE_LABEL: ("", "")},
            }
        }

    @property
    def title_item(self):
        return self.master.titleLabel

    @property
    def axis_items(self):
        return [value["item"] for value in self.master.axes.values()]


class OWCalibrationPlot(widget.OWWidget):
    name = "Calibration Plot"
    description = "Calibration plot based on evaluation of classifiers."
    icon = "icons/CalibrationPlot.svg"
    priority = 1030
    keywords = []

    class Inputs:
        evaluation_results = Input("Evaluation Results", Results)

    class Outputs:
        calibrated_model = Output("Calibrated Model", Model)

    class Error(widget.OWWidget.Error):
        non_discrete_target = Msg("Calibration plot requires a categorical "
                                  "target variable.")
        empty_input = widget.Msg("Empty result on input. Nothing to display.")
        nan_classes = \
            widget.Msg("Remove test data instances with unknown classes.")
        all_target_class = widget.Msg(
            "All data instances belong to target class.")
        no_target_class = widget.Msg(
            "No data instances belong to target class.")

    class Warning(widget.OWWidget.Warning):
        omitted_folds = widget.Msg(
            "Test folds where all data belongs to (non)-target are not shown.")
        omitted_nan_prob_points = widget.Msg(
            "Instance for which the model couldn't compute probabilities are"
            "skipped.")
        no_valid_data = widget.Msg("No valid data for model(s) {}")

    class Information(widget.OWWidget.Information):
        no_output = Msg("Can't output a model: {}")

    settingsHandler = EvaluationResultsContextHandler()
    target_index = settings.ContextSetting(0)
    selected_classifiers = settings.ContextSetting([])
    score = settings.Setting(0)
    output_calibration = settings.Setting(0)
    fold_curves = settings.Setting(False)
    display_rug = settings.Setting(True)
    threshold = settings.Setting(0.5)
    visual_settings = settings.Setting({}, schema_only=True)
    auto_commit = settings.Setting(True)

    graph_name = "plot"

    def __init__(self):
        super().__init__()

        self.results = None
        self.scores = None
        self.classifier_names = []
        self.colors = []
        self.line = None

        self._last_score_value = -1

        box = gui.vBox(self.controlArea, box="Settings")
        self.target_cb = gui.comboBox(
            box, self, "target_index", label="Target:",
            orientation=Qt.Horizontal, callback=self.target_index_changed,
            contentsLength=8, searchable=True)
        gui.checkBox(
            box, self, "display_rug", "Show rug",
            callback=self._on_display_rug_changed)
        gui.checkBox(
            box, self, "fold_curves", "Curves for individual folds",
            callback=self._replot)

        self.classifiers_list_box = gui.listBox(
            self.controlArea, self, "selected_classifiers", "classifier_names",
            box="Classifier", selectionMode=QListWidget.ExtendedSelection,
            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),
            sizeHint=QSize(150, 40),
            callback=self._on_selection_changed)

        box = gui.vBox(self.controlArea, "Metrics")
        combo = gui.comboBox(
            box, self, "score", items=(metric.name for metric in Metrics),
            callback=self.score_changed)

        self.explanation = gui.widgetLabel(
            box, wordWrap=True, fixedWidth=combo.sizeHint().width())
        self.explanation.setContentsMargins(8, 8, 0, 0)
        font = self.explanation.font()
        font.setPointSizeF(0.85 * font.pointSizeF())
        self.explanation.setFont(font)

        gui.radioButtons(
            box, self, value="output_calibration",
            btnLabels=("Sigmoid calibration", "Isotonic calibration"),
            label="Output model calibration", callback=self.commit.deferred)

        self.info_box = gui.widgetBox(self.controlArea, "Info")
        self.info_label = gui.widgetLabel(self.info_box)

        gui.auto_apply(self.buttonsArea, self, "auto_commit")

        self.plotview = GraphicsView()
        self.plot = PlotItem(enableMenu=False)
        self.plot.parameter_setter = ParameterSetter(self.plot)
        self.plot.setMouseEnabled(False, False)
        self.plot.hideButtons()
        for axis_name in ("bottom", "left"):
            axis = self.plot.getAxis(axis_name)
            # Remove the condition (that is, allow setting this for bottom
            # axis) when pyqtgraph is fixed
            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930
            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932
            if axis_name != "bottom":  # remove if when pyqtgraph is fixed
                axis.setStyle(stopAxisAtTick=(True, True))

        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)
        self.plotview.setCentralItem(self.plot)

        self.mainArea.layout().addWidget(self.plotview)
        self._set_explanation()

        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)

    @Inputs.evaluation_results
    def set_results(self, results):
        self.closeContext()
        self.clear()
        self.Error.clear()
        self.Information.clear()

        self.results = None
        if results is not None:
            if not results.domain.has_discrete_class:
                self.Error.non_discrete_target()
            elif not results.actual.size:
                self.Error.empty_input()
            elif np.any(np.isnan(results.actual)):
                self.Error.nan_classes()
            else:
                self.results = results
                self._initialize(results)
                class_var = self.results.domain.class_var
                self.target_index = int(len(class_var.values) == 2)
                self.openContext(class_var, self.classifier_names)
                self._replot()

        self.commit.now()

    def clear(self):
        self.plot.clear()
        self.results = None
        self.classifier_names = []
        self.selected_classifiers = []
        self.target_cb.clear()
        self.colors = []

    def target_index_changed(self):
        if len(self.results.domain.class_var.values) == 2:
            self.threshold = 1 - self.threshold
        self._set_explanation()
        self._replot()
        self.commit.deferred()

    def score_changed(self):
        self._set_explanation()
        self._replot()
        if self._last_score_value != self.score:
            self.commit.deferred()
            self._last_score_value = self.score

    def _set_explanation(self):
        explanation = Metrics[self.score].explanation
        if explanation:
            self.explanation.setText(explanation)
            self.explanation.show()
        else:
            self.explanation.hide()

        if self.score == 0:
            self.controls.output_calibration.show()
            self.info_box.hide()
        else:
            self.controls.output_calibration.hide()
            self.info_box.show()

        axis = self.plot.getAxis("bottom")
        axis.setLabel("Predicted probability" if self.score == 0
                      else "Threshold probability to classify as positive")

        axis = self.plot.getAxis("left")
        axis.setLabel(Metrics[self.score].name)

    def _initialize(self, results):
        n = len(results.predicted)
        names = getattr(results, "learner_names", None)
        if names is None:
            names = ["#{}".format(i + 1) for i in range(n)]

        self.classifier_names = names
        self.colors = colorpalettes.get_default_curve_colors(n)

        for i in range(n):
            item = self.classifiers_list_box.item(i)
            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))

        self.selected_classifiers = list(range(n))
        self.target_cb.addItems(results.domain.class_var.values)
        self.target_index = 0

    def _rug(self, data, pen_args):
        color = pen_args["pen"].color()
        rh = 0.025
        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]
        rug_x_true = rug_x[data.ytrue].ravel()
        rug_x_false = rug_x[~data.ytrue].ravel()

        rug_y_true = np.ones_like(rug_x_true)
        rug_y_true[1::2] = 1 - rh
        rug_y_false = np.zeros_like(rug_x_false)
        rug_y_false[1::2] = rh

        self.plot.plot(
            rug_x_false, rug_y_false,
            pen=color, connect="pairs", antialias=True)
        self.plot.plot(
            rug_x_true, rug_y_true,
            pen=color, connect="pairs", antialias=True)

    def plot_metrics(self, data, metrics, pen_args):
        if metrics is None:
            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)
        ys = [metric(data) for metric in metrics]
        for y in ys:
            self.plot.plot(data.probs, y, **pen_args)
        return data.probs, ys

    def _prob_curve(self, ytrue, probs, pen_args):
        xmin, xmax = probs.min(), probs.max()
        x = np.linspace(xmin, xmax, 100)
        if xmax != xmin:
            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))
            y = f(x)
        else:
            y = np.full(100, xmax)

        self.plot.plot(x, y, symbol="+", symbolSize=4, **pen_args)
        return x, (y, )

    def _setup_plot(self):
        target = self.target_index
        results = self.results
        metrics = Metrics[self.score].functions
        plot_folds = self.fold_curves and results.folds is not None
        self.scores = []

        if not self._check_class_presence(results.actual == target):
            return

        self.Warning.omitted_folds.clear()
        self.Warning.omitted_nan_prob_points.clear()
        no_valid_models = []
        shadow_width = 4 + 4 * plot_folds
        for clsf in self.selected_classifiers:
            data = Curves.from_results(results, target, clsf)
            if data.tot == 0:  # all probabilities are nan
                no_valid_models.append(clsf)
                continue
            if data.tot != results.probabilities.shape[1]:  # some are nan
                self.Warning.omitted_nan_prob_points()

            color = self.colors[clsf]
            pen_args = dict(
                pen=pg.mkPen(color, width=1), antiAlias=True,
                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))
            self.scores.append(
                (self.classifier_names[clsf],
                 self.plot_metrics(data, metrics, pen_args)))

            if self.display_rug:
                self._rug(data, pen_args)

            if plot_folds:
                pen_args = dict(
                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),
                    antiAlias=True)
                for fold in range(len(results.folds)):
                    fold_results = results.get_fold(fold)
                    fold_curve = Curves.from_results(fold_results, target, clsf)
                    # Can't check this before: p and n can be 0 because of
                    # nan probabilities
                    if fold_curve.p * fold_curve.n == 0:
                        self.Warning.omitted_folds()
                    self.plot_metrics(fold_curve, metrics, pen_args)

        if no_valid_models:
            self.Warning.no_valid_data(
                ", ".join(self.classifier_names[i] for i in no_valid_models))

        if self.score == 0:
            self.plot.plot([0, 1], [0, 1], antialias=True)
        else:
            self.line = pg.InfiniteLine(
                pos=self.threshold, movable=True,
                pen=pg.mkPen(color="k", style=Qt.DashLine, width=2),
                hoverPen=pg.mkPen(color="k", style=Qt.DashLine, width=3),
                bounds=(0, 1),
            )
            self.line.sigPositionChanged.connect(self.threshold_change)
            self.line.sigPositionChangeFinished.connect(
                self.threshold_change_done)
            self.plot.addItem(self.line)

    def _check_class_presence(self, ytrue):
        self.Error.all_target_class.clear()
        self.Error.no_target_class.clear()
        if np.max(ytrue) == 0:
            self.Error.no_target_class()
            return False
        if np.min(ytrue) == 1:
            self.Error.all_target_class()
            return False
        return True

    def _replot(self):
        self.plot.clear()
        if self.results is not None:
            self._setup_plot()
        self._update_info()

    def _on_display_rug_changed(self):
        self._replot()

    def _on_selection_changed(self):
        self._replot()
        self.commit.deferred()

    def threshold_change(self):
        self.threshold = round(self.line.pos().x(), 2)
        self.line.setPos(self.threshold)
        self._update_info()

    def get_info_text(self, short):
        if short:
            def elided(s):
                return s[:17] + "..." if len(s) > 20 else s

            text = f"""<table>
                            <tr>
                                <th align='right'>Threshold: p=</th>
                                <td colspan='4'>{self.threshold:.2f}<br/></td>
                            </tr>"""

        else:
            def elided(s):
                return s

            text = f"""<table>
                            <tr>
                                <th align='right'>Threshold:</th>
                                <td colspan='4'>p = {self.threshold:.2f}<br/>
                                </td>
                                <tr/>
                            </tr>"""

        if self.scores is not None:
            short_names = Metrics[self.score].short_names
            if short_names:
                text += f"""<tr>
                                <th></th>
                                {"<td></td>".join(f"<td align='right'>{n}</td>"
                                                  for n in short_names)}
                            </tr>"""
            for name, (probs, curves) in self.scores:
                ind = min(np.searchsorted(probs, self.threshold),
                          len(probs) - 1)
                text += f"<tr><th align='right'>{elided(name)}:</th>"
                text += "<td>/</td>".join(f'<td>{curve[ind]:.3f}</td>'
                                          for curve in curves)
                text += "</tr>"
            text += "<table>"
            return text
        return None

    def _update_info(self):
        self.info_label.setText(self.get_info_text(short=True))

    def threshold_change_done(self):
        self.commit.deferred()

    @gui.deferred
    def commit(self):
        self.Information.no_output.clear()
        wrapped = None
        results = self.results
        if results is not None:
            problems = [
                msg for condition, msg in (
                    (len(results.folds) > 1,
                     "each training data sample produces a different model"),
                    (results.models is None,
                     "test results do not contain stored models - try testing "
                     "on separate data or on training data"),
                    (len(self.selected_classifiers) != 1,
                     "select a single model - the widget can output only one"),
                    (self.score != 0 and len(results.domain.class_var.values) != 2,
                     "cannot calibrate non-binary classes"))
                if condition]
            if len(problems) == 1:
                self.Information.no_output(problems[0])
            elif problems:
                self.Information.no_output(
                    "".join(f"\n - {problem}" for problem in problems))
            else:
                clsf_idx = self.selected_classifiers[0]
                model = results.models[0, clsf_idx]
                if self.score == 0:
                    cal_learner = CalibratedLearner(
                        None, self.output_calibration)
                    wrapped = cal_learner.get_model(
                        model, results.actual, results.probabilities[clsf_idx])
                else:
                    threshold = [1 - self.threshold,
                                 self.threshold][self.target_index]
                    wrapped = ThresholdClassifier(model, threshold)

        self.Outputs.calibrated_model.send(wrapped)

    def send_report(self):
        if self.results is None:
            return
        self.report_items((
            ("Target class", self.target_cb.currentText()),
            ("Output model calibration",
             self.score == 0
             and ("Sigmoid calibration",
                  "Isotonic calibration")[self.output_calibration])
        ))
        caption = report.list_legend(self.classifiers_list_box,
                                     self.selected_classifiers)
        self.report_plot()
        self.report_caption(caption)
        self.report_caption(self.controls.score.currentText())

        if self.score != 0:
            self.report_raw(self.get_info_text(short=False))

    def set_visual_settings(self, key, value):
        self.plot.parameter_setter.set_parameter(key, value)
        self.visual_settings[key] = value


def gaussian_smoother(x, y, sigma=1.0):
    x = np.asarray(x)
    y = np.asarray(y)

    gamma = 1. / (2 * sigma ** 2)
    a = 1. / (sigma * np.sqrt(2 * np.pi))

    if x.shape != y.shape:
        raise ValueError

    def smoother(xs):
        W = a * np.exp(-gamma * ((xs - x) ** 2))
        return np.average(y, weights=W)

    return np.vectorize(smoother, otypes=[float])


if __name__ == "__main__":  # pragma: no cover
    WidgetPreview(OWCalibrationPlot).run(results_for_preview())

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Calibration Plot crashes when Results have no folds

**Description:**
When the Calibration Plot widget processes evaluation results that do not include any folds (i.e., `results.folds` is `None`), it crashes with a `TypeError`. This happens because the widget attempts to determine the number of folds using `len(results.folds)` without first verifying if `folds` is `None`.

**Example Buggy Code:**
```python
results = EvaluationResults(folds=None, models=...)
widget = OWCalibrationPlot()
widget.set_results(results)
widget.commit.now()
```

**Expected Behavior:**
The Calibration Plot should handle cases where `results.folds` is `None` gracefully. It should either skip fold-related processing or notify the user that no fold data is available, avoiding any crashes.

**Actual Behavior:**
A `TypeError` is raised with the message: "object of type 'NoneType' has no len()", causing the Calibration Plot widget to crash.
[/ISSUE]
{
    "TestOWCalibrationPlot.test_apply_no_output": "PASSED",
    "TestOWCalibrationPlot.test_calibration_curve": "PASSED",
    "TestOWCalibrationPlot.test_change_target_class": "PASSED",
    "TestOWCalibrationPlot.test_changing_score_explanation": "PASSED",
    "TestOWCalibrationPlot.test_contexts": "PASSED",
    "TestOWCalibrationPlot.test_empty_input_error": "PASSED",
    "TestOWCalibrationPlot.test_image_export": "PASSED",
    "TestOWCalibrationPlot.test_initialization": "PASSED",
    "TestOWCalibrationPlot.test_many_evaluation_results": "PASSED",
    "TestOWCalibrationPlot.test_minimum_size": "PASSED",
    "TestOWCalibrationPlot.test_msg_base_class": "PASSED",
    "TestOWCalibrationPlot.test_multiple_fold_curves": "PASSED",
    "TestOWCalibrationPlot.test_no_folds": "PASSED",
    "TestOWCalibrationPlot.test_output_calibrated_classifier": "PASSED",
    "TestOWCalibrationPlot.test_output_threshold_classifier": "PASSED",
    "TestOWCalibrationPlot.test_plotting_curves": "PASSED",
    "TestOWCalibrationPlot.test_regression_input_error": "PASSED",
    "TestOWCalibrationPlot.test_report": "PASSED",
    "TestOWCalibrationPlot.test_rug": "PASSED",
    "TestOWCalibrationPlot.test_single_class": "PASSED",
    "TestOWCalibrationPlot.test_single_class_folds": "PASSED",
    "TestOWCalibrationPlot.test_threshold_change_updates_info": "PASSED",
    "TestOWCalibrationPlot.test_threshold_flips_on_two_classes": "PASSED",
    "TestOWCalibrationPlot.test_threshold_rounding": "PASSED",
    "TestOWCalibrationPlot.test_warn_nan_probabilities": "PASSED",
    "test_filename": "ERROR"
}
diff --git a/Orange/widgets/evaluate/owcalibrationplot.py b/Orange/widgets/evaluate/owcalibrationplot.py
index 72c2822d8..853c7ae3c 100644
--- a/Orange/widgets/evaluate/owcalibrationplot.py
+++ b/Orange/widgets/evaluate/owcalibrationplot.py
@@ -487,7 +487,7 @@ class OWCalibrationPlot(widget.OWWidget):
         if results is not None:
             problems = [
                 msg for condition, msg in (
-                    (len(results.folds) > 1,
+                    (results.folds is not None and len(results.folds) > 1,
                      "each training data sample produces a different model"),
                     (results.models is None,
                      "test results do not contain stored models - try testing "

{"Orange/widgets/evaluate/owcalibrationplot.py": "from collections import namedtuple\n\nimport numpy as np\n\nfrom AnyQt.QtCore import Qt, QSize\nfrom AnyQt.QtWidgets import QListWidget, QSizePolicy\n\nimport pyqtgraph as pg\n\nfrom orangewidget.utils.visual_settings_dlg import VisualSettingsDialog\n\nfrom Orange.base import Model\nfrom Orange.classification import ThresholdClassifier, CalibratedLearner\nfrom Orange.evaluation import Results\nfrom Orange.evaluation.performance_curves import Curves\nfrom Orange.widgets import widget, gui, settings\nfrom Orange.widgets.evaluate.contexthandlers import \\\n    EvaluationResultsContextHandler\nfrom Orange.widgets.evaluate.utils import results_for_preview\nfrom Orange.widgets.utils import colorpalettes\nfrom Orange.widgets.utils.widgetpreview import WidgetPreview\nfrom Orange.widgets.visualize.utils.customizableplot import \\\n    CommonParameterSetter\nfrom Orange.widgets.visualize.utils.plotutils import GraphicsView, PlotItem\nfrom Orange.widgets.widget import Input, Output, Msg\nfrom Orange.widgets import report\n\n\nMetricDefinition = namedtuple(\n    \"metric_definition\",\n    (\"name\", \"functions\", \"short_names\", \"explanation\"))\n\nMetrics = [MetricDefinition(*args) for args in (\n    (\"Calibration curve\", None, (), \"\"),\n    (\"Classification accuracy\", (Curves.ca, ), (), \"\"),\n    (\"F1\", (Curves.f1, ), (), \"\"),\n    (\"Sensitivity and specificity\",\n     (Curves.sensitivity, Curves.specificity),\n     (\"sens\", \"spec\"),\n     \"<p><b>Sensitivity</b> (falling) is the proportion of correctly \"\n     \"detected positive instances (TP&nbsp;/&nbsp;P).</p>\"\n     \"<p><b>Specificity</b> (rising) is the proportion of detected \"\n     \"negative instances (TN&nbsp;/&nbsp;N).</p>\"),\n    (\"Precision and recall\",\n     (Curves.precision, Curves.recall),\n     (\"prec\", \"recall\"),\n     \"<p><b>Precision</b> (rising) is the fraction of retrieved instances \"\n     \"that are relevant, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>\"\n     \"<p><b>Recall</b> (falling) is the proportion of discovered relevant \"\n     \"instances, TP&nbsp;/&nbsp;P.</p>\"),\n    (\"Pos and neg predictive value\",\n     (Curves.ppv, Curves.npv),\n     (\"PPV\", \"TPV\"),\n     \"<p><b>Positive predictive value</b> (rising) is the proportion of \"\n     \"correct positives, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>\"\n     \"<p><b>Negative predictive value</b> is the proportion of correct \"\n     \"negatives, TN&nbsp;/&nbsp;(TN&nbsp;+&nbsp;FN).</p>\"),\n    (\"True and false positive rate\",\n     (Curves.tpr, Curves.fpr),\n     (\"TPR\", \"FPR\"),\n     \"<p><b>True and false positive rate</b> are proportions of detected \"\n     \"and omitted positive instances</p>\"),\n)]\n\n\nclass ParameterSetter(CommonParameterSetter):\n\n    def __init__(self, master):\n        super().__init__()\n        self.master = master\n\n    def update_setters(self):\n        self.initial_settings = {\n            self.LABELS_BOX: {\n                self.FONT_FAMILY_LABEL: self.FONT_FAMILY_SETTING,\n                self.TITLE_LABEL: self.FONT_SETTING,\n                self.AXIS_TITLE_LABEL: self.FONT_SETTING,\n                self.AXIS_TICKS_LABEL: self.FONT_SETTING,\n            },\n            self.ANNOT_BOX: {\n                self.TITLE_LABEL: {self.TITLE_LABEL: (\"\", \"\")},\n            }\n        }\n\n    @property\n    def title_item(self):\n        return self.master.titleLabel\n\n    @property\n    def axis_items(self):\n        return [value[\"item\"] for value in self.master.axes.values()]\n\n\nclass OWCalibrationPlot(widget.OWWidget):\n    name = \"Calibration Plot\"\n    description = \"Calibration plot based on evaluation of classifiers.\"\n    icon = \"icons/CalibrationPlot.svg\"\n    priority = 1030\n    keywords = []\n\n    class Inputs:\n        evaluation_results = Input(\"Evaluation Results\", Results)\n\n    class Outputs:\n        calibrated_model = Output(\"Calibrated Model\", Model)\n\n    class Error(widget.OWWidget.Error):\n        non_discrete_target = Msg(\"Calibration plot requires a categorical \"\n                                  \"target variable.\")\n        empty_input = widget.Msg(\"Empty result on input. Nothing to display.\")\n        nan_classes = \\\n            widget.Msg(\"Remove test data instances with unknown classes.\")\n        all_target_class = widget.Msg(\n            \"All data instances belong to target class.\")\n        no_target_class = widget.Msg(\n            \"No data instances belong to target class.\")\n\n    class Warning(widget.OWWidget.Warning):\n        omitted_folds = widget.Msg(\n            \"Test folds where all data belongs to (non)-target are not shown.\")\n        omitted_nan_prob_points = widget.Msg(\n            \"Instance for which the model couldn't compute probabilities are\"\n            \"skipped.\")\n        no_valid_data = widget.Msg(\"No valid data for model(s) {}\")\n\n    class Information(widget.OWWidget.Information):\n        no_output = Msg(\"Can't output a model: {}\")\n\n    settingsHandler = EvaluationResultsContextHandler()\n    target_index = settings.ContextSetting(0)\n    selected_classifiers = settings.ContextSetting([])\n    score = settings.Setting(0)\n    output_calibration = settings.Setting(0)\n    fold_curves = settings.Setting(False)\n    display_rug = settings.Setting(True)\n    threshold = settings.Setting(0.5)\n    visual_settings = settings.Setting({}, schema_only=True)\n    auto_commit = settings.Setting(True)\n\n    graph_name = \"plot\"\n\n    def __init__(self):\n        super().__init__()\n\n        self.results = None\n        self.scores = None\n        self.classifier_names = []\n        self.colors = []\n        self.line = None\n\n        self._last_score_value = -1\n\n        box = gui.vBox(self.controlArea, box=\"Settings\")\n        self.target_cb = gui.comboBox(\n            box, self, \"target_index\", label=\"Target:\",\n            orientation=Qt.Horizontal, callback=self.target_index_changed,\n            contentsLength=8, searchable=True)\n        gui.checkBox(\n            box, self, \"display_rug\", \"Show rug\",\n            callback=self._on_display_rug_changed)\n        gui.checkBox(\n            box, self, \"fold_curves\", \"Curves for individual folds\",\n            callback=self._replot)\n\n        self.classifiers_list_box = gui.listBox(\n            self.controlArea, self, \"selected_classifiers\", \"classifier_names\",\n            box=\"Classifier\", selectionMode=QListWidget.ExtendedSelection,\n            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),\n            sizeHint=QSize(150, 40),\n            callback=self._on_selection_changed)\n\n        box = gui.vBox(self.controlArea, \"Metrics\")\n        combo = gui.comboBox(\n            box, self, \"score\", items=(metric.name for metric in Metrics),\n            callback=self.score_changed)\n\n        self.explanation = gui.widgetLabel(\n            box, wordWrap=True, fixedWidth=combo.sizeHint().width())\n        self.explanation.setContentsMargins(8, 8, 0, 0)\n        font = self.explanation.font()\n        font.setPointSizeF(0.85 * font.pointSizeF())\n        self.explanation.setFont(font)\n\n        gui.radioButtons(\n            box, self, value=\"output_calibration\",\n            btnLabels=(\"Sigmoid calibration\", \"Isotonic calibration\"),\n            label=\"Output model calibration\", callback=self.commit.deferred)\n\n        self.info_box = gui.widgetBox(self.controlArea, \"Info\")\n        self.info_label = gui.widgetLabel(self.info_box)\n\n        gui.auto_apply(self.buttonsArea, self, \"auto_commit\")\n\n        self.plotview = GraphicsView()\n        self.plot = PlotItem(enableMenu=False)\n        self.plot.parameter_setter = ParameterSetter(self.plot)\n        self.plot.setMouseEnabled(False, False)\n        self.plot.hideButtons()\n        for axis_name in (\"bottom\", \"left\"):\n            axis = self.plot.getAxis(axis_name)\n            # Remove the condition (that is, allow setting this for bottom\n            # axis) when pyqtgraph is fixed\n            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930\n            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932\n            if axis_name != \"bottom\":  # remove if when pyqtgraph is fixed\n                axis.setStyle(stopAxisAtTick=(True, True))\n\n        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)\n        self.plotview.setCentralItem(self.plot)\n\n        self.mainArea.layout().addWidget(self.plotview)\n        self._set_explanation()\n\n        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)\n\n    @Inputs.evaluation_results\n    def set_results(self, results):\n        self.closeContext()\n        self.clear()\n        self.Error.clear()\n        self.Information.clear()\n\n        self.results = None\n        if results is not None:\n            if not results.domain.has_discrete_class:\n                self.Error.non_discrete_target()\n            elif not results.actual.size:\n                self.Error.empty_input()\n            elif np.any(np.isnan(results.actual)):\n                self.Error.nan_classes()\n            else:\n                self.results = results\n                self._initialize(results)\n                class_var = self.results.domain.class_var\n                self.target_index = int(len(class_var.values) == 2)\n                self.openContext(class_var, self.classifier_names)\n                self._replot()\n\n        self.commit.now()\n\n    def clear(self):\n        self.plot.clear()\n        self.results = None\n        self.classifier_names = []\n        self.selected_classifiers = []\n        self.target_cb.clear()\n        self.colors = []\n\n    def target_index_changed(self):\n        if len(self.results.domain.class_var.values) == 2:\n            self.threshold = 1 - self.threshold\n        self._set_explanation()\n        self._replot()\n        self.commit.deferred()\n\n    def score_changed(self):\n        self._set_explanation()\n        self._replot()\n        if self._last_score_value != self.score:\n            self.commit.deferred()\n            self._last_score_value = self.score\n\n    def _set_explanation(self):\n        explanation = Metrics[self.score].explanation\n        if explanation:\n            self.explanation.setText(explanation)\n            self.explanation.show()\n        else:\n            self.explanation.hide()\n\n        if self.score == 0:\n            self.controls.output_calibration.show()\n            self.info_box.hide()\n        else:\n            self.controls.output_calibration.hide()\n            self.info_box.show()\n\n        axis = self.plot.getAxis(\"bottom\")\n        axis.setLabel(\"Predicted probability\" if self.score == 0\n                      else \"Threshold probability to classify as positive\")\n\n        axis = self.plot.getAxis(\"left\")\n        axis.setLabel(Metrics[self.score].name)\n\n    def _initialize(self, results):\n        n = len(results.predicted)\n        names = getattr(results, \"learner_names\", None)\n        if names is None:\n            names = [\"#{}\".format(i + 1) for i in range(n)]\n\n        self.classifier_names = names\n        self.colors = colorpalettes.get_default_curve_colors(n)\n\n        for i in range(n):\n            item = self.classifiers_list_box.item(i)\n            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))\n\n        self.selected_classifiers = list(range(n))\n        self.target_cb.addItems(results.domain.class_var.values)\n        self.target_index = 0\n\n    def _rug(self, data, pen_args):\n        color = pen_args[\"pen\"].color()\n        rh = 0.025\n        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]\n        rug_x_true = rug_x[data.ytrue].ravel()\n        rug_x_false = rug_x[~data.ytrue].ravel()\n\n        rug_y_true = np.ones_like(rug_x_true)\n        rug_y_true[1::2] = 1 - rh\n        rug_y_false = np.zeros_like(rug_x_false)\n        rug_y_false[1::2] = rh\n\n        self.plot.plot(\n            rug_x_false, rug_y_false,\n            pen=color, connect=\"pairs\", antialias=True)\n        self.plot.plot(\n            rug_x_true, rug_y_true,\n            pen=color, connect=\"pairs\", antialias=True)\n\n    def plot_metrics(self, data, metrics, pen_args):\n        if metrics is None:\n            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)\n        ys = [metric(data) for metric in metrics]\n        for y in ys:\n            self.plot.plot(data.probs, y, **pen_args)\n        return data.probs, ys\n\n    def _prob_curve(self, ytrue, probs, pen_args):\n        xmin, xmax = probs.min(), probs.max()\n        x = np.linspace(xmin, xmax, 100)\n        if xmax != xmin:\n            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))\n            y = f(x)\n        else:\n            y = np.full(100, xmax)\n\n        self.plot.plot(x, y, symbol=\"+\", symbolSize=4, **pen_args)\n        return x, (y, )\n\n    def _setup_plot(self):\n        target = self.target_index\n        results = self.results\n        metrics = Metrics[self.score].functions\n        plot_folds = self.fold_curves and results.folds is not None\n        self.scores = []\n\n        if not self._check_class_presence(results.actual == target):\n            return\n\n        self.Warning.omitted_folds.clear()\n        self.Warning.omitted_nan_prob_points.clear()\n        no_valid_models = []\n        shadow_width = 4 + 4 * plot_folds\n        for clsf in self.selected_classifiers:\n            data = Curves.from_results(results, target, clsf)\n            if data.tot == 0:  # all probabilities are nan\n                no_valid_models.append(clsf)\n                continue\n            if data.tot != results.probabilities.shape[1]:  # some are nan\n                self.Warning.omitted_nan_prob_points()\n\n            color = self.colors[clsf]\n            pen_args = dict(\n                pen=pg.mkPen(color, width=1), antiAlias=True,\n                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))\n            self.scores.append(\n                (self.classifier_names[clsf],\n                 self.plot_metrics(data, metrics, pen_args)))\n\n            if self.display_rug:\n                self._rug(data, pen_args)\n\n            if plot_folds:\n                pen_args = dict(\n                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),\n                    antiAlias=True)\n                for fold in range(len(results.folds)):\n                    fold_results = results.get_fold(fold)\n                    fold_curve = Curves.from_results(fold_results, target, clsf)\n                    # Can't check this before: p and n can be 0 because of\n                    # nan probabilities\n                    if fold_curve.p * fold_curve.n == 0:\n                        self.Warning.omitted_folds()\n                    self.plot_metrics(fold_curve, metrics, pen_args)\n\n        if no_valid_models:\n            self.Warning.no_valid_data(\n                \", \".join(self.classifier_names[i] for i in no_valid_models))\n\n        if self.score == 0:\n            self.plot.plot([0, 1], [0, 1], antialias=True)\n        else:\n            self.line = pg.InfiniteLine(\n                pos=self.threshold, movable=True,\n                pen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=2),\n                hoverPen=pg.mkPen(color=\"k\", style=Qt.DashLine, width=3),\n                bounds=(0, 1),\n            )\n            self.line.sigPositionChanged.connect(self.threshold_change)\n            self.line.sigPositionChangeFinished.connect(\n                self.threshold_change_done)\n            self.plot.addItem(self.line)\n\n    def _check_class_presence(self, ytrue):\n        self.Error.all_target_class.clear()\n        self.Error.no_target_class.clear()\n        if np.max(ytrue) == 0:\n            self.Error.no_target_class()\n            return False\n        if np.min(ytrue) == 1:\n            self.Error.all_target_class()\n            return False\n        return True\n\n    def _replot(self):\n        self.plot.clear()\n        if self.results is not None:\n            self._setup_plot()\n        self._update_info()\n\n    def _on_display_rug_changed(self):\n        self._replot()\n\n    def _on_selection_changed(self):\n        self._replot()\n        self.commit.deferred()\n\n    def threshold_change(self):\n        self.threshold = round(self.line.pos().x(), 2)\n        self.line.setPos(self.threshold)\n        self._update_info()\n\n    def get_info_text(self, short):\n        if short:\n            def elided(s):\n                return s[:17] + \"...\" if len(s) > 20 else s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold: p=</th>\n                                <td colspan='4'>{self.threshold:.2f}<br/></td>\n                            </tr>\"\"\"\n\n        else:\n            def elided(s):\n                return s\n\n            text = f\"\"\"<table>\n                            <tr>\n                                <th align='right'>Threshold:</th>\n                                <td colspan='4'>p = {self.threshold:.2f}<br/>\n                                </td>\n                                <tr/>\n                            </tr>\"\"\"\n\n        if self.scores is not None:\n            short_names = Metrics[self.score].short_names\n            if short_names:\n                text += f\"\"\"<tr>\n                                <th></th>\n                                {\"<td></td>\".join(f\"<td align='right'>{n}</td>\"\n                                                  for n in short_names)}\n                            </tr>\"\"\"\n            for name, (probs, curves) in self.scores:\n                ind = min(np.searchsorted(probs, self.threshold),\n                          len(probs) - 1)\n                text += f\"<tr><th align='right'>{elided(name)}:</th>\"\n                text += \"<td>/</td>\".join(f'<td>{curve[ind]:.3f}</td>'\n                                          for curve in curves)\n                text += \"</tr>\"\n            text += \"<table>\"\n            return text\n        return None\n\n    def _update_info(self):\n        self.info_label.setText(self.get_info_text(short=True))\n\n    def threshold_change_done(self):\n        self.commit.deferred()\n\n    @gui.deferred\n    def commit(self):\n        self.Information.no_output.clear()\n        wrapped = None\n        results = self.results\n        if results is not None:\n            problems = [\n                msg for condition, msg in (\n                    (len(results.folds) > 1,\n                     \"each training data sample produces a different model\"),\n                    (results.models is None,\n                     \"test results do not contain stored models - try testing \"\n                     \"on separate data or on training data\"),\n                    (len(self.selected_classifiers) != 1,\n                     \"select a single model - the widget can output only one\"),\n                    (self.score != 0 and len(results.domain.class_var.values) != 2,\n                     \"cannot calibrate non-binary classes\"))\n                if condition]\n            if len(problems) == 1:\n                self.Information.no_output(problems[0])\n            elif problems:\n                self.Information.no_output(\n                    \"\".join(f\"\\n - {problem}\" for problem in problems))\n            else:\n                clsf_idx = self.selected_classifiers[0]\n                model = results.models[0, clsf_idx]\n                if self.score == 0:\n                    cal_learner = CalibratedLearner(\n                        None, self.output_calibration)\n                    wrapped = cal_learner.get_model(\n                        model, results.actual, results.probabilities[clsf_idx])\n                else:\n                    threshold = [1 - self.threshold,\n                                 self.threshold][self.target_index]\n                    wrapped = ThresholdClassifier(model, threshold)\n\n        self.Outputs.calibrated_model.send(wrapped)\n\n    def send_report(self):\n        if self.results is None:\n            return\n        self.report_items((\n            (\"Target class\", self.target_cb.currentText()),\n            (\"Output model calibration\",\n             self.score == 0\n             and (\"Sigmoid calibration\",\n                  \"Isotonic calibration\")[self.output_calibration])\n        ))\n        caption = report.list_legend(self.classifiers_list_box,\n                                     self.selected_classifiers)\n        self.report_plot()\n        self.report_caption(caption)\n        self.report_caption(self.controls.score.currentText())\n\n        if self.score != 0:\n            self.report_raw(self.get_info_text(short=False))\n\n    def set_visual_settings(self, key, value):\n        self.plot.parameter_setter.set_parameter(key, value)\n        self.visual_settings[key] = value\n\n\ndef gaussian_smoother(x, y, sigma=1.0):\n    x = np.asarray(x)\n    y = np.asarray(y)\n\n    gamma = 1. / (2 * sigma ** 2)\n    a = 1. / (sigma * np.sqrt(2 * np.pi))\n\n    if x.shape != y.shape:\n        raise ValueError\n\n    def smoother(xs):\n        W = a * np.exp(-gamma * ((xs - x) ** 2))\n        return np.average(y, weights=W)\n\n    return np.vectorize(smoother, otypes=[float])\n\n\nif __name__ == \"__main__\":  # pragma: no cover\n    WidgetPreview(OWCalibrationPlot).run(results_for_preview())\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Calibration Plot crashes when Results have no folds

**Description:**
When the Calibration Plot widget processes evaluation results that do not include any folds (i.e., `results.folds` is `None`), it crashes with a `TypeError`. This happens because the widget attempts to determine the number of folds using `len(results.folds)` without first verifying if `folds` is `None`.

**Example Buggy Code:**
```python
results = EvaluationResults(folds=None, models=...)
widget = OWCalibrationPlot()
widget.set_results(results)
widget.commit.now()
```

**Expected Behavior:**
The Calibration Plot should handle cases where `results.folds` is `None` gracefully. It should either skip fold-related processing or notify the user that no fold data is available, avoiding any crashes.

**Actual Behavior:**
A `TypeError` is raised with the message: "object of type 'NoneType' has no len()", causing the Calibration Plot widget to crash.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### Orange/widgets/evaluate/owcalibrationplot.py
from collections import namedtuple

import numpy as np

from AnyQt.QtCore import Qt, QSize
from AnyQt.QtWidgets import QListWidget, QSizePolicy

import pyqtgraph as pg

from orangewidget.utils.visual_settings_dlg import VisualSettingsDialog

from Orange.base import Model
from Orange.classification import ThresholdClassifier, CalibratedLearner
from Orange.evaluation import Results
from Orange.evaluation.performance_curves import Curves
from Orange.widgets import widget, gui, settings
from Orange.widgets.evaluate.contexthandlers import \
    EvaluationResultsContextHandler
from Orange.widgets.evaluate.utils import results_for_preview
from Orange.widgets.utils import colorpalettes
from Orange.widgets.utils.widgetpreview import WidgetPreview
from Orange.widgets.visualize.utils.customizableplot import \
    CommonParameterSetter
from Orange.widgets.visualize.utils.plotutils import GraphicsView, PlotItem
from Orange.widgets.widget import Input, Output, Msg
from Orange.widgets import report


MetricDefinition = namedtuple(
    "metric_definition",
    ("name", "functions", "short_names", "explanation"))

Metrics = [MetricDefinition(*args) for args in (
    ("Calibration curve", None, (), ""),
    ("Classification accuracy", (Curves.ca, ), (), ""),
    ("F1", (Curves.f1, ), (), ""),
    ("Sensitivity and specificity",
     (Curves.sensitivity, Curves.specificity),
     ("sens", "spec"),
     "<p><b>Sensitivity</b> (falling) is the proportion of correctly "
     "detected positive instances (TP&nbsp;/&nbsp;P).</p>"
     "<p><b>Specificity</b> (rising) is the proportion of detected "
     "negative instances (TN&nbsp;/&nbsp;N).</p>"),
    ("Precision and recall",
     (Curves.precision, Curves.recall),
     ("prec", "recall"),
     "<p><b>Precision</b> (rising) is the fraction of retrieved instances "
     "that are relevant, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>"
     "<p><b>Recall</b> (falling) is the proportion of discovered relevant "
     "instances, TP&nbsp;/&nbsp;P.</p>"),
    ("Pos and neg predictive value",
     (Curves.ppv, Curves.npv),
     ("PPV", "TPV"),
     "<p><b>Positive predictive value</b> (rising) is the proportion of "
     "correct positives, TP&nbsp;/&nbsp;(TP&nbsp;+&nbsp;FP).</p>"
     "<p><b>Negative predictive value</b> is the proportion of correct "
     "negatives, TN&nbsp;/&nbsp;(TN&nbsp;+&nbsp;FN).</p>"),
    ("True and false positive rate",
     (Curves.tpr, Curves.fpr),
     ("TPR", "FPR"),
     "<p><b>True and false positive rate</b> are proportions of detected "
     "and omitted positive instances</p>"),
)]


class ParameterSetter(CommonParameterSetter):

    def __init__(self, master):
        super().__init__()
        self.master = master

    def update_setters(self):
        self.initial_settings = {
            self.LABELS_BOX: {
                self.FONT_FAMILY_LABEL: self.FONT_FAMILY_SETTING,
                self.TITLE_LABEL: self.FONT_SETTING,
                self.AXIS_TITLE_LABEL: self.FONT_SETTING,
                self.AXIS_TICKS_LABEL: self.FONT_SETTING,
            },
            self.ANNOT_BOX: {
                self.TITLE_LABEL: {self.TITLE_LABEL: ("", "")},
            }
        }

    @property
    def title_item(self):
        return self.master.titleLabel

    @property
    def axis_items(self):
        return [value["item"] for value in self.master.axes.values()]


class OWCalibrationPlot(widget.OWWidget):
    name = "Calibration Plot"
    description = "Calibration plot based on evaluation of classifiers."
    icon = "icons/CalibrationPlot.svg"
    priority = 1030
    keywords = []

    class Inputs:
        evaluation_results = Input("Evaluation Results", Results)

    class Outputs:
        calibrated_model = Output("Calibrated Model", Model)

    class Error(widget.OWWidget.Error):
        non_discrete_target = Msg("Calibration plot requires a categorical "
                                  "target variable.")
        empty_input = widget.Msg("Empty result on input. Nothing to display.")
        nan_classes = \
            widget.Msg("Remove test data instances with unknown classes.")
        all_target_class = widget.Msg(
            "All data instances belong to target class.")
        no_target_class = widget.Msg(
            "No data instances belong to target class.")

    class Warning(widget.OWWidget.Warning):
        omitted_folds = widget.Msg(
            "Test folds where all data belongs to (non)-target are not shown.")
        omitted_nan_prob_points = widget.Msg(
            "Instance for which the model couldn't compute probabilities are"
            "skipped.")
        no_valid_data = widget.Msg("No valid data for model(s) {}")

    class Information(widget.OWWidget.Information):
        no_output = Msg("Can't output a model: {}")

    settingsHandler = EvaluationResultsContextHandler()
    target_index = settings.ContextSetting(0)
    selected_classifiers = settings.ContextSetting([])
    score = settings.Setting(0)
    output_calibration = settings.Setting(0)
    fold_curves = settings.Setting(False)
    display_rug = settings.Setting(True)
    threshold = settings.Setting(0.5)
    visual_settings = settings.Setting({}, schema_only=True)
    auto_commit = settings.Setting(True)

    graph_name = "plot"

    def __init__(self):
        super().__init__()

        self.results = None
        self.scores = None
        self.classifier_names = []
        self.colors = []
        self.line = None

        self._last_score_value = -1

        box = gui.vBox(self.controlArea, box="Settings")
        self.target_cb = gui.comboBox(
            box, self, "target_index", label="Target:",
            orientation=Qt.Horizontal, callback=self.target_index_changed,
            contentsLength=8, searchable=True)
        gui.checkBox(
            box, self, "display_rug", "Show rug",
            callback=self._on_display_rug_changed)
        gui.checkBox(
            box, self, "fold_curves", "Curves for individual folds",
            callback=self._replot)

        self.classifiers_list_box = gui.listBox(
            self.controlArea, self, "selected_classifiers", "classifier_names",
            box="Classifier", selectionMode=QListWidget.ExtendedSelection,
            sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),
            sizeHint=QSize(150, 40),
            callback=self._on_selection_changed)

        box = gui.vBox(self.controlArea, "Metrics")
        combo = gui.comboBox(
            box, self, "score", items=(metric.name for metric in Metrics),
            callback=self.score_changed)

        self.explanation = gui.widgetLabel(
            box, wordWrap=True, fixedWidth=combo.sizeHint().width())
        self.explanation.setContentsMargins(8, 8, 0, 0)
        font = self.explanation.font()
        font.setPointSizeF(0.85 * font.pointSizeF())
        self.explanation.setFont(font)

        gui.radioButtons(
            box, self, value="output_calibration",
            btnLabels=("Sigmoid calibration", "Isotonic calibration"),
            label="Output model calibration", callback=self.commit.deferred)

        self.info_box = gui.widgetBox(self.controlArea, "Info")
        self.info_label = gui.widgetLabel(self.info_box)

        gui.auto_apply(self.buttonsArea, self, "auto_commit")

        self.plotview = GraphicsView()
        self.plot = PlotItem(enableMenu=False)
        self.plot.parameter_setter = ParameterSetter(self.plot)
        self.plot.setMouseEnabled(False, False)
        self.plot.hideButtons()
        for axis_name in ("bottom", "left"):
            axis = self.plot.getAxis(axis_name)
            # Remove the condition (that is, allow setting this for bottom
            # axis) when pyqtgraph is fixed
            # Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930
            # Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932
            if axis_name != "bottom":  # remove if when pyqtgraph is fixed
                axis.setStyle(stopAxisAtTick=(True, True))

        self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)
        self.plotview.setCentralItem(self.plot)

        self.mainArea.layout().addWidget(self.plotview)
        self._set_explanation()

        VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)

    @Inputs.evaluation_results
    def set_results(self, results):
        self.closeContext()
        self.clear()
        self.Error.clear()
        self.Information.clear()

        self.results = None
        if results is not None:
            if not results.domain.has_discrete_class:
                self.Error.non_discrete_target()
            elif not results.actual.size:
                self.Error.empty_input()
            elif np.any(np.isnan(results.actual)):
                self.Error.nan_classes()
            else:
                self.results = results
                self._initialize(results)
                class_var = self.results.domain.class_var
                self.target_index = int(len(class_var.values) == 2)
                self.openContext(class_var, self.classifier_names)
                self._replot()

        self.commit.now()

    def clear(self):
        self.plot.clear()
        self.results = None
        self.classifier_names = []
        self.selected_classifiers = []
        self.target_cb.clear()
        self.colors = []

    def target_index_changed(self):
        if len(self.results.domain.class_var.values) == 2:
            self.threshold = 1 - self.threshold
        self._set_explanation()
        self._replot()
        self.commit.deferred()

    def score_changed(self):
        self._set_explanation()
        self._replot()
        if self._last_score_value != self.score:
            self.commit.deferred()
            self._last_score_value = self.score

    def _set_explanation(self):
        explanation = Metrics[self.score].explanation
        if explanation:
            self.explanation.setText(explanation)
            self.explanation.show()
        else:
            self.explanation.hide()

        if self.score == 0:
            self.controls.output_calibration.show()
            self.info_box.hide()
        else:
            self.controls.output_calibration.hide()
            self.info_box.show()

        axis = self.plot.getAxis("bottom")
        axis.setLabel("Predicted probability" if self.score == 0
                      else "Threshold probability to classify as positive")

        axis = self.plot.getAxis("left")
        axis.setLabel(Metrics[self.score].name)

    def _initialize(self, results):
        n = len(results.predicted)
        names = getattr(results, "learner_names", None)
        if names is None:
            names = ["#{}".format(i + 1) for i in range(n)]

        self.classifier_names = names
        self.colors = colorpalettes.get_default_curve_colors(n)

        for i in range(n):
            item = self.classifiers_list_box.item(i)
            item.setIcon(colorpalettes.ColorIcon(self.colors[i]))

        self.selected_classifiers = list(range(n))
        self.target_cb.addItems(results.domain.class_var.values)
        self.target_index = 0

    def _rug(self, data, pen_args):
        color = pen_args["pen"].color()
        rh = 0.025
        rug_x = np.c_[data.probs[:-1], data.probs[:-1]]
        rug_x_true = rug_x[data.ytrue].ravel()
        rug_x_false = rug_x[~data.ytrue].ravel()

        rug_y_true = np.ones_like(rug_x_true)
        rug_y_true[1::2] = 1 - rh
        rug_y_false = np.zeros_like(rug_x_false)
        rug_y_false[1::2] = rh

        self.plot.plot(
            rug_x_false, rug_y_false,
            pen=color, connect="pairs", antialias=True)
        self.plot.plot(
            rug_x_true, rug_y_true,
            pen=color, connect="pairs", antialias=True)

    def plot_metrics(self, data, metrics, pen_args):
        if metrics is None:
            return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)
        ys = [metric(data) for metric in metrics]
        for y in ys:
            self.plot.plot(data.probs, y, **pen_args)
        return data.probs, ys

    def _prob_curve(self, ytrue, probs, pen_args):
        xmin, xmax = probs.min(), probs.max()
        x = np.linspace(xmin, xmax, 100)
        if xmax != xmin:
            f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))
            y = f(x)
        else:
            y = np.full(100, xmax)

        self.plot.plot(x, y, symbol="+", symbolSize=4, **pen_args)
        return x, (y, )

    def _setup_plot(self):
        target = self.target_index
        results = self.results
        metrics = Metrics[self.score].functions
        plot_folds = self.fold_curves and results.folds is not None
        self.scores = []

        if not self._check_class_presence(results.actual == target):
            return

        self.Warning.omitted_folds.clear()
        self.Warning.omitted_nan_prob_points.clear()
        no_valid_models = []
        shadow_width = 4 + 4 * plot_folds
        for clsf in self.selected_classifiers:
            data = Curves.from_results(results, target, clsf)
            if data.tot == 0:  # all probabilities are nan
                no_valid_models.append(clsf)
                continue
            if data.tot != results.probabilities.shape[1]:  # some are nan
                self.Warning.omitted_nan_prob_points()

            color = self.colors[clsf]
            pen_args = dict(
                pen=pg.mkPen(color, width=1), antiAlias=True,
                shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))
            self.scores.append(
                (self.classifier_names[clsf],
                 self.plot_metrics(data, metrics, pen_args)))

            if self.display_rug:
                self._rug(data, pen_args)

            if plot_folds:
                pen_args = dict(
                    pen=pg.mkPen(color, width=1, style=Qt.DashLine),
                    antiAlias=True)
                for fold in range(len(results.folds)):
                    fold_results = results.get_fold(fold)
                    fold_curve = Curves.from_results(fold_results, target, clsf)
                    # Can't check this before: p and n can be 0 because of
                    # nan probabilities
                    if fold_curve.p * fold_curve.n == 0:
                        self.Warning.omitted_folds()
                    self.plot_metrics(fold_curve, metrics, pen_args)

        if no_valid_models:
            self.Warning.no_valid_data(
                ", ".join(self.classifier_names[i] for i in no_valid_models))

        if self.score == 0:
            self.plot.plot([0, 1], [0, 1], antialias=True)
        else:
            self.line = pg.InfiniteLine(
                pos=self.threshold, movable=True,
                pen=pg.mkPen(color="k", style=Qt.DashLine, width=2),
                hoverPen=pg.mkPen(color="k", style=Qt.DashLine, width=3),
                bounds=(0, 1),
            )
            self.line.sigPositionChanged.connect(self.threshold_change)
            self.line.sigPositionChangeFinished.connect(
                self.threshold_change_done)
            self.plot.addItem(self.line)

    def _check_class_presence(self, ytrue):
        self.Error.all_target_class.clear()
        self.Error.no_target_class.clear()
        if np.max(ytrue) == 0:
            self.Error.no_target_class()
            return False
        if np.min(ytrue) == 1:
            self.Error.all_target_class()
            return False
        return True

    def _replot(self):
        self.plot.clear()
        if self.results is not None:
            self._setup_plot()
        self._update_info()

    def _on_display_rug_changed(self):
        self._replot()

    def _on_selection_changed(self):
        self._replot()
        self.commit.deferred()

    def threshold_change(self):
        self.threshold = round(self.line.pos().x(), 2)
        self.line.setPos(self.threshold)
        self._update_info()

    def get_info_text(self, short):
        if short:
            def elided(s):
                return s[:17] + "..." if len(s) > 20 else s

            text = f"""<table>
                            <tr>
                                <th align='right'>Threshold: p=</th>
                                <td colspan='4'>{self.threshold:.2f}<br/></td>
                            </tr>"""

        else:
            def elided(s):
                return s

            text = f"""<table>
                            <tr>
                                <th align='right'>Threshold:</th>
                                <td colspan='4'>p = {self.threshold:.2f}<br/>
                                </td>
                                <tr/>
                            </tr>"""

        if self.scores is not None:
            short_names = Metrics[self.score].short_names
            if short_names:
                text += f"""<tr>
                                <th></th>
                                {"<td></td>".join(f"<td align='right'>{n}</td>"
                                                  for n in short_names)}
                            </tr>"""
            for name, (probs, curves) in self.scores:
                ind = min(np.searchsorted(probs, self.threshold),
                          len(probs) - 1)
                text += f"<tr><th align='right'>{elided(name)}:</th>"
                text += "<td>/</td>".join(f'<td>{curve[ind]:.3f}</td>'
                                          for curve in curves)
                text += "</tr>"
            text += "<table>"
            return text
        return None

    def _update_info(self):
        self.info_label.setText(self.get_info_text(short=True))

    def threshold_change_done(self):
        self.commit.deferred()

    @gui.deferred
    def commit(self):
        self.Information.no_output.clear()
        wrapped = None
        results = self.results
        if results is not None:
            problems = [
                msg for condition, msg in (
                    (len(results.folds) > 1,
                     "each training data sample produces a different model"),
                    (results.models is None,
                     "test results do not contain stored models - try testing "
                     "on separate data or on training data"),
                    (len(self.selected_classifiers) != 1,
                     "select a single model - the widget can output only one"),
                    (self.score != 0 and len(results.domain.class_var.values) != 2,
                     "cannot calibrate non-binary classes"))
                if condition]
            if len(problems) == 1:
                self.Information.no_output(problems[0])
            elif problems:
                self.Information.no_output(
                    "".join(f"\n - {problem}" for problem in problems))
            else:
                clsf_idx = self.selected_classifiers[0]
                model = results.models[0, clsf_idx]
                if self.score == 0:
                    cal_learner = CalibratedLearner(
                        None, self.output_calibration)
                    wrapped = cal_learner.get_model(
                        model, results.actual, results.probabilities[clsf_idx])
                else:
                    threshold = [1 - self.threshold,
                                 self.threshold][self.target_index]
                    wrapped = ThresholdClassifier(model, threshold)

        self.Outputs.calibrated_model.send(wrapped)

    def send_report(self):
        if self.results is None:
            return
        self.report_items((
            ("Target class", self.target_cb.currentText()),
            ("Output model calibration",
             self.score == 0
             and ("Sigmoid calibration",
                  "Isotonic calibration")[self.output_calibration])
        ))
        caption = report.list_legend(self.classifiers_list_box,
                                     self.selected_classifiers)
        self.report_plot()
        self.report_caption(caption)
        self.report_caption(self.controls.score.currentText())

        if self.score != 0:
            self.report_raw(self.get_info_text(short=False))

    def set_visual_settings(self, key, value):
        self.plot.parameter_setter.set_parameter(key, value)
        self.visual_settings[key] = value


def gaussian_smoother(x, y, sigma=1.0):
    x = np.asarray(x)
    y = np.asarray(y)

    gamma = 1. / (2 * sigma ** 2)
    a = 1. / (sigma * np.sqrt(2 * np.pi))

    if x.shape != y.shape:
        raise ValueError

    def smoother(xs):
        W = a * np.exp(-gamma * ((xs - x) ** 2))
        return np.average(y, weights=W)

    return np.vectorize(smoother, otypes=[float])


if __name__ == "__main__":  # pragma: no cover
    WidgetPreview(OWCalibrationPlot).run(results_for_preview())

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user