{
    "file_diffs": [
        {
            "old_file_content": "# pylint doesn't understand the Settings magic\n# pylint: disable=invalid-sequence-index\n# pylint: disable=too-many-lines,too-many-instance-attributes\nimport abc\nimport enum\nimport logging\nimport traceback\nimport copy\nfrom functools import partial, reduce\n\nfrom concurrent.futures import Future\nfrom collections import OrderedDict, namedtuple\nfrom itertools import count\nfrom typing import Any, Optional, List, Dict, Callable\n\nimport numpy as np\nimport baycomp\n\nfrom AnyQt import QtGui\nfrom AnyQt.QtCore import Qt, QSize, QThread\nfrom AnyQt.QtCore import pyqtSlot as Slot\nfrom AnyQt.QtGui import QStandardItem, QDoubleValidator\nfrom AnyQt.QtWidgets import QHeaderView, QTableWidget, QLabel, QApplication\n\nfrom Orange.base import Learner\nimport Orange.classification\nfrom Orange.data import Table, DiscreteVariable\nfrom Orange.data.table import DomainTransformationError\nfrom Orange.data.filter import HasClass\nfrom Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT\nimport Orange.evaluation\nfrom Orange.evaluation import Results\nfrom Orange.preprocess.preprocess import Preprocess\nimport Orange.regression\nfrom Orange.widgets import gui, settings, widget\nfrom Orange.widgets.evaluate.utils import \\\n    usable_scorers, ScoreTable, learner_name, scorer_caller\nfrom Orange.widgets.utils.itemmodels import DomainModel\nfrom Orange.widgets.utils.widgetpreview import WidgetPreview\nfrom Orange.widgets.utils.concurrent import ThreadExecutor, TaskState\nfrom Orange.widgets.utils.state_summary import (format_multiple_summaries,\n                                                format_summary_details)\nfrom Orange.widgets.widget import OWWidget, Msg, Input, Output\n\nlog = logging.getLogger(__name__)\n\nInputLearner = namedtuple(\n    \"InputLearner\",\n    [\"learner\",  # :: Orange.base.Learner\n     \"results\",  # :: Option[Try[Orange.evaluation.Results]]\n     \"stats\"]    # :: Option[Sequence[Try[float]]]\n)\n\n\nclass Try(abc.ABC):\n    \"\"\"Try to walk in a Turing tar pit.\"\"\"\n\n    class Success:\n        \"\"\"Data type for instance constructed on success\"\"\"\n        __slots__ = (\"__value\",)\n#         __bool__ = lambda self: True\n        success = property(lambda self: True)\n        value = property(lambda self: self.__value)\n\n        def __init__(self, value):\n            self.__value = value\n\n        def __getnewargs__(self):\n            return (self.value,)\n\n        def __repr__(self):\n            return \"{}({!r})\".format(self.__class__.__qualname__,\n                                     self.value)\n\n        def map(self, fn):\n            return Try(lambda: fn(self.value))\n\n    class Fail:\n        \"\"\"Data type for instance constructed on fail\"\"\"\n        __slots__ = (\"__exception\", )\n#         __bool__ = lambda self: False\n        success = property(lambda self: False)\n        exception = property(lambda self: self.__exception)\n\n        def __init__(self, exception):\n            self.__exception = exception\n\n        def __getnewargs__(self):\n            return (self.exception, )\n\n        def __repr__(self):\n            return \"{}({!r})\".format(self.__class__.__qualname__,\n                                     self.exception)\n\n        def map(self, _fn):\n            return self\n\n    def __new__(cls, f, *args, **kwargs):\n        try:\n            rval = f(*args, **kwargs)\n        except BaseException as ex:\n            # IMPORANT: Will capture traceback in ex.__traceback__\n            return Try.Fail(ex)\n        else:\n            return Try.Success(rval)\n\nTry.register(Try.Success)\nTry.register(Try.Fail)\n\n\ndef raise_(exc):\n    raise exc\n\nTry.register = lambda cls: raise_(TypeError())\n\n\nclass State(enum.Enum):\n    \"\"\"\n    OWTestAndScore's runtime state.\n    \"\"\"\n    #: No or insufficient input (i.e. no data or no learners)\n    Waiting = \"Waiting\"\n    #: Executing/running the evaluations\n    Running = \"Running\"\n    #: Finished running evaluations (success or error)\n    Done = \"Done\"\n    #: Evaluation cancelled\n    Cancelled = \"Cancelled\"\n\n\nclass OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        train_data_empty = Msg(\"Train dataset is empty.\")\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required = Msg(\"Train data input requires a target variable.\")\n        too_many_classes = Msg(\"Too many target variables.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        no_class_values = Msg(\"Target variable has no values.\")\n        only_one_class_var_value = Msg(\"Target variable has only one value.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_empty.clear()\n        self.Error.class_required.clear()\n        self.Error.too_many_classes.clear()\n        self.Error.no_class_values.clear()\n        self.Error.only_one_class_var_value.clear()\n        if data is not None and not data:\n            self.Error.train_data_empty()\n            data = None\n        if data:\n            conds = [not data.domain.class_vars,\n                     len(data.domain.class_vars) > 1,\n                     np.isnan(data.Y).all(),\n                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]\n            errors = [self.Error.class_required,\n                      self.Error.too_many_classes,\n                      self.Error.no_class_values,\n                      self.Error.only_one_class_var_value]\n            for cond, error in zip(conds, errors):\n                if cond:\n                    error()\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()\n\n\ndef add_task_to_dispose_queue(task: TaskState):\n    # transfer ownership of task to Qt, and delete it after completion\n    # all other signals from task should be disconnected.\n    assert task.parent() is None\n    app = QApplication.instance()\n    task.setParent(app)\n    task.watcher.finished.connect(task.deleteLater)\n\n\nclass UserInterrupt(BaseException):\n    \"\"\"\n    A BaseException subclass used for cooperative task/thread cancellation\n    \"\"\"\n\n\ndef results_add_by_model(x, y):\n    def is_empty(res):\n        return (getattr(res, \"models\", None) is None\n                and getattr(res, \"row_indices\", None) is None)\n\n    if is_empty(x):\n        return y\n    elif is_empty(y):\n        return x\n\n    assert (x.row_indices == y.row_indices).all()\n    assert (x.actual == y.actual).all()\n\n    res = Orange.evaluation.Results()\n    res.data = x.data\n    res.domain = x.domain\n    res.row_indices = x.row_indices\n    res.folds = x.folds\n    res.actual = x.actual\n    res.predicted = np.vstack((x.predicted, y.predicted))\n    if getattr(x, \"probabilities\", None) is not None \\\n            and getattr(y, \"probabilities\") is not None:\n        res.probabilities = np.vstack((x.probabilities, y.probabilities))\n\n    if x.models is not None:\n        res.models = np.hstack((x.models, y.models))\n    return res\n\n\ndef results_merge(results):\n    return reduce(results_add_by_model, results, Orange.evaluation.Results())\n\n\ndef results_one_vs_rest(results, pos_index):\n    from Orange.preprocess.transformation import Indicator\n    actual = results.actual == pos_index\n    predicted = results.predicted == pos_index\n    if results.probabilities is not None:\n        c = results.probabilities.shape[2]\n        assert c >= 2\n        neg_indices = [i for i in range(c) if i != pos_index]\n        pos_prob = results.probabilities[:, :, [pos_index]]\n        neg_prob = np.sum(results.probabilities[:, :, neg_indices],\n                          axis=2, keepdims=True)\n        probabilities = np.dstack((neg_prob, pos_prob))\n    else:\n        probabilities = None\n\n    res = Orange.evaluation.Results()\n    res.actual = actual\n    res.predicted = predicted\n    res.folds = results.folds\n    res.row_indices = results.row_indices\n    res.probabilities = probabilities\n\n    value = results.domain.class_var.values[pos_index]\n    class_var = Orange.data.DiscreteVariable(\n        \"I({}=={})\".format(results.domain.class_var.name, value),\n        values=(\"False\", \"True\"),\n        compute_value=Indicator(results.domain.class_var, pos_index)\n    )\n    domain = Orange.data.Domain(\n        results.domain.attributes,\n        [class_var],\n        results.domain.metas\n    )\n    res.data = None\n    res.domain = domain\n    return res\n\n\nif __name__ == \"__main__\":  # pragma: no cover\n    filename = \"iris\"\n    preview_data = Table(filename)\n    if preview_data.domain.class_var.is_discrete:\n        prev_learners = [lambda data: 1 / 0,\n                         Orange.classification.LogisticRegressionLearner(),\n                         Orange.classification.MajorityLearner(),\n                         Orange.classification.NaiveBayesLearner()]\n    else:\n        prev_learners = [lambda data: 1 / 0,\n                         Orange.regression.MeanLearner(),\n                         Orange.regression.KNNRegressionLearner(),\n                         Orange.regression.RidgeRegressionLearner()]\n\n    WidgetPreview(OWTestAndScore).run(\n        set_train_data=preview_data,\n        set_test_data=preview_data,\n        set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]\n    )\n",
            "new_file_content": "# pylint doesn't understand the Settings magic\n# pylint: disable=invalid-sequence-index\n# pylint: disable=too-many-lines,too-many-instance-attributes\nimport abc\nimport enum\nimport logging\nimport traceback\nimport copy\nfrom functools import partial, reduce\n\nfrom concurrent.futures import Future\nfrom collections import OrderedDict, namedtuple\nfrom itertools import count\nfrom typing import Any, Optional, List, Dict, Callable\n\nimport numpy as np\nimport baycomp\n\nfrom AnyQt import QtGui\nfrom AnyQt.QtCore import Qt, QSize, QThread\nfrom AnyQt.QtCore import pyqtSlot as Slot\nfrom AnyQt.QtGui import QStandardItem, QDoubleValidator\nfrom AnyQt.QtWidgets import QHeaderView, QTableWidget, QLabel, QApplication\n\nfrom Orange.base import Learner\nimport Orange.classification\nfrom Orange.data import Table, DiscreteVariable\nfrom Orange.data.table import DomainTransformationError\nfrom Orange.data.filter import HasClass\nfrom Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT\nimport Orange.evaluation\nfrom Orange.evaluation import Results\nfrom Orange.preprocess.preprocess import Preprocess\nimport Orange.regression\nfrom Orange.statistics.util import unique\nfrom Orange.widgets import gui, settings, widget\nfrom Orange.widgets.evaluate.utils import \\\n    usable_scorers, ScoreTable, learner_name, scorer_caller\nfrom Orange.widgets.utils.itemmodels import DomainModel\nfrom Orange.widgets.utils.widgetpreview import WidgetPreview\nfrom Orange.widgets.utils.concurrent import ThreadExecutor, TaskState\nfrom Orange.widgets.utils.state_summary import (format_multiple_summaries,\n                                                format_summary_details)\nfrom Orange.widgets.widget import OWWidget, Msg, Input, Output\n\nlog = logging.getLogger(__name__)\n\nInputLearner = namedtuple(\n    \"InputLearner\",\n    [\"learner\",  # :: Orange.base.Learner\n     \"results\",  # :: Option[Try[Orange.evaluation.Results]]\n     \"stats\"]    # :: Option[Sequence[Try[float]]]\n)\n\n\nclass Try(abc.ABC):\n    \"\"\"Try to walk in a Turing tar pit.\"\"\"\n\n    class Success:\n        \"\"\"Data type for instance constructed on success\"\"\"\n        __slots__ = (\"__value\",)\n#         __bool__ = lambda self: True\n        success = property(lambda self: True)\n        value = property(lambda self: self.__value)\n\n        def __init__(self, value):\n            self.__value = value\n\n        def __getnewargs__(self):\n            return (self.value,)\n\n        def __repr__(self):\n            return \"{}({!r})\".format(self.__class__.__qualname__,\n                                     self.value)\n\n        def map(self, fn):\n            return Try(lambda: fn(self.value))\n\n    class Fail:\n        \"\"\"Data type for instance constructed on fail\"\"\"\n        __slots__ = (\"__exception\", )\n#         __bool__ = lambda self: False\n        success = property(lambda self: False)\n        exception = property(lambda self: self.__exception)\n\n        def __init__(self, exception):\n            self.__exception = exception\n\n        def __getnewargs__(self):\n            return (self.exception, )\n\n        def __repr__(self):\n            return \"{}({!r})\".format(self.__class__.__qualname__,\n                                     self.exception)\n\n        def map(self, _fn):\n            return self\n\n    def __new__(cls, f, *args, **kwargs):\n        try:\n            rval = f(*args, **kwargs)\n        except BaseException as ex:\n            # IMPORANT: Will capture traceback in ex.__traceback__\n            return Try.Fail(ex)\n        else:\n            return Try.Success(rval)\n\nTry.register(Try.Success)\nTry.register(Try.Fail)\n\n\ndef raise_(exc):\n    raise exc\n\nTry.register = lambda cls: raise_(TypeError())\n\n\nclass State(enum.Enum):\n    \"\"\"\n    OWTestAndScore's runtime state.\n    \"\"\"\n    #: No or insufficient input (i.e. no data or no learners)\n    Waiting = \"Waiting\"\n    #: Executing/running the evaluations\n    Running = \"Running\"\n    #: Finished running evaluations (success or error)\n    Done = \"Done\"\n    #: Evaluation cancelled\n    Cancelled = \"Cancelled\"\n\n\nclass OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n        train_data_error = Msg(\"{}\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_error.clear()\n\n        if data is not None:\n            data_errors = [\n                (\"Train dataset is empty.\", len(data) == 0),\n                (\n                    \"Train data input requires a target variable.\",\n                    not data.domain.class_vars\n                ),\n                (\"Too many target variables.\", len(data.domain.class_vars) > 1),\n                (\"Target variable has no values.\", np.isnan(data.Y).all()),\n                (\n                    \"Target variable has only one value.\",\n                    data.domain.has_discrete_class and len(unique(data.Y)) < 2\n                ),\n                (\"Data has no features to learn from.\", data.X.shape[1] == 0),\n            ]\n\n            for error_msg, cond in data_errors:\n                if cond:\n                    self.Error.train_data_error(error_msg)\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()\n\n\ndef add_task_to_dispose_queue(task: TaskState):\n    # transfer ownership of task to Qt, and delete it after completion\n    # all other signals from task should be disconnected.\n    assert task.parent() is None\n    app = QApplication.instance()\n    task.setParent(app)\n    task.watcher.finished.connect(task.deleteLater)\n\n\nclass UserInterrupt(BaseException):\n    \"\"\"\n    A BaseException subclass used for cooperative task/thread cancellation\n    \"\"\"\n\n\ndef results_add_by_model(x, y):\n    def is_empty(res):\n        return (getattr(res, \"models\", None) is None\n                and getattr(res, \"row_indices\", None) is None)\n\n    if is_empty(x):\n        return y\n    elif is_empty(y):\n        return x\n\n    assert (x.row_indices == y.row_indices).all()\n    assert (x.actual == y.actual).all()\n\n    res = Orange.evaluation.Results()\n    res.data = x.data\n    res.domain = x.domain\n    res.row_indices = x.row_indices\n    res.folds = x.folds\n    res.actual = x.actual\n    res.predicted = np.vstack((x.predicted, y.predicted))\n    if getattr(x, \"probabilities\", None) is not None \\\n            and getattr(y, \"probabilities\") is not None:\n        res.probabilities = np.vstack((x.probabilities, y.probabilities))\n\n    if x.models is not None:\n        res.models = np.hstack((x.models, y.models))\n    return res\n\n\ndef results_merge(results):\n    return reduce(results_add_by_model, results, Orange.evaluation.Results())\n\n\ndef results_one_vs_rest(results, pos_index):\n    from Orange.preprocess.transformation import Indicator\n    actual = results.actual == pos_index\n    predicted = results.predicted == pos_index\n    if results.probabilities is not None:\n        c = results.probabilities.shape[2]\n        assert c >= 2\n        neg_indices = [i for i in range(c) if i != pos_index]\n        pos_prob = results.probabilities[:, :, [pos_index]]\n        neg_prob = np.sum(results.probabilities[:, :, neg_indices],\n                          axis=2, keepdims=True)\n        probabilities = np.dstack((neg_prob, pos_prob))\n    else:\n        probabilities = None\n\n    res = Orange.evaluation.Results()\n    res.actual = actual\n    res.predicted = predicted\n    res.folds = results.folds\n    res.row_indices = results.row_indices\n    res.probabilities = probabilities\n\n    value = results.domain.class_var.values[pos_index]\n    class_var = Orange.data.DiscreteVariable(\n        \"I({}=={})\".format(results.domain.class_var.name, value),\n        values=(\"False\", \"True\"),\n        compute_value=Indicator(results.domain.class_var, pos_index)\n    )\n    domain = Orange.data.Domain(\n        results.domain.attributes,\n        [class_var],\n        results.domain.metas\n    )\n    res.data = None\n    res.domain = domain\n    return res\n\n\nif __name__ == \"__main__\":  # pragma: no cover\n    filename = \"iris\"\n    preview_data = Table(filename)\n    if preview_data.domain.class_var.is_discrete:\n        prev_learners = [lambda data: 1 / 0,\n                         Orange.classification.LogisticRegressionLearner(),\n                         Orange.classification.MajorityLearner(),\n                         Orange.classification.NaiveBayesLearner()]\n    else:\n        prev_learners = [lambda data: 1 / 0,\n                         Orange.regression.MeanLearner(),\n                         Orange.regression.KNNRegressionLearner(),\n                         Orange.regression.RidgeRegressionLearner()]\n\n    WidgetPreview(OWTestAndScore).run(\n        set_train_data=preview_data,\n        set_test_data=preview_data,\n        set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]\n    )\n",
            "header": {
                "file": {
                    "path": "Orange/widgets/evaluate/owtestandscore.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "03fb23a3d",
                "new_commit_hash": "bffccdb6a",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/Orange/widgets/evaluate/owtestandscore.py"
            },
            "plus_file": {
                "path": "b/Orange/widgets/evaluate/owtestandscore.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 32,
                            "length": 6
                        },
                        "new_range": {
                            "start": 32,
                            "length": 7
                        },
                        "section": "import Orange.evaluation"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from Orange.evaluation import Results",
                                "type": "context"
                            },
                            {
                                "content": "from Orange.preprocess.preprocess import Preprocess",
                                "type": "context"
                            },
                            {
                                "content": "import Orange.regression",
                                "type": "context"
                            },
                            {
                                "content": "from Orange.statistics.util import unique",
                                "type": "added"
                            },
                            {
                                "content": "from Orange.widgets import gui, settings, widget",
                                "type": "context"
                            },
                            {
                                "content": "from Orange.widgets.evaluate.utils import \\",
                                "type": "context"
                            },
                            {
                                "content": "    usable_scorers, ScoreTable, learner_name, scorer_caller",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/owtestandscore.py",
                            "type": "statement",
                            "name": "",
                            "content": "from Orange.statistics.util import unique",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 35,
                            "end_lineno": 35,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 189,
                            "length": 19
                        },
                        "new_range": {
                            "start": 190,
                            "length": 15
                        },
                        "section": "class OWTestAndScore(OWWidget):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    class_selection = settings.ContextSetting(TARGET_AVERAGE)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    class Error(OWWidget.Error):",
                                "type": "context"
                            },
                            {
                                "content": "        train_data_empty = Msg(\"Train dataset is empty.\")",
                                "type": "deleted"
                            },
                            {
                                "content": "        test_data_empty = Msg(\"Test dataset is empty.\")",
                                "type": "context"
                            },
                            {
                                "content": "        class_required = Msg(\"Train data input requires a target variable.\")",
                                "type": "deleted"
                            },
                            {
                                "content": "        too_many_classes = Msg(\"Too many target variables.\")",
                                "type": "deleted"
                            },
                            {
                                "content": "        class_required_test = Msg(\"Test data input requires a target variable.\")",
                                "type": "context"
                            },
                            {
                                "content": "        too_many_folds = Msg(\"Number of folds exceeds the data size\")",
                                "type": "context"
                            },
                            {
                                "content": "        class_inconsistent = Msg(\"Test and train datasets \"",
                                "type": "context"
                            },
                            {
                                "content": "                                 \"have different target variables.\")",
                                "type": "context"
                            },
                            {
                                "content": "        memory_error = Msg(\"Not enough memory.\")",
                                "type": "context"
                            },
                            {
                                "content": "        no_class_values = Msg(\"Target variable has no values.\")",
                                "type": "deleted"
                            },
                            {
                                "content": "        only_one_class_var_value = Msg(\"Target variable has only one value.\")",
                                "type": "deleted"
                            },
                            {
                                "content": "        test_data_incompatible = Msg(",
                                "type": "context"
                            },
                            {
                                "content": "            \"Test data may be incompatible with train data.\")",
                                "type": "context"
                            },
                            {
                                "content": "        train_data_error = Msg(\"{}\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    class Warning(OWWidget.Warning):",
                                "type": "context"
                            },
                            {
                                "content": "        missing_data = \\",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/owtestandscore.py",
                            "type": "class",
                            "name": "OWTestAndScore",
                            "content": "class OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        train_data_empty = Msg(\"Train dataset is empty.\")\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required = Msg(\"Train data input requires a target variable.\")\n        too_many_classes = Msg(\"Too many target variables.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        no_class_values = Msg(\"Target variable has no values.\")\n        only_one_class_var_value = Msg(\"Target variable has only one value.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_empty.clear()\n        self.Error.class_required.clear()\n        self.Error.too_many_classes.clear()\n        self.Error.no_class_values.clear()\n        self.Error.only_one_class_var_value.clear()\n        if data is not None and not data:\n            self.Error.train_data_empty()\n            data = None\n        if data:\n            conds = [not data.domain.class_vars,\n                     len(data.domain.class_vars) > 1,\n                     np.isnan(data.Y).all(),\n                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]\n            errors = [self.Error.class_required,\n                      self.Error.too_many_classes,\n                      self.Error.no_class_values,\n                      self.Error.only_one_class_var_value]\n            for cond, error in zip(conds, errors):\n                if cond:\n                    error()\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 131,
                            "end_lineno": 1124,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 387,
                            "length": 26
                        },
                        "new_range": {
                            "start": 384,
                            "length": 27
                        },
                        "section": "class OWTestAndScore(OWWidget):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "        self.cancel()",
                                "type": "context"
                            },
                            {
                                "content": "        self.Information.data_sampled.clear()",
                                "type": "context"
                            },
                            {
                                "content": "        self.Error.train_data_empty.clear()",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.Error.class_required.clear()",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.Error.too_many_classes.clear()",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.Error.no_class_values.clear()",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.Error.only_one_class_var_value.clear()",
                                "type": "deleted"
                            },
                            {
                                "content": "        if data is not None and not data:",
                                "type": "deleted"
                            },
                            {
                                "content": "            self.Error.train_data_empty()",
                                "type": "deleted"
                            },
                            {
                                "content": "            data = None",
                                "type": "deleted"
                            },
                            {
                                "content": "        if data:",
                                "type": "deleted"
                            },
                            {
                                "content": "            conds = [not data.domain.class_vars,",
                                "type": "deleted"
                            },
                            {
                                "content": "                     len(data.domain.class_vars) > 1,",
                                "type": "deleted"
                            },
                            {
                                "content": "                     np.isnan(data.Y).all(),",
                                "type": "deleted"
                            },
                            {
                                "content": "                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]",
                                "type": "deleted"
                            },
                            {
                                "content": "            errors = [self.Error.class_required,",
                                "type": "deleted"
                            },
                            {
                                "content": "                      self.Error.too_many_classes,",
                                "type": "deleted"
                            },
                            {
                                "content": "                      self.Error.no_class_values,",
                                "type": "deleted"
                            },
                            {
                                "content": "                      self.Error.only_one_class_var_value]",
                                "type": "deleted"
                            },
                            {
                                "content": "            for cond, error in zip(conds, errors):",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.Error.train_data_error.clear()",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        if data is not None:",
                                "type": "added"
                            },
                            {
                                "content": "            data_errors = [",
                                "type": "added"
                            },
                            {
                                "content": "                (\"Train dataset is empty.\", len(data) == 0),",
                                "type": "added"
                            },
                            {
                                "content": "                (",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Train data input requires a target variable.\",",
                                "type": "added"
                            },
                            {
                                "content": "                    not data.domain.class_vars",
                                "type": "added"
                            },
                            {
                                "content": "                ),",
                                "type": "added"
                            },
                            {
                                "content": "                (\"Too many target variables.\", len(data.domain.class_vars) > 1),",
                                "type": "added"
                            },
                            {
                                "content": "                (\"Target variable has no values.\", np.isnan(data.Y).all()),",
                                "type": "added"
                            },
                            {
                                "content": "                (",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Target variable has only one value.\",",
                                "type": "added"
                            },
                            {
                                "content": "                    data.domain.has_discrete_class and len(unique(data.Y)) < 2",
                                "type": "added"
                            },
                            {
                                "content": "                ),",
                                "type": "added"
                            },
                            {
                                "content": "                (\"Data has no features to learn from.\", data.X.shape[1] == 0),",
                                "type": "added"
                            },
                            {
                                "content": "            ]",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "            for error_msg, cond in data_errors:",
                                "type": "added"
                            },
                            {
                                "content": "                if cond:",
                                "type": "context"
                            },
                            {
                                "content": "                    error()",
                                "type": "deleted"
                            },
                            {
                                "content": "                    self.Error.train_data_error(error_msg)",
                                "type": "added"
                            },
                            {
                                "content": "                    data = None",
                                "type": "context"
                            },
                            {
                                "content": "                    break",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/owtestandscore.py",
                            "type": "class",
                            "name": "OWTestAndScore",
                            "content": "class OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        train_data_empty = Msg(\"Train dataset is empty.\")\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required = Msg(\"Train data input requires a target variable.\")\n        too_many_classes = Msg(\"Too many target variables.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        no_class_values = Msg(\"Target variable has no values.\")\n        only_one_class_var_value = Msg(\"Target variable has only one value.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_empty.clear()\n        self.Error.class_required.clear()\n        self.Error.too_many_classes.clear()\n        self.Error.no_class_values.clear()\n        self.Error.only_one_class_var_value.clear()\n        if data is not None and not data:\n            self.Error.train_data_empty()\n            data = None\n        if data:\n            conds = [not data.domain.class_vars,\n                     len(data.domain.class_vars) > 1,\n                     np.isnan(data.Y).all(),\n                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]\n            errors = [self.Error.class_required,\n                      self.Error.too_many_classes,\n                      self.Error.no_class_values,\n                      self.Error.only_one_class_var_value]\n            for cond, error in zip(conds, errors):\n                if cond:\n                    error()\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 131,
                            "end_lineno": 1124,
                            "parent": null
                        },
                        {
                            "file_name": "Orange/widgets/evaluate/owtestandscore.py",
                            "type": "method",
                            "name": "OWTestAndScore.set_train_data",
                            "content": "class OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        train_data_empty = Msg(\"Train dataset is empty.\")\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required = Msg(\"Train data input requires a target variable.\")\n        too_many_classes = Msg(\"Too many target variables.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        no_class_values = Msg(\"Target variable has no values.\")\n        only_one_class_var_value = Msg(\"Target variable has only one value.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_empty.clear()\n        self.Error.class_required.clear()\n        self.Error.too_many_classes.clear()\n        self.Error.no_class_values.clear()\n        self.Error.only_one_class_var_value.clear()\n        if data is not None and not data:\n            self.Error.train_data_empty()\n            data = None\n        if data:\n            conds = [not data.domain.class_vars,\n                     len(data.domain.class_vars) > 1,\n                     np.isnan(data.Y).all(),\n                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]\n            errors = [self.Error.class_required,\n                      self.Error.too_many_classes,\n                      self.Error.no_class_values,\n                      self.Error.only_one_class_var_value]\n            for cond, error in zip(conds, errors):\n                if cond:\n                    error()\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 380,
                            "end_lineno": 440,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/owtestandscore.py",
                                "type": "class",
                                "name": "OWTestAndScore",
                                "content": "class OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        train_data_empty = Msg(\"Train dataset is empty.\")\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required = Msg(\"Train data input requires a target variable.\")\n        too_many_classes = Msg(\"Too many target variables.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        no_class_values = Msg(\"Target variable has no values.\")\n        only_one_class_var_value = Msg(\"Target variable has only one value.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_empty.clear()\n        self.Error.class_required.clear()\n        self.Error.too_many_classes.clear()\n        self.Error.no_class_values.clear()\n        self.Error.only_one_class_var_value.clear()\n        if data is not None and not data:\n            self.Error.train_data_empty()\n            data = None\n        if data:\n            conds = [not data.domain.class_vars,\n                     len(data.domain.class_vars) > 1,\n                     np.isnan(data.Y).all(),\n                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]\n            errors = [self.Error.class_required,\n                      self.Error.too_many_classes,\n                      self.Error.no_class_values,\n                      self.Error.only_one_class_var_value]\n            for cond, error in zip(conds, errors):\n                if cond:\n                    error()\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 131,
                                "end_lineno": 1124,
                                "parent": null
                            }
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# pylint: disable=missing-docstring\n# pylint: disable=protected-access\nimport unittest\nfrom unittest.mock import Mock, patch\nimport warnings\n\nimport numpy as np\nfrom AnyQt.QtCore import Qt\nfrom AnyQt.QtTest import QTest\nimport baycomp\n\nfrom Orange.classification import MajorityLearner, LogisticRegressionLearner\nfrom Orange.classification.majority import ConstantModel\nfrom Orange.data import Table, Domain, DiscreteVariable, ContinuousVariable\nfrom Orange.evaluation import Results, TestOnTestData, scoring\nfrom Orange.evaluation.scoring import ClassificationScore, RegressionScore, \\\n    Score\nfrom Orange.base import Learner\nfrom Orange.modelling import ConstantLearner\nfrom Orange.regression import MeanLearner\nfrom Orange.widgets.evaluate.owtestandscore import (\n    OWTestAndScore, results_one_vs_rest)\nfrom Orange.widgets.evaluate.utils import BUILTIN_SCORERS_ORDER\nfrom Orange.widgets.settings import (\n    ClassValuesContextHandler, PerfectDomainContextHandler)\nfrom Orange.widgets.tests.base import WidgetTest\nfrom Orange.widgets.tests.utils import simulate\nfrom Orange.widgets.utils.state_summary import (format_summary_details,\n                                                format_multiple_summaries)\nfrom Orange.tests import test_filename\n\n\nclass BadLearner(Learner):\n    def fit(self, *_, **_2):  # pylint: disable=arguments-differ\n        return 1 / 0\n\n\nclass TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\nclass TestHelpers(unittest.TestCase):\n    def test_results_one_vs_rest(self):\n        data = Table(test_filename(\"datasets/lenses.tab\"))\n        learners = [MajorityLearner()]\n        res = TestOnTestData()(data[1::2], data[::2], learners=learners)\n        r1 = results_one_vs_rest(res, pos_index=0)\n        r2 = results_one_vs_rest(res, pos_index=1)\n        r3 = results_one_vs_rest(res, pos_index=2)\n\n        np.testing.assert_almost_equal(np.sum(r1.probabilities, axis=2), 1.0)\n        np.testing.assert_almost_equal(np.sum(r2.probabilities, axis=2), 1.0)\n        np.testing.assert_almost_equal(np.sum(r3.probabilities, axis=2), 1.0)\n\n        np.testing.assert_almost_equal(\n            r1.probabilities[:, :, 1] +\n            r2.probabilities[:, :, 1] +\n            r3.probabilities[:, :, 1],\n            1.0\n        )\n        self.assertEqual(r1.folds, res.folds)\n        self.assertEqual(r2.folds, res.folds)\n        self.assertEqual(r3.folds, res.folds)\n\n        np.testing.assert_equal(r1.row_indices, res.row_indices)\n        np.testing.assert_equal(r2.row_indices, res.row_indices)\n        np.testing.assert_equal(r3.row_indices, res.row_indices)\n\n\nif __name__ == \"__main__\":\n    unittest.main()\n",
            "new_file_content": "# pylint: disable=missing-docstring\n# pylint: disable=protected-access\nimport unittest\nfrom unittest.mock import Mock, patch\nimport warnings\n\nimport numpy as np\nfrom AnyQt.QtCore import Qt\nfrom AnyQt.QtTest import QTest\nimport baycomp\n\nfrom Orange.classification import MajorityLearner, LogisticRegressionLearner\nfrom Orange.classification.majority import ConstantModel\nfrom Orange.data import Table, Domain, DiscreteVariable, ContinuousVariable\nfrom Orange.evaluation import Results, TestOnTestData, scoring\nfrom Orange.evaluation.scoring import ClassificationScore, RegressionScore, \\\n    Score\nfrom Orange.base import Learner\nfrom Orange.modelling import ConstantLearner\nfrom Orange.regression import MeanLearner\nfrom Orange.widgets.evaluate.owtestandscore import (\n    OWTestAndScore, results_one_vs_rest)\nfrom Orange.widgets.evaluate.utils import BUILTIN_SCORERS_ORDER\nfrom Orange.widgets.settings import (\n    ClassValuesContextHandler, PerfectDomainContextHandler)\nfrom Orange.widgets.tests.base import WidgetTest\nfrom Orange.widgets.tests.utils import simulate\nfrom Orange.widgets.utils.state_summary import (format_summary_details,\n                                                format_multiple_summaries)\nfrom Orange.tests import test_filename\n\n\nclass BadLearner(Learner):\n    def fit(self, *_, **_2):  # pylint: disable=arguments-differ\n        return 1 / 0\n\n\nclass TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.train_data_error.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.train_data_error.is_shown())\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n\n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n\n        ):\n            assertErrorShown(data, is_shown, message)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(\n            self._test_scores(\n                table, table[:3], ConstantLearner(),\n                OWTestAndScore.TestOnTest, None\n            ),\n            (None, 1, 1, 1, 1)\n        )\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\nclass TestHelpers(unittest.TestCase):\n    def test_results_one_vs_rest(self):\n        data = Table(test_filename(\"datasets/lenses.tab\"))\n        learners = [MajorityLearner()]\n        res = TestOnTestData()(data[1::2], data[::2], learners=learners)\n        r1 = results_one_vs_rest(res, pos_index=0)\n        r2 = results_one_vs_rest(res, pos_index=1)\n        r3 = results_one_vs_rest(res, pos_index=2)\n\n        np.testing.assert_almost_equal(np.sum(r1.probabilities, axis=2), 1.0)\n        np.testing.assert_almost_equal(np.sum(r2.probabilities, axis=2), 1.0)\n        np.testing.assert_almost_equal(np.sum(r3.probabilities, axis=2), 1.0)\n\n        np.testing.assert_almost_equal(\n            r1.probabilities[:, :, 1] +\n            r2.probabilities[:, :, 1] +\n            r3.probabilities[:, :, 1],\n            1.0\n        )\n        self.assertEqual(r1.folds, res.folds)\n        self.assertEqual(r2.folds, res.folds)\n        self.assertEqual(r3.folds, res.folds)\n\n        np.testing.assert_equal(r1.row_indices, res.row_indices)\n        np.testing.assert_equal(r2.row_indices, res.row_indices)\n        np.testing.assert_equal(r3.row_indices, res.row_indices)\n\n\nif __name__ == \"__main__\":\n    unittest.main()\n",
            "header": {
                "file": {
                    "path": "Orange/widgets/evaluate/tests/test_owtestandscore.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "b351ec254",
                "new_commit_hash": "f63148342",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/Orange/widgets/evaluate/tests/test_owtestandscore.py"
            },
            "plus_file": {
                "path": "b/Orange/widgets/evaluate/tests/test_owtestandscore.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 168,
                            "length": 25
                        },
                        "new_range": {
                            "start": 168,
                            "length": 38
                        },
                        "section": "class TestOWTestAndScore(WidgetTest):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                \"yyyy\"))",
                                "type": "context"
                            },
                            {
                                "content": "        )",
                                "type": "context"
                            },
                            {
                                "content": "        self.widget.n_folds = 0",
                                "type": "context"
                            },
                            {
                                "content": "        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.assertFalse(self.widget.Error.train_data_error.is_shown())",
                                "type": "added"
                            },
                            {
                                "content": "        self.send_signal(\"Data\", table)",
                                "type": "context"
                            },
                            {
                                "content": "        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)",
                                "type": "context"
                            },
                            {
                                "content": "        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.assertTrue(self.widget.Error.train_data_error.is_shown())",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def test_nan_class(self):",
                                "type": "deleted"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "deleted"
                            },
                            {
                                "content": "        Do not crash on a data with only nan class values.",
                                "type": "deleted"
                            },
                            {
                                "content": "        GH-2751",
                                "type": "deleted"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "deleted"
                            },
                            {
                                "content": "        def assertErrorShown(data, is_shown):",
                                "type": "deleted"
                            },
                            {
                                "content": "    def test_data_errors(self):",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\" Test all data_errors \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "        def assertErrorShown(data, is_shown, message):",
                                "type": "added"
                            },
                            {
                                "content": "            self.send_signal(\"Data\", data)",
                                "type": "context"
                            },
                            {
                                "content": "            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())",
                                "type": "deleted"
                            },
                            {
                                "content": "            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())",
                                "type": "added"
                            },
                            {
                                "content": "            self.assertEqual(message, str(self.widget.Error.train_data_error))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        data = Table(\"iris\")[::30]",
                                "type": "context"
                            },
                            {
                                "content": "        data.Y[:] = np.nan",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):",
                                "type": "deleted"
                            },
                            {
                                "content": "            assertErrorShown(data, is_shown)",
                                "type": "deleted"
                            },
                            {
                                "content": "        iris_empty_x = Table.from_table(",
                                "type": "added"
                            },
                            {
                                "content": "            Domain([], data.domain.class_var), Table(\"iris\")",
                                "type": "added"
                            },
                            {
                                "content": "        )",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        for data, is_shown, message in zip(",
                                "type": "added"
                            },
                            {
                                "content": "                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],",
                                "type": "added"
                            },
                            {
                                "content": "                [False, True, True, True, True],",
                                "type": "added"
                            },
                            {
                                "content": "                [",
                                "type": "added"
                            },
                            {
                                "content": "                    \"\",",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Target variable has no values.\",",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Target variable has only one value.\",",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Data has no features to learn from.\",",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Train dataset is empty.\"",
                                "type": "added"
                            },
                            {
                                "content": "                ]",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        ):",
                                "type": "added"
                            },
                            {
                                "content": "            assertErrorShown(data, is_shown, message)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def test_addon_scorers(self):",
                                "type": "context"
                            },
                            {
                                "content": "        try:",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "class",
                            "name": "TestOWTestAndScore",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 38,
                            "end_lineno": 662,
                            "parent": null
                        },
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "method",
                            "name": "TestOWTestAndScore.test_one_class_value",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 155,
                            "end_lineno": 174,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                                "type": "class",
                                "name": "TestOWTestAndScore",
                                "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 38,
                                "end_lineno": 662,
                                "parent": null
                            }
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "method",
                            "name": "TestOWTestAndScore.test_data_errors",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.train_data_error.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.train_data_error.is_shown())\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n\n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n\n        ):\n            assertErrorShown(data, is_shown, message)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(\n            self._test_scores(\n                table, table[:3], ConstantLearner(),\n                OWTestAndScore.TestOnTest, None\n            ),\n            (None, 1, 1, 1, 1)\n        )\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 176,
                            "end_lineno": 202,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                                "type": "class",
                                "name": "TestOWTestAndScore",
                                "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.train_data_error.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.train_data_error.is_shown())\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n\n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n\n        ):\n            assertErrorShown(data, is_shown, message)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(\n            self._test_scores(\n                table, table[:3], ConstantLearner(),\n                OWTestAndScore.TestOnTest, None\n            ),\n            (None, 1, 1, 1, 1)\n        )\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 38,
                                "end_lineno": 679,
                                "parent": null
                            }
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "method",
                            "name": "TestOWTestAndScore.test_nan_class",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 176,
                            "end_lineno": 189,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                                "type": "class",
                                "name": "TestOWTestAndScore",
                                "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 38,
                                "end_lineno": 662,
                                "parent": null
                            }
                        }
                    ]
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 321,
                            "length": 15
                        },
                        "new_range": {
                            "start": 334,
                            "length": 19
                        },
                        "section": "class TestOWTestAndScore(WidgetTest):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)",
                                "type": "context"
                            },
                            {
                                "content": "        return self._retrieve_scores()",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def test_scores_constant_all_same(self):",
                                "type": "deleted"
                            },
                            {
                                "content": "    def test_scores_constant(self):",
                                "type": "added"
                            },
                            {
                                "content": "        table = Table.from_list(",
                                "type": "context"
                            },
                            {
                                "content": "            self.scores_domain,",
                                "type": "context"
                            },
                            {
                                "content": "            list(zip(*self.scores_table_values + [list(\"yyyy\")]))",
                                "type": "deleted"
                            },
                            {
                                "content": "            list(zip(*self.scores_table_values + [list(\"yyyn\")]))",
                                "type": "added"
                            },
                            {
                                "content": "        )",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        self.assertTupleEqual(self._test_scores(",
                                "type": "deleted"
                            },
                            {
                                "content": "            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),",
                                "type": "deleted"
                            },
                            {
                                "content": "                              (None, 1, 1, 1, 1))",
                                "type": "deleted"
                            },
                            {
                                "content": "        self.assertTupleEqual(",
                                "type": "added"
                            },
                            {
                                "content": "            self._test_scores(",
                                "type": "added"
                            },
                            {
                                "content": "                table, table[:3], ConstantLearner(),",
                                "type": "added"
                            },
                            {
                                "content": "                OWTestAndScore.TestOnTest, None",
                                "type": "added"
                            },
                            {
                                "content": "            ),",
                                "type": "added"
                            },
                            {
                                "content": "            (None, 1, 1, 1, 1)",
                                "type": "added"
                            },
                            {
                                "content": "        )",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def test_scores_log_reg_overfitted(self):",
                                "type": "context"
                            },
                            {
                                "content": "        table = Table.from_list(",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "class",
                            "name": "TestOWTestAndScore",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 38,
                            "end_lineno": 662,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "method",
                            "name": "TestOWTestAndScore.test_scores_constant",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.train_data_error.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.train_data_error.is_shown())\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n\n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n\n        ):\n            assertErrorShown(data, is_shown, message)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(\n            self._test_scores(\n                table, table[:3], ConstantLearner(),\n                OWTestAndScore.TestOnTest, None\n            ),\n            (None, 1, 1, 1, 1)\n        )\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 337,
                            "end_lineno": 349,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                                "type": "class",
                                "name": "TestOWTestAndScore",
                                "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.train_data_error.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.train_data_error.is_shown())\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n\n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n\n        ):\n            assertErrorShown(data, is_shown, message)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(\n            self._test_scores(\n                table, table[:3], ConstantLearner(),\n                OWTestAndScore.TestOnTest, None\n            ),\n            (None, 1, 1, 1, 1)\n        )\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 38,
                                "end_lineno": 679,
                                "parent": null
                            }
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                            "type": "method",
                            "name": "TestOWTestAndScore.test_scores_constant_all_same",
                            "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 324,
                            "end_lineno": 332,
                            "parent": {
                                "file_name": "Orange/widgets/evaluate/tests/test_owtestandscore.py",
                                "type": "class",
                                "name": "TestOWTestAndScore",
                                "content": "class TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())\n\n    def test_nan_class(self):\n        \"\"\"\n        Do not crash on a data with only nan class values.\n        GH-2751\n        \"\"\"\n        def assertErrorShown(data, is_shown):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        for data, is_shown in zip([None, data, Table(\"iris\")[:30]], [False, True, False]):\n            assertErrorShown(data, is_shown)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant_all_same(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyy\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),\n                              (None, 1, 1, 1, 1))\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 38,
                                "end_lineno": 662,
                                "parent": null
                            }
                        }
                    ]
                }
            ]
        }
    ],
    "old_commit_hash": "67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e^",
    "new_commit_hash": "67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e",
    "commit_message": "Merge pull request #4738 from PrimozGodec/test-and-score-warning\n\n[FIX] Test and Score: Improve data errors",
    "commit_date": "2020-05-08T15:21:48+02:00",
    "metadata": {}
}
{
    "repo_name": "orange3",
    "new_commit_hash": "67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e",
    "test_file_codes": [
        "# pylint: disable=missing-docstring\n# pylint: disable=protected-access\nimport unittest\nfrom unittest.mock import Mock, patch\nimport warnings\n\nimport numpy as np\nfrom AnyQt.QtCore import Qt\nfrom AnyQt.QtTest import QTest\nimport baycomp\n\nfrom Orange.classification import MajorityLearner, LogisticRegressionLearner\nfrom Orange.classification.majority import ConstantModel\nfrom Orange.data import Table, Domain, DiscreteVariable, ContinuousVariable\nfrom Orange.evaluation import Results, TestOnTestData, scoring\nfrom Orange.evaluation.scoring import ClassificationScore, RegressionScore, \\\n    Score\nfrom Orange.base import Learner\nfrom Orange.modelling import ConstantLearner\nfrom Orange.regression import MeanLearner\nfrom Orange.widgets.evaluate.owtestandscore import (\n    OWTestAndScore, results_one_vs_rest)\nfrom Orange.widgets.evaluate.utils import BUILTIN_SCORERS_ORDER\nfrom Orange.widgets.settings import (\n    ClassValuesContextHandler, PerfectDomainContextHandler)\nfrom Orange.widgets.tests.base import WidgetTest\nfrom Orange.widgets.tests.utils import simulate\nfrom Orange.widgets.utils.state_summary import (format_summary_details,\n                                                format_multiple_summaries)\nfrom Orange.tests import test_filename\n\n\nclass BadLearner(Learner):\n    def fit(self, *_, **_2):  # pylint: disable=arguments-differ\n        return 1 / 0\n\n\nclass TestOWTestAndScore(WidgetTest):\n    def setUp(self):\n        super().setUp()\n        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore\n\n        self.scores_domain = Domain(\n            [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n            [DiscreteVariable(\"c\", values=(\"y\", \"n\"))])\n\n        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]\n\n    def tearDown(self):\n        self.widget.onDeleteWidget()\n        super().tearDown()\n\n    def test_basic(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n        self.assertIsNotNone(res.probabilities)\n\n        self.send_signal(self.widget.Inputs.learner, None, 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(res)\n\n        data = Table(\"housing\")[::10]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsInstance(res, Results)\n        self.assertIsNotNone(res.domain)\n        self.assertIsNotNone(res.data)\n\n    def test_more_learners(self):\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 1)\n        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])\n\n    def test_testOnTest(self):\n        data = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        self.send_signal(self.widget.Inputs.test_data, data)\n\n    def test_testOnTest_incompatible_domain(self):\n        iris = Table(\"iris\")\n        self.send_signal(self.widget.Inputs.train_data, iris)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())\n        self.widget.resampling = OWTestAndScore.TestOnTest\n        # test data with the same class (otherwise the widget shows a different error)\n        # and a non-nan X\n        iris_test = iris.transform(Domain([ContinuousVariable(\"x\")],\n                                          class_vars=iris.domain.class_vars))\n        iris_test.X[:, 0] = 1\n        self.send_signal(self.widget.Inputs.test_data, iris_test)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())\n\n    def test_CrossValidationByFeature(self):\n        data = Table(\"iris\")\n        attrs = data.domain.attributes\n        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)\n        data_with_disc_metas = Table.from_table(domain, data)\n        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]\n\n        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(rb.isEnabled())\n        self.assertFalse(self.widget.features_combo.isEnabled())\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)\n        self.assertTrue(rb.isEnabled())\n        rb.click()\n        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)\n        self.assertTrue(self.widget.features_combo.isEnabled())\n        self.assertEqual(self.widget.features_combo.currentText(), \"iris\")\n        self.assertEqual(len(self.widget.features_combo.model()), 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        self.assertFalse(rb.isEnabled())\n        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)\n        self.assertFalse(self.widget.features_combo.isEnabled())\n\n    def test_migrate_removes_invalid_contexts(self):\n        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])\n        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)\n        settings = {'context_settings': [context_invalid, context_valid]}\n        self.widget.migrate_settings(settings, 2)\n        self.assertEqual(settings['context_settings'], [context_valid])\n\n    def test_memory_error(self):\n        \"\"\"\n        Handling memory error.\n        GH-2316\n        \"\"\"\n        data = Table(\"iris\")[::15]\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.assertFalse(self.widget.Error.memory_error.is_shown())\n\n        with unittest.mock.patch(\n                \"Orange.evaluation.testing.Results.get_augmented_data\",\n                side_effect=MemoryError):\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)\n            self.assertTrue(self.widget.Error.memory_error.is_shown())\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n        self.assertFalse(self.widget.Error.train_data_error.is_shown())\n        self.send_signal(\"Data\", table)\n        self.send_signal(\"Learner\", MajorityLearner(), 0, wait=1000)\n        self.assertTrue(self.widget.Error.train_data_error.is_shown())\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n\n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n\n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n\n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n\n        ):\n            assertErrorShown(data, is_shown, message)\n\n    def test_addon_scorers(self):\n        try:\n            # These classes are registered, pylint: disable=unused-variable\n            class NewScore(Score):\n                class_types = (DiscreteVariable, ContinuousVariable)\n\n            class NewClassificationScore(ClassificationScore):\n                pass\n\n            class NewRegressionScore(RegressionScore):\n                pass\n\n            builtins = BUILTIN_SCORERS_ORDER\n            self.send_signal(\"Data\", Table(\"iris\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[DiscreteVariable])]),\n                builtins[DiscreteVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertIn(\"NewClassificationScore\", scorer_names)\n            self.assertNotIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", Table(\"housing\"))\n            scorer_names = [scorer.name for scorer in self.widget.scorers]\n            self.assertEqual(\n                tuple(scorer_names[:len(builtins[ContinuousVariable])]),\n                builtins[ContinuousVariable])\n            self.assertIn(\"NewScore\", scorer_names)\n            self.assertNotIn(\"NewClassificationScore\", scorer_names)\n            self.assertIn(\"NewRegressionScore\", scorer_names)\n\n            self.send_signal(\"Data\", None)\n            self.assertEqual(self.widget.scorers, [])\n        finally:\n            del Score.registry[\"NewScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewClassificationScore\"]  # pylint: disable=no-member\n            del Score.registry[\"NewRegressionScore\"]  # pylint: disable=no-member\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n\n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n\n        average_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n\n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n\n        self.assertGreater(average_auc, versicolor_auc)\n        self.assertGreater(average_auc, virginica_auc)\n        self.assertLess(average_auc, setosa_auc)\n        self.assertGreater(setosa_auc, versicolor_auc)\n        self.assertGreater(setosa_auc, virginica_auc)\n\n    def test_resort_on_data_change(self):\n        iris = Table(\"iris\")\n        # one example is included from the other class\n        # to keep F1 from complaining\n        setosa = iris[:51]\n        versicolor = iris[49:100]\n\n        class SetosaLearner:\n            def __call__(self, data):\n                model = ConstantModel([1., 0, 0])\n                model.domain = iris.domain\n                return model\n\n        class VersicolorLearner:\n            def __call__(self, data):\n                model = ConstantModel([0, 1., 0])\n                model.domain = iris.domain\n                return model\n\n        # this is done manually to avoid multiple computations\n        self.widget.resampling = 5\n        self.widget.set_train_data(iris)\n        self.widget.set_learner(SetosaLearner(), 1)\n        self.widget.set_learner(VersicolorLearner(), 2)\n\n        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)\n\n        self.widget.adjustSize()\n        view = self.widget.score_table.view\n        header = view.horizontalHeader()\n        p = header.rect().center()\n        # second visible header section (after 'Model')\n        _, idx, *_ = (i for i in range(header.count())\n                      if not header.isSectionHidden(i))\n        p.setX(header.sectionPosition(idx) + 5)\n        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)\n\n        # Ensure that the click on header caused an ascending sort\n        # Ascending sort means that wrong model should be listed first\n        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)\n        self.assertEqual(view.model().index(0, 0).data(), \"VersicolorLearner\")\n\n        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)\n        self.assertEqual(view.model().index(0, 0).data(), \"SetosaLearner\")\n\n    def _retrieve_scores(self):\n        w = self.widget\n        model = w.score_table.model\n        auc = model.item(0, 3).text()\n        auc = float(auc) if auc != \"\" else None\n        ca = float(model.item(0, 4).text())\n        f1 = float(model.item(0, 5).text())\n        precision = float(model.item(0, 6).text())\n        recall = float(model.item(0, 7).text())\n        return auc, ca, f1, precision, recall\n\n    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):\n        w = self.widget  #: OWTestAndScore\n        w.controls.resampling.buttons[sampling].click()\n        if n_folds is not None:\n            w.n_folds = n_folds\n\n        self.send_signal(self.widget.Inputs.train_data, train_data)\n        if test_data is not None:\n            self.send_signal(self.widget.Inputs.test_data, test_data)\n        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)\n        return self._retrieve_scores()\n\n    def test_scores_constant(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(\n            self._test_scores(\n                table, table[:3], ConstantLearner(),\n                OWTestAndScore.TestOnTest, None\n            ),\n            (None, 1, 1, 1, 1)\n        )\n\n    def test_scores_log_reg_overfitted(self):\n        table = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table, table, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (1, 1, 1, 1, 1))\n\n    def test_scores_log_reg_bad(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"nnny\")]))\n        )\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*self.scores_table_values + [list(\"yyyn\")]))\n        )\n\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_bad2(self):\n        table_train = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"nnyy\")]))))\n        table_test = Table.from_list(\n            self.scores_domain,\n            list(zip(*(self.scores_table_values + [list(\"yynn\")]))))\n        self.assertTupleEqual(self._test_scores(\n            table_train, table_test, LogisticRegressionLearner(),\n            OWTestAndScore.TestOnTest, None),\n                              (0, 0, 0, 0, 0))\n\n    def test_scores_log_reg_advanced(self):\n        table_train = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yyynn\"))\n        )\n        table_test = Table.from_list(\n            self.scores_domain, list(zip(\n                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], \"yynnn\"))\n        )\n\n        np.testing.assert_almost_equal(\n            self._test_scores(table_train, table_test,\n                              LogisticRegressionLearner(),\n                              OWTestAndScore.TestOnTest, None),\n            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))\n\n    def test_scores_cross_validation(self):\n        \"\"\"\n        Test more than two classes and cross-validation\n        \"\"\"\n        self.assertTrue(\n            all(x >= y for x, y in zip(\n                self._test_scores(\n                    Table(\"iris\")[::15], None, LogisticRegressionLearner(),\n                    OWTestAndScore.KFold, 0),\n                (0.8, 0.5, 0.5, 0.5, 0.5))))\n\n    def test_no_pregressbar_warning(self):\n        data = Table(\"iris\")[::15]\n\n        with warnings.catch_warnings(record=True) as w:\n            self.send_signal(self.widget.Inputs.train_data, data)\n            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)\n            assert not w\n\n    def _set_comparison_score(self, score):\n        w = self.widget\n        control = w.controls.comparison_criterion\n        control.setCurrentText(score)\n        w.comparison_criterion = control.findText(score)\n\n    def _set_three_majorities(self):\n        w = self.widget\n        data = Table(\"iris\")[::15]\n        self.send_signal(w.Inputs.train_data, data)\n        for i, name in enumerate([\"maja\", \"majb\", \"majc\"]):\n            learner = MajorityLearner()\n            learner.name = name\n            self.send_signal(w.Inputs.learner, learner, i)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n    @patch(\"baycomp.two_on_single\", Mock(wraps=baycomp.two_on_single))\n    def test_comparison_requires_cv(self):\n        w = self.widget\n        self.send_signal(w.Inputs.train_data, Table(\"iris\")[::15])\n\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n        self.assertFalse(w.modcompbox.isEnabled())\n        self.assertFalse(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_not_called()\n        baycomp.two_on_single.reset_mock()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertTrue(w.modcompbox.isEnabled())\n        self.assertTrue(w.comparison_table.isEnabled())\n        baycomp.two_on_single.assert_called()\n        baycomp.two_on_single.reset_mock()\n\n    def test_comparison_requires_multiple_models(self):\n        w = self.widget\n        w.comparison_criterion = 1\n        rbs = w.controls.resampling.buttons\n\n        self._set_three_majorities()\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n        self.send_signal(w.Inputs.learner, None, 2)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.LeaveOneOut].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        learner = MajorityLearner()\n        learner.name = \"majd\"\n        self.send_signal(w.Inputs.learner, learner, 1)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertFalse(w.comparison_table.isEnabled())\n\n        rbs[OWTestAndScore.KFold].click()\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertTrue(w.comparison_table.isEnabled())\n\n    def test_comparison_bad_slots(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))\n        self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n        self.assertEqual(len(w._successful_slots()), 2)\n\n    def test_comparison_bad_scores(self):\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"Classification accuracy\")\n        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)\n\n        score_calls = -1\n\n        def fail_on_first(*_, **_2):\n            nonlocal score_calls\n            score_calls += 1\n            return 1 / score_calls\n\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            w.update_comparison_table()\n\n            self.assertIsNone(w.comparison_table.cellWidget(0, 1))\n            self.assertIsNone(w.comparison_table.cellWidget(0, 2))\n            self.assertIsNone(w.comparison_table.cellWidget(1, 0))\n            self.assertIsNone(w.comparison_table.cellWidget(2, 0))\n            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))\n            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))\n            self.assertTrue(w.Warning.scores_not_computed.is_shown())\n\n        score_calls = -1\n        with patch.object(scoring.CA, \"compute_score\", new=fail_on_first):\n            slots = w._successful_slots()\n            self.assertEqual(len(slots), 3)\n            scores = w._scores_by_folds(slots)\n            self.assertIsNone(scores[0])\n            self.assertEqual(scores[1][0], 1)\n            self.assertAlmostEqual(scores[2][0], 1 / 11)\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n\n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 1)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[2])\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"target\"], 2)\n            self.assertFalse(\"average\" in kwargs)\n\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            OWTestAndScore.TARGET_AVERAGE)\n            _, kwargs = f1mock.call_args\n            self.assertEqual(kwargs[\"average\"], \"weighted\")\n            self.assertFalse(\"target\" in kwargs)\n\n    def test_fill_table(self):\n        w = self.widget\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def probs(p1, p2, rope):\n            p1 += 1\n            p2 += 1\n            norm = p1 + p2 + rope * (p1 + p2)\n            if rope == 0:\n                return p1 / norm, p2 / norm\n            else:\n                return p1 / norm, rope / norm, p2 / norm\n\n        def two_on_single(res1, res2, rope=0):\n            return probs(scores.index(res1), scores.index(res2), rope)\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):\n                w._fill_table(slots, scores)\n                for row in range(3):\n                    for col in range(3):\n                        if row == col:\n                            continue\n                        label = w.comparison_table.cellWidget(row, col)\n                        self.assertEqual(label.text(),\n                                         f\"{(row + 1) / (row + col + 2):.3f}\")\n                        self.assertIn(f\"{(row + 1) / (row + col + 2):.3f}\",\n                                      label.toolTip())\n\n            w.use_rope = True\n            w.rope = 0.25\n            w._fill_table(slots, scores)\n            for row in range(3):\n                for col in range(3):\n                    if row == col:\n                        continue\n                    label = w.comparison_table.cellWidget(row, col)\n                    for text in (label.text(), label.toolTip()):\n                        self.assertIn(f\"{probs(row, col, w.rope)[0]:.3f}\", text)\n                        self.assertIn(f\"{probs(row, col, w.rope)[1]:.3f}\", text)\n\n    def test_nan_on_comparison(self):\n        w = self.widget\n        w.use_rope = True\n        self._set_three_majorities()\n        scores = [object(), object(), object()]\n        slots = w._successful_slots()\n\n        def two_on_single(_1, _2, rope=0):\n            if rope:\n                return np.nan, np.nan, np.nan\n            else:\n                return np.nan, np.nan\n\n        with patch(\"baycomp.two_on_single\", new=two_on_single):\n            for w.rope in (0, 0.1):\n                w._fill_table(slots, scores)\n                label = w.comparison_table.cellWidget(1, 0)\n                self.assertEqual(label.text(), \"NA\")\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n\n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n\n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\n        self.send_signal(self.widget.Inputs.test_data, None)\n        self.assertEqual(info._StateInfo__input_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__input_summary.details, no_input)\n        self.assertEqual(info._StateInfo__output_summary.brief, \"\")\n        self.assertEqual(info._StateInfo__output_summary.details, no_output)\n\nclass TestHelpers(unittest.TestCase):\n    def test_results_one_vs_rest(self):\n        data = Table(test_filename(\"datasets/lenses.tab\"))\n        learners = [MajorityLearner()]\n        res = TestOnTestData()(data[1::2], data[::2], learners=learners)\n        r1 = results_one_vs_rest(res, pos_index=0)\n        r2 = results_one_vs_rest(res, pos_index=1)\n        r3 = results_one_vs_rest(res, pos_index=2)\n\n        np.testing.assert_almost_equal(np.sum(r1.probabilities, axis=2), 1.0)\n        np.testing.assert_almost_equal(np.sum(r2.probabilities, axis=2), 1.0)\n        np.testing.assert_almost_equal(np.sum(r3.probabilities, axis=2), 1.0)\n\n        np.testing.assert_almost_equal(\n            r1.probabilities[:, :, 1] +\n            r2.probabilities[:, :, 1] +\n            r3.probabilities[:, :, 1],\n            1.0\n        )\n        self.assertEqual(r1.folds, res.folds)\n        self.assertEqual(r2.folds, res.folds)\n        self.assertEqual(r3.folds, res.folds)\n\n        np.testing.assert_equal(r1.row_indices, res.row_indices)\n        np.testing.assert_equal(r2.row_indices, res.row_indices)\n        np.testing.assert_equal(r3.row_indices, res.row_indices)\n\n\nif __name__ == \"__main__\":\n    unittest.main()\n"
    ],
    "test_file_names": [
        "test_1.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Orange installation attempts...\nAttempting installation with Python 3.7...\nrunning build_ext\nrunning build_src\nbuild_src\nbuilding extension \"Orange.classification._simple_tree\" sources\nbuilding extension \"Orange.classification._tree_scorers\" sources\nbuilding extension \"Orange.data._valuecount\" sources\nbuilding extension \"Orange.data._contingency\" sources\nbuilding extension \"Orange.data._io\" sources\nbuilding extension \"Orange.data._variable\" sources\nbuilding extension \"Orange.distance._distance\" sources\nbuilding extension \"Orange.preprocess._discretize\" sources\nbuilding extension \"Orange.preprocess._relieff\" sources\nbuilding extension \"Orange.projection._som\" sources\nbuilding extension \"Orange.widgets.utils._grid_density\" sources\nbuild_src: building npy-pkg config files\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpr30a3c9_/tmp\ncreating /tmp/tmpr30a3c9_/tmp/tmpr30a3c9_\ncompile options: '-MMD -MF /tmp/tmpr30a3c9_/file.c.d -c'\ncc: /tmp/tmpr30a3c9_/file.c\ncustomize UnixCCompiler using build_ext\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpd09o9q16/tmp\ncreating /tmp/tmpd09o9q16/tmp/tmpd09o9q16\ncompile options: '-MMD -MF /tmp/tmpd09o9q16/file.c.d -c'\ncc: /tmp/tmpd09o9q16/file.c\ncustomize UnixCCompiler using build_ext\nbuilding 'Orange.classification._simple_tree' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange\ncreating build/temp.linux-x86_64-3.7/Orange/classification\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/classification/_simple_tree.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/classification/_simple_tree.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/classification/_simple_tree.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.classification._tree_scorers' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/classification/_tree_scorers.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/classification/_tree_scorers.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/classification/_tree_scorers.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._valuecount' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/data\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_valuecount.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_valuecount.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_valuecount.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._contingency' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_contingency.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_contingency.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_contingency.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._io' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_io.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_io.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_io.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.data._variable' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/data/_variable.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/data/_variable.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/data/_variable.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.distance._distance' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/distance\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/distance/_distance.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/distance/_distance.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/distance/_distance.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.preprocess._discretize' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/preprocess\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/preprocess/_discretize.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/preprocess/_discretize.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/preprocess/_discretize.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.preprocess._relieff' extension\ncompiling C++ sources\nC compiler: c++ -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\nc++: Orange/preprocess/_relieff.cpp\nc++ -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/preprocess/_relieff.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/preprocess/_relieff.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.projection._som' extension\ncompiling C sources\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/projection\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\ncc: Orange/projection/_som.c\ncc -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/projection/_som.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/projection/_som.cpython-37m-x86_64-linux-gnu.so\nbuilding 'Orange.widgets.utils._grid_density' extension\ncompiling C++ sources\nC compiler: c++ -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating build/temp.linux-x86_64-3.7/Orange/widgets\ncreating build/temp.linux-x86_64-3.7/Orange/widgets/utils\ncompile options: '-I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include -I/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/include -I/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/include/python3.7m -c'\nc++: Orange/widgets/utils/_grid_density.cpp\nc++ -pthread -shared -L/tools/deps/lib build/temp.linux-x86_64-3.7/Orange/widgets/utils/_grid_density.o -L/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib -lm -lpython3.7m -o Orange/widgets/utils/_grid_density.cpython-37m-x86_64-linux-gnu.so\nrunning develop\nrunning build_scripts\nrunning egg_info\nrunning build_src\nbuild_src\nbuilding extension \"Orange.classification._simple_tree\" sources\nbuilding extension \"Orange.classification._tree_scorers\" sources\nbuilding extension \"Orange.data._valuecount\" sources\nbuilding extension \"Orange.data._contingency\" sources\nbuilding extension \"Orange.data._io\" sources\nbuilding extension \"Orange.data._variable\" sources\nbuilding extension \"Orange.distance._distance\" sources\nbuilding extension \"Orange.preprocess._discretize\" sources\nbuilding extension \"Orange.preprocess._relieff\" sources\nbuilding extension \"Orange.projection._som\" sources\nbuilding extension \"Orange.widgets.utils._grid_density\" sources\nbuild_src: building npy-pkg config files\ncreating Orange3.egg-info\nwriting Orange3.egg-info/PKG-INFO\nwriting dependency_links to Orange3.egg-info/dependency_links.txt\nwriting entry points to Orange3.egg-info/entry_points.txt\nwriting requirements to Orange3.egg-info/requires.txt\nwriting top-level names to Orange3.egg-info/top_level.txt\nwriting manifest file 'Orange3.egg-info/SOURCES.txt'\nreading manifest file 'Orange3.egg-info/SOURCES.txt'\nreading manifest template 'MANIFEST.in'\nwarning: no files found matching 'README*' under directory 'Orange'\nwarning: no files found matching 'LICENSE' under directory 'Orange'\nwarning: no files found matching '*.basket' under directory 'Orange/datasets'\nwarning: no files found matching '*.js' under directory 'Orange/widgets'\nwarning: no files found matching '*.css' under directory 'Orange/widgets'\nwarning: no files found matching '*.html' under directory 'Orange/widgets'\nwarning: no files found matching 'COPYING'\nadding license file 'LICENSE'\nwriting manifest file 'Orange3.egg-info/SOURCES.txt'\nrunning build_ext\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpqgi479rn/tmp\ncreating /tmp/tmpqgi479rn/tmp/tmpqgi479rn\ncompile options: '-MMD -MF /tmp/tmpqgi479rn/file.c.d -c'\ncc: /tmp/tmpqgi479rn/file.c\ncustomize UnixCCompiler using build_ext\ncustomize UnixCCompiler\nC compiler: cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tools/deps/include -I/tools/deps/include/ncursesw -fPIC\n\ncreating /tmp/tmpa1efwg2z/tmp\ncreating /tmp/tmpa1efwg2z/tmp/tmpa1efwg2z\ncompile options: '-MMD -MF /tmp/tmpa1efwg2z/file.c.d -c'\ncc: /tmp/tmpa1efwg2z/file.c\ncustomize UnixCCompiler using build_ext\nCreating /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/Orange3.egg-link (link to .)\nAdding Orange3 3.26.0.dev0+67ed8aa to easy-install.pth file\nInstalling orange-canvas script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nInstalled /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e\nProcessing dependencies for Orange3==3.26.0.dev0+67ed8aa\nSearching for XlsxWriter==3.2.0\nBest match: XlsxWriter 3.2.0\nAdding XlsxWriter 3.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for xlrd==2.0.1\nBest match: xlrd 2.0.1\nAdding xlrd 2.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for setuptools==59.8.0\nBest match: setuptools 59.8.0\nAdding setuptools 59.8.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for serverfiles==0.3.1\nBest match: serverfiles 0.3.1\nAdding serverfiles 0.3.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for scipy==1.7.3\nBest match: scipy 1.7.3\nAdding scipy 1.7.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for scikit-learn==1.0.2\nBest match: scikit-learn 1.0.2\nAdding scikit-learn 1.0.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for requests==2.31.0\nBest match: requests 2.31.0\nAdding requests 2.31.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for PyYAML==6.0.1\nBest match: PyYAML 6.0.1\nAdding PyYAML 6.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for python-louvain==0.16\nBest match: python-louvain 0.16\nAdding python-louvain 0.16 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for pyqtgraph==0.12.4\nBest match: pyqtgraph 0.12.4\nAdding pyqtgraph 0.12.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for pip==24.0\nBest match: pip 24.0\nAdding pip 24.0 to easy-install.pth file\nInstalling pip script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling pip3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling pip3.10 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for pandas==1.3.5\nBest match: pandas 1.3.5\nAdding pandas 1.3.5 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for orange-widget-base==4.22.0\nBest match: orange-widget-base 4.22.0\nAdding orange-widget-base 4.22.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for orange-canvas-core==0.1.35\nBest match: orange-canvas-core 0.1.35\nAdding orange-canvas-core 0.1.35 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for openpyxl==3.1.3\nBest match: openpyxl 3.1.3\nAdding openpyxl 3.1.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for openTSNE==1.0.1\nBest match: openTSNE 1.0.1\nAdding openTSNE 1.0.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for numpy==1.17.5\nBest match: numpy 1.17.5\nAdding numpy 1.17.5 to easy-install.pth file\nInstalling f2py script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling f2py3 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling f2py3.7 script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for networkx==2.6.3\nBest match: networkx 2.6.3\nAdding networkx 2.6.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for matplotlib==3.5.3\nBest match: matplotlib 3.5.3\nAdding matplotlib 3.5.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for keyrings.alt==4.2.0\nBest match: keyrings.alt 4.2.0\nAdding keyrings.alt 4.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for keyring==24.1.1\nBest match: keyring 24.1.1\nAdding keyring 24.1.1 to easy-install.pth file\nInstalling keyring script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for joblib==1.3.2\nBest match: joblib 1.3.2\nAdding joblib 1.3.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for httpx==0.24.1\nBest match: httpx 0.24.1\nAdding httpx 0.24.1 to easy-install.pth file\nInstalling httpx script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for docutils==0.20.1\nBest match: docutils 0.20.1\nAdding docutils 0.20.1 to easy-install.pth file\nInstalling docutils script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for chardet==5.2.0\nBest match: chardet 5.2.0\nAdding chardet 5.2.0 to easy-install.pth file\nInstalling chardetect script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for Bottleneck==1.4.0\nBest match: Bottleneck 1.4.0\nAdding Bottleneck 1.4.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for baycomp==1.0.3\nBest match: baycomp 1.0.3\nAdding baycomp 1.0.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for AnyQt==0.2.0\nBest match: AnyQt 0.2.0\nAdding AnyQt 0.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for threadpoolctl==3.1.0\nBest match: threadpoolctl 3.1.0\nAdding threadpoolctl 3.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for urllib3==2.0.7\nBest match: urllib3 2.0.7\nAdding urllib3 2.0.7 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for idna==3.10\nBest match: idna 3.10\nAdding idna 3.10 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for charset-normalizer==3.4.1\nBest match: charset-normalizer 3.4.1\nAdding charset-normalizer 3.4.1 to easy-install.pth file\nInstalling normalizer script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for certifi==2024.12.14\nBest match: certifi 2024.12.14\nAdding certifi 2024.12.14 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for pytz==2024.2\nBest match: pytz 2024.2\nAdding pytz 2024.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for python-dateutil==2.9.0.post0\nBest match: python-dateutil 2.9.0.post0\nAdding python-dateutil 2.9.0.post0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for typing-extensions==4.7.1\nBest match: typing-extensions 4.7.1\nAdding typing-extensions 4.7.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for requests-cache==1.1.1\nBest match: requests-cache 1.1.1\nAdding requests-cache 1.1.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for packaging==24.0\nBest match: packaging 24.0\nAdding packaging 24.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for importlib-metadata==6.7.0\nBest match: importlib-metadata 6.7.0\nAdding importlib-metadata 6.7.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for qasync==0.25.0\nBest match: qasync 0.25.0\nAdding qasync 0.25.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for commonmark==0.9.1\nBest match: commonmark 0.9.1\nAdding commonmark 0.9.1 to easy-install.pth file\nInstalling cmark script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for dictdiffer==0.9.0\nBest match: dictdiffer 0.9.0\nAdding dictdiffer 0.9.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for et-xmlfile==1.1.0\nBest match: et-xmlfile 1.1.0\nAdding et-xmlfile 1.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for fonttools==4.38.0\nBest match: fonttools 4.38.0\nAdding fonttools 4.38.0 to easy-install.pth file\nInstalling fonttools script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling pyftmerge script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling pyftsubset script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\nInstalling ttx script to /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/bin\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for pyparsing==3.1.4\nBest match: pyparsing 3.1.4\nAdding pyparsing 3.1.4 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for Pillow==9.5.0\nBest match: Pillow 9.5.0\nAdding Pillow 9.5.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for kiwisolver==1.4.5\nBest match: kiwisolver 1.4.5\nAdding kiwisolver 1.4.5 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for cycler==0.11.0\nBest match: cycler 0.11.0\nAdding cycler 0.11.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for jaraco.classes==3.2.3\nBest match: jaraco.classes 3.2.3\nAdding jaraco.classes 3.2.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for importlib-resources==5.12.0\nBest match: importlib-resources 5.12.0\nAdding importlib-resources 5.12.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for jeepney==0.8.0\nBest match: jeepney 0.8.0\nAdding jeepney 0.8.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for SecretStorage==3.3.3\nBest match: SecretStorage 3.3.3\nAdding SecretStorage 3.3.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for httpcore==0.17.3\nBest match: httpcore 0.17.3\nAdding httpcore 0.17.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for sniffio==1.3.1\nBest match: sniffio 1.3.1\nAdding sniffio 1.3.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for six==1.17.0\nBest match: six 1.17.0\nAdding six 1.17.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for cattrs==23.1.2\nBest match: cattrs 23.1.2\nAdding cattrs 23.1.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for attrs==24.2.0\nBest match: attrs 24.2.0\nAdding attrs 24.2.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for url-normalize==1.4.3\nBest match: url-normalize 1.4.3\nAdding url-normalize 1.4.3 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for platformdirs==4.0.0\nBest match: platformdirs 4.0.0\nAdding platformdirs 4.0.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for zipp==3.15.0\nBest match: zipp 3.15.0\nAdding zipp 3.15.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for more-itertools==9.1.0\nBest match: more-itertools 9.1.0\nAdding more-itertools 9.1.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for cryptography==44.0.0\nBest match: cryptography 44.0.0\nAdding cryptography 44.0.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for h11==0.14.0\nBest match: h11 0.14.0\nAdding h11 0.14.0 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for anyio==3.7.1\nBest match: anyio 3.7.1\nAdding anyio 3.7.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for exceptiongroup==1.2.2\nBest match: exceptiongroup 1.2.2\nAdding exceptiongroup 1.2.2 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for cffi==1.15.1\nBest match: cffi 1.15.1\nAdding cffi 1.15.1 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nSearching for pycparser==2.21\nBest match: pycparser 2.21\nAdding pycparser 2.21 to easy-install.pth file\n\nUsing /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages\nFinished processing dependencies for Orange3==3.26.0.dev0+67ed8aa\nVerifying Orange installation...\n✅ Orange installation successful!\nSuccessfully installed orange using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 18 packages in 156ms\nPrepared 18 packages in 34ms\nInstalled 18 packages in 240ms\n + cython==0.29.37\n + exceptiongroup==1.2.2\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + numpy==1.17.5\n + packaging==24.0\n + pluggy==1.2.0\n + pyqt5==5.15.10\n + pyqt5-qt5==5.15.16\n + pyqt5-sip==12.13.0\n + pyqtwebengine==5.15.6\n + pyqtwebengine-qt5==5.15.16\n + pytest==7.4.4\n + setuptools==59.8.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + wheel==0.42.0\n + zipp==3.15.0\nResolved 54 packages in 354ms\nInstalled 47 packages in 862ms\n + anyio==3.7.1\n + baycomp==1.0.3\n + bottleneck==1.4.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + cycler==0.11.0\n + et-xmlfile==1.1.0\n + fonttools==4.38.0\n + h11==0.14.0\n + httpcore==0.17.3\n + httpx==0.24.1\n + idna==3.10\n + importlib-resources==5.12.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + joblib==1.3.2\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + kiwisolver==1.4.5\n + matplotlib==3.5.3\n + more-itertools==9.1.0\n + networkx==2.6.3\n + openpyxl==3.1.3\n + opentsne==1.0.1\n + pandas==1.3.5\n + pillow==9.5.0\n + pip==24.0\n + pycparser==2.21\n + pyparsing==3.1.4\n + python-dateutil==2.9.0.post0\n + python-louvain==0.16\n + pytz==2024.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + scikit-learn==1.0.2\n + scipy==1.7.3\n + secretstorage==3.3.3\n + serverfiles==0.3.1\n + six==1.17.0\n + sniffio==1.3.1\n + threadpoolctl==3.1.0\n + urllib3==2.0.7\n + xlrd==2.0.1\n + xlsxwriter==3.2.0\nResolved 34 packages in 71ms\nInstalled 13 packages in 103ms\n + anyqt==0.2.0\n + attrs==24.2.0\n + cattrs==23.1.2\n + commonmark==0.9.1\n + dictdiffer==0.9.0\n + docutils==0.20.1\n + orange-canvas-core==0.1.35\n + orange-widget-base==4.22.0\n + platformdirs==4.0.0\n + pyqtgraph==0.12.4\n + qasync==0.25.0\n + requests-cache==1.1.1\n + url-normalize==1.4.3\nResolved 2 packages in 9ms\nInstalled 2 packages in 33ms\n + psycopg2==2.9.9\n + pymssql==2.3.2\nAudited 1 package in 2ms\nAudited 2 packages in 3ms\nOrange/classification/_simple_tree.c: In function ‘gain_ratio_c’:\nOrange/classification/_simple_tree.c:63:9: warning: implicit declaration of function ‘qsort_r’; did you mean ‘qsort’? [-Wimplicit-function-declaration]\n   63 |         qsort_r(base, nel, size, compar, thunk)\n      |         ^~~~~~~\nOrange/classification/_simple_tree.c:147:9: note: in expansion of macro ‘QSORT_R_FUNC’\n  147 |         QSORT_R_FUNC(examples, size, sizeof(struct Example), (void*) &compar_attr, compar_examples);\n      |         ^~~~~~~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/classification/_tree_scorers.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_2find_threshold_entropy’:\nOrange/classification/_tree_scorers.c:3115:27: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 3115 |   __pyx_t_7 = ((__pyx_v_N <= __pyx_v_min_leaf) != 0);\n      |                           ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_6find_threshold_MSE’:\nOrange/classification/_tree_scorers.c:4619:27: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 4619 |   __pyx_t_1 = ((__pyx_v_N <= __pyx_v_min_leaf) != 0);\n      |                           ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_8find_binarization_MSE’:\nOrange/classification/_tree_scorers.c:5396:43: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5396 |           for (__pyx_t_22 = 0; __pyx_t_22 < __pyx_t_21; __pyx_t_22+=1) {\n      |                                           ^\nOrange/classification/_tree_scorers.c:5517:39: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5517 |           __pyx_t_27 = ((__pyx_v_left >= __pyx_v_min_leaf) != 0);\n      |                                       ^~\nOrange/classification/_tree_scorers.c:5523:53: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 5523 |           __pyx_t_27 = (((__pyx_v_N - __pyx_v_left) >= __pyx_v_min_leaf) != 0);\n      |                                                     ^~\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_14compute_predictions_csr’:\nOrange/classification/_tree_scorers.c:7051:41: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 7051 |         for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) {\n      |                                         ^\nOrange/classification/_tree_scorers.c: In function ‘__pyx_pf_6Orange_14classification_13_tree_scorers_16compute_predictions_csc’:\nOrange/classification/_tree_scorers.c:7741:41: warning: comparison of integer expressions of different signedness: ‘unsigned int’ and ‘int’ [-Wsign-compare]\n 7741 |         for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) {\n      |                                         ^\nOrange/classification/_tree_scorers.c:7801:143: warning: comparison of integer expressions of different signedness: ‘__pyx_t_5numpy_int32_t’ {aka ‘int’} and ‘unsigned int’ [-Wsign-compare]\n 7801 |               __pyx_t_17 = (((*((__pyx_t_5numpy_int32_t *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_18 * __pyx_v_indices.strides[0]) ))) != __pyx_v_i) != 0);\n      |                                                                                                                                               ^~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/data/_valuecount.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/data/_contingency.c:598:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/data/_io.c: In function ‘__pyx_pf_6Orange_4data_3_io_sparse_read_float.constprop’:\nOrange/data/_io.c:3881:24: warning: ‘__pyx_v_decs’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3881 |           __pyx_v_decs = (__pyx_v_decs / 10.0);\n      |           ~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:4070:24: warning: ‘__pyx_v_value’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4070 |           __pyx_t_10 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 221, __pyx_L5_error)\n      |                        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:480:40: warning: ‘__pyx_v_attr_index’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n  480 |   #define PyInt_FromLong               PyLong_FromLong\n      |                                        ^~~~~~~~~~~~~~~\nOrange/data/_io.c:1301:7: note: ‘__pyx_v_attr_index’ was declared here\n 1301 |   int __pyx_v_attr_index;\n      |       ^~~~~~~~~~~~~~~~~~\nOrange/data/_io.c:2935:25: warning: ‘__pyx_v_atomp’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 2935 |           __pyx_v_atomp = (__pyx_v_atomp + 1);\n      |           ~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/distance/_distance.c:598:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/preprocess/_discretize.c:567:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/preprocess/_relieff.cpp:581:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nIn file included from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarraytypes.h:1830,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/ndarrayobject.h:12,\n                 from /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/arrayobject.h:4,\n                 from Orange/projection/_som.c:598:\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h:17:2: warning: #warning \"Using deprecated NumPy API, disable it with \" \"#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION\" [-Wcpp]\n   17 | #warning \"Using deprecated NumPy API, disable it with \" \\\n      |  ^~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_5update_hex’:\nOrange/projection/_som.c:3655:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3655 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:3654:7: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3654 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_7get_winners_sparse’:\nOrange/projection/_som.c:4547:223: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4547 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_38, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_39, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_y;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:4114:7: note: ‘__pyx_v_win_y’ was declared here\n 4114 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:4536:223: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4536 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_36, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_37, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_x;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:4113:7: note: ‘__pyx_v_win_x’ was declared here\n 4113 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pf_6Orange_10projection_4_som_8update_sparse.constprop’:\nOrange/projection/_som.c:4746:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4746 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:4745:7: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 4745 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_3update’:\nOrange/projection/_som.c:3141:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 3141 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pw_6Orange_10projection_4_som_1get_winners’:\nOrange/projection/_som.c:2964:223: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 2964 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_27, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_28, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_y;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:2665:7: note: ‘__pyx_v_win_y’ was declared here\n 2665 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:2953:223: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 2953 |           *__Pyx_BufPtrStrided2d(__pyx_t_5numpy_int16_t *, __pyx_pybuffernd_winners.rcbuffer->pybuffer.buf, __pyx_t_25, __pyx_pybuffernd_winners.diminfo[0].strides, __pyx_t_26, __pyx_pybuffernd_winners.diminfo[1].strides) = __pyx_v_win_x;\n      |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~\nOrange/projection/_som.c:2664:7: note: ‘__pyx_v_win_x’ was declared here\n 2664 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c: In function ‘__pyx_pf_6Orange_10projection_4_som_10update_sparse_hex.constprop’:\nOrange/projection/_som.c:5515:7: warning: ‘__pyx_v_win_y’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 5515 |   int __pyx_v_win_y;\n      |       ^~~~~~~~~~~~~\nOrange/projection/_som.c:5514:7: warning: ‘__pyx_v_win_x’ may be used uninitialized in this function [-Wmaybe-uninitialized]\n 5514 |   int __pyx_v_win_x;\n      |       ^~~~~~~~~~~~~\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n  setuptools.SetuptoolsDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/setuptools/command/easy_install.py:159: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools.\n  EasyInstallDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n  setuptools.SetuptoolsDeprecationWarning,\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/.venv/lib/python3.7/site-packages/setuptools/command/egg_info.py:628: SetuptoolsDeprecationWarning: Custom 'build_py' does not implement 'get_data_files_without_manifest'.\nPlease extend command classes from setuptools instead of distutils.\n  SetuptoolsDeprecationWarning\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e\nplugins: anyio-3.7.1\ncollected 35 items\n\nr2e_tests/test_1.py sssE.....F....................FF...                  [100%]\n\n==================================== ERRORS ====================================\n_______________________ ERROR at setup of test_filename ________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/Orange/tests/__init__.py, line 65\n  def test_filename(path):\nE       fixture 'path' not found\n>       available fixtures: anyio_backend, anyio_backend_name, anyio_backend_options, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/Orange/tests/__init__.py:65\n=================================== FAILURES ===================================\n_______________ TestOWTestAndScore.test_comparison_binary_score ________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_comparison_binary_score>\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n    \n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n>           _, kwargs = f1mock.call_args\nE           TypeError: cannot unpack non-iterable NoneType object\n\nr2e_tests/test_1.py:563: TypeError\n_______________________ TestOWTestAndScore.test_summary ________________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_summary>\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n    \n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n    \n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n    \n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n>       self.assertEqual(info._StateInfo__output_summary.brief, \"\")\nE       AssertionError: '-' != ''\nE       - -\nE       +\n\nr2e_tests/test_1.py:672: AssertionError\n___________________ TestOWTestAndScore.test_target_changing ____________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_target_changing>\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n    \n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n    \n        average_auc = float(model.item(0, 3).text())\n    \n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n    \n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n    \n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n    \n>       self.assertGreater(average_auc, versicolor_auc)\nE       AssertionError: 0.9875333333333333 not greater than 0.9875333333333333\n\nr2e_tests/test_1.py:263: AssertionError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_CrossValidationByFeature\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_addon_scorers\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_basic\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_bad_scores\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_bad_slots\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_requires_cv\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_requires_multiple_models\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_data_errors\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_fill_table\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_image_export\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_memory_error\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_migrate_removes_invalid_contexts\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_minimum_size\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_more_learners\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_msg_base_class\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_nan_on_comparison\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_no_pregressbar_warning\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_one_class_value\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_resort_on_data_change\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_constant\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_cross_validation\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_advanced\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_bad\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_bad2\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_overfitted\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_testOnTest\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_testOnTest_incompatible_domain\nPASSED r2e_tests/test_1.py::TestHelpers::test_results_one_vs_rest\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:244: image exporting not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:238: minimum size not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:250: msg base class not tested as .widget was not set\nERROR r2e_tests/test_1.py::test_filename\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_binary_score\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_summary - AssertionError...\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_target_changing - Assert...\n=============== 3 failed, 28 passed, 3 skipped, 1 error in 7.91s ===============\nWidgets left: 0    Max widgets: 4762 \n\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e\nplugins: anyio-3.7.1\ncollected 35 items\n\nr2e_tests/test_1.py sssE.....F..F.........F.......FF...                  [100%]\n\n==================================== ERRORS ====================================\n_______________________ ERROR at setup of test_filename ________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/Orange/tests/__init__.py, line 65\n  def test_filename(path):\nE       fixture 'path' not found\n>       available fixtures: anyio_backend, anyio_backend_name, anyio_backend_options, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/orange3_67ed8aa9c709cc53f3cd2b05ef0e08b64bf3b30e/Orange/tests/__init__.py:65\n=================================== FAILURES ===================================\n_______________ TestOWTestAndScore.test_comparison_binary_score ________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_comparison_binary_score>\n\n    def test_comparison_binary_score(self):\n        # false warning at call_arg.kwargs\n        # pylint: disable=unpacking-non-sequence\n        w = self.widget\n        self._set_three_majorities()\n        self._set_comparison_score(\"F1\")\n        f1mock = Mock(wraps=scoring.F1)\n    \n        iris = Table(\"iris\")\n        with patch.object(scoring.F1, \"compute_score\", f1mock):\n            simulate.combobox_activate_item(w.controls.class_selection,\n                                            iris.domain.class_var.values[1])\n>           _, kwargs = f1mock.call_args\nE           TypeError: cannot unpack non-iterable NoneType object\n\nr2e_tests/test_1.py:563: TypeError\n_____________________ TestOWTestAndScore.test_data_errors ______________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_data_errors>\n\n    def test_data_errors(self):\n        \"\"\" Test all data_errors \"\"\"\n        def assertErrorShown(data, is_shown, message):\n            self.send_signal(\"Data\", data)\n            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\n            self.assertEqual(message, str(self.widget.Error.train_data_error))\n    \n        data = Table(\"iris\")[::30]\n        data.Y[:] = np.nan\n    \n        iris_empty_x = Table.from_table(\n            Domain([], data.domain.class_var), Table(\"iris\")\n        )\n    \n        for data, is_shown, message in zip(\n                [None, data, Table(\"iris\")[:30], iris_empty_x, data[:0]],\n                [False, True, True, True, True],\n                [\n                    \"\",\n                    \"Target variable has no values.\",\n                    \"Target variable has only one value.\",\n                    \"Data has no features to learn from.\",\n                    \"Train dataset is empty.\"\n                ]\n    \n        ):\n>           assertErrorShown(data, is_shown, message)\n\nr2e_tests/test_1.py:202: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\ndata = None, is_shown = False, message = ''\n\n    def assertErrorShown(data, is_shown, message):\n        self.send_signal(\"Data\", data)\n>       self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())\nE       AttributeError: 'Error' object has no attribute 'train_data_error'\n\nr2e_tests/test_1.py:180: AttributeError\n___________________ TestOWTestAndScore.test_one_class_value ____________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_one_class_value>\n\n    def test_one_class_value(self):\n        \"\"\"\n        Data with a class with one value causes widget to crash when that value\n        is selected.\n        GH-2351\n        \"\"\"\n        table = Table.from_list(\n            Domain(\n                [ContinuousVariable(\"a\"), ContinuousVariable(\"b\")],\n                [DiscreteVariable(\"c\", values=(\"y\", ))]),\n            list(zip(\n                [42.48, 16.84, 15.23, 23.8],\n                [1., 2., 3., 4.],\n                \"yyyy\"))\n        )\n        self.widget.n_folds = 0\n>       self.assertFalse(self.widget.Error.train_data_error.is_shown())\nE       AttributeError: 'Error' object has no attribute 'train_data_error'\n\nr2e_tests/test_1.py:171: AttributeError\n_______________________ TestOWTestAndScore.test_summary ________________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_summary>\n\n    def test_summary(self):\n        \"\"\"Check if the status bar updates when data is on input\"\"\"\n        iris = Table(\"iris\")\n        train, test = iris[:120], iris[120:]\n        info = self.widget.info\n        no_input, no_output = \"No data on input\", \"No data on output\"\n    \n        self.send_signal(self.widget.Inputs.train_data, train)\n        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)\n        summary, details = f\"{len(train)}\", format_summary_details(train)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n    \n        self.send_signal(self.widget.Inputs.test_data, test)\n        summary = f\"{len(train)}, {len(test)}\"\n        details = format_multiple_summaries([(\"Data\", train), (\"Test data\", test)])\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n        output = self.get_output(self.widget.Outputs.predictions)\n        summary, details = f\"{len(output)}\", format_summary_details(output)\n        self.assertEqual(info._StateInfo__output_summary.brief, summary)\n        self.assertEqual(info._StateInfo__output_summary.details, details)\n    \n        self.send_signal(self.widget.Inputs.train_data, None)\n        summary, details = f\"{len(test)}\", format_summary_details(test)\n        self.assertEqual(info._StateInfo__input_summary.brief, summary)\n        self.assertEqual(info._StateInfo__input_summary.details, details)\n>       self.assertEqual(info._StateInfo__output_summary.brief, \"\")\nE       AssertionError: '-' != ''\nE       - -\nE       +\n\nr2e_tests/test_1.py:672: AssertionError\n___________________ TestOWTestAndScore.test_target_changing ____________________\n\nself = <r2e_tests.test_1.TestOWTestAndScore testMethod=test_target_changing>\n\n    def test_target_changing(self):\n        data = Table(\"iris\")\n        w = self.widget  #: OWTestAndScore\n        model = w.score_table.model\n    \n        w.n_folds = 2\n        self.send_signal(self.widget.Inputs.train_data, data)\n        self.send_signal(self.widget.Inputs.learner,\n                         LogisticRegressionLearner(), 0, wait=5000)\n    \n        average_auc = float(model.item(0, 3).text())\n    \n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-setosa\")\n        setosa_auc = float(model.item(0, 3).text())\n    \n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-versicolor\")\n        versicolor_auc = float(model.item(0, 3).text())\n    \n        simulate.combobox_activate_item(w.controls.class_selection, \"Iris-virginica\")\n        virginica_auc = float(model.item(0, 3).text())\n    \n>       self.assertGreater(average_auc, versicolor_auc)\nE       AssertionError: 0.9875333333333333 not greater than 0.9875333333333333\n\nr2e_tests/test_1.py:263: AssertionError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_CrossValidationByFeature\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_addon_scorers\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_basic\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_bad_scores\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_bad_slots\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_requires_cv\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_requires_multiple_models\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_fill_table\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_image_export\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_memory_error\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_migrate_removes_invalid_contexts\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_minimum_size\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_more_learners\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_msg_base_class\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_nan_on_comparison\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_no_pregressbar_warning\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_resort_on_data_change\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_constant\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_cross_validation\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_advanced\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_bad\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_bad2\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_scores_log_reg_overfitted\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_testOnTest\nPASSED r2e_tests/test_1.py::TestOWTestAndScore::test_testOnTest_incompatible_domain\nPASSED r2e_tests/test_1.py::TestHelpers::test_results_one_vs_rest\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:244: image exporting not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:238: minimum size not tested as .widget was not set\nSKIPPED [1] .venv/lib/python3.7/site-packages/orangewidget/tests/base.py:250: msg base class not tested as .widget was not set\nERROR r2e_tests/test_1.py::test_filename\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_comparison_binary_score\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_data_errors - AttributeE...\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_one_class_value - Attrib...\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_summary - AssertionError...\nFAILED r2e_tests/test_1.py::TestOWTestAndScore::test_target_changing - Assert...\n=============== 5 failed, 26 passed, 3 skipped, 1 error in 5.82s ===============\nWidgets left: 0    Max widgets: 4762 \n\n",
    "old_commit_res_stderr": ""
}
Orange/widgets/evaluate/owtestandscore.py Orange/widgets/evaluate/tests/test_owtestandscore.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
**Analysis**

- **Key Parts:**
  - **Commit Message:** `[FIX] Test and Score: Improve data errors.`
  - **Code Changes:** Consolidated multiple error messages into a single `train_data_error` with dynamic messages.
  - **Test Results:** Previously, tests `test_data_errors` and `test_one_class_value` failed with `AttributeError` because `train_data_error` did not exist.
  
- **Main Problem:** The `Error` class in `OWTestAndScore` lacks the `train_data_error` attribute, causing AttributeErrors when the widget tries to handle training data errors.

- **Expected Behavior:** The widget should handle and display training data errors using the `train_data_error` attribute without raising errors.

- **Actual Behavior:** Attempting to access `train_data_error` results in an `AttributeError`, preventing proper error handling.

**GitHub Issue**

[ISSUE]

**Title:** AttributeError: 'Error' object has no attribute 'train_data_error' when handling training data errors

**Description**

When the `OWTestAndScore` widget processes training data, it attempts to access the `train_data_error` attribute to display relevant error messages. However, this attribute does not exist in the `Error` class, leading to an AttributeError.

**Example Code**

```python
def test_handle_empty_train_data():
    data = None
    widget.Error.train_data_error.clear()
    if data is not None and not data:
        widget.Error.train_data_error("Train dataset is empty.")
```

**Expected Behavior**

The widget should successfully handle training data errors by accessing the `train_data_error` attribute and displaying the appropriate error message without raising any exceptions.

**Actual Behavior**

Accessing `train_data_error` results in the following error:

```
AttributeError: 'Error' object has no attribute 'train_data_error'
```

This prevents the widget from properly notifying users about issues with the training dataset.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### Orange/widgets/evaluate/owtestandscore.py
# pylint doesn't understand the Settings magic
# pylint: disable=invalid-sequence-index
# pylint: disable=too-many-lines,too-many-instance-attributes
import abc
import enum
import logging
import traceback
import copy
from functools import partial, reduce

from concurrent.futures import Future
from collections import OrderedDict, namedtuple
from itertools import count
from typing import Any, Optional, List, Dict, Callable

import numpy as np
import baycomp

from AnyQt import QtGui
from AnyQt.QtCore import Qt, QSize, QThread
from AnyQt.QtCore import pyqtSlot as Slot
from AnyQt.QtGui import QStandardItem, QDoubleValidator
from AnyQt.QtWidgets import QHeaderView, QTableWidget, QLabel, QApplication

from Orange.base import Learner
import Orange.classification
from Orange.data import Table, DiscreteVariable
from Orange.data.table import DomainTransformationError
from Orange.data.filter import HasClass
from Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT
import Orange.evaluation
from Orange.evaluation import Results
from Orange.preprocess.preprocess import Preprocess
import Orange.regression
from Orange.widgets import gui, settings, widget
from Orange.widgets.evaluate.utils import \
    usable_scorers, ScoreTable, learner_name, scorer_caller
from Orange.widgets.utils.itemmodels import DomainModel
from Orange.widgets.utils.widgetpreview import WidgetPreview
from Orange.widgets.utils.concurrent import ThreadExecutor, TaskState
from Orange.widgets.utils.state_summary import (format_multiple_summaries,
                                                format_summary_details)
from Orange.widgets.widget import OWWidget, Msg, Input, Output

log = logging.getLogger(__name__)

InputLearner = namedtuple(
    "InputLearner",
    ["learner",  # :: Orange.base.Learner
     "results",  # :: Option[Try[Orange.evaluation.Results]]
     "stats"]    # :: Option[Sequence[Try[float]]]
)


class Try(abc.ABC):
    """Try to walk in a Turing tar pit."""

    class Success:
        """Data type for instance constructed on success"""
        __slots__ = ("__value",)
#         __bool__ = lambda self: True
        success = property(lambda self: True)
        value = property(lambda self: self.__value)

        def __init__(self, value):
            self.__value = value

        def __getnewargs__(self):
            return (self.value,)

        def __repr__(self):
            return "{}({!r})".format(self.__class__.__qualname__,
                                     self.value)

        def map(self, fn):
            return Try(lambda: fn(self.value))

    class Fail:
        """Data type for instance constructed on fail"""
        __slots__ = ("__exception", )
#         __bool__ = lambda self: False
        success = property(lambda self: False)
        exception = property(lambda self: self.__exception)

        def __init__(self, exception):
            self.__exception = exception

        def __getnewargs__(self):
            return (self.exception, )

        def __repr__(self):
            return "{}({!r})".format(self.__class__.__qualname__,
                                     self.exception)

        def map(self, _fn):
            return self

    def __new__(cls, f, *args, **kwargs):
        try:
            rval = f(*args, **kwargs)
        except BaseException as ex:
            # IMPORANT: Will capture traceback in ex.__traceback__
            return Try.Fail(ex)
        else:
            return Try.Success(rval)

Try.register(Try.Success)
Try.register(Try.Fail)


def raise_(exc):
    raise exc

Try.register = lambda cls: raise_(TypeError())


class State(enum.Enum):
    """
    OWTestAndScore's runtime state.
    """
    #: No or insufficient input (i.e. no data or no learners)
    Waiting = "Waiting"
    #: Executing/running the evaluations
    Running = "Running"
    #: Finished running evaluations (success or error)
    Done = "Done"
    #: Evaluation cancelled
    Cancelled = "Cancelled"


class OWTestAndScore(OWWidget):
    name = "Test and Score"
    description = "Cross-validation accuracy estimation."
    icon = "icons/TestLearners1.svg"
    priority = 100
    keywords = ['Cross Validation', 'CV']
    replaces = ["Orange.widgets.evaluate.owtestlearners.OWTestLearners"]

    class Inputs:
        train_data = Input("Data", Table, default=True)
        test_data = Input("Test Data", Table)
        learner = Input("Learner", Learner, multiple=True)
        preprocessor = Input("Preprocessor", Preprocess)

    class Outputs:
        predictions = Output("Predictions", Table)
        evaluations_results = Output("Evaluation Results", Results)

    settings_version = 3
    UserAdviceMessages = [
        widget.Message(
            "Click on the table header to select shown columns",
            "click_header")]

    settingsHandler = settings.PerfectDomainContextHandler()
    score_table = settings.SettingProvider(ScoreTable)

    #: Resampling/testing types
    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \
        = 0, 1, 2, 3, 4, 5
    #: Numbers of folds
    NFolds = [2, 3, 5, 10, 20]
    #: Number of repetitions
    NRepeats = [2, 3, 5, 10, 20, 50, 100]
    #: Sample sizes
    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]

    #: Selected resampling type
    resampling = settings.Setting(0)
    #: Number of folds for K-fold cross validation
    n_folds = settings.Setting(3)
    #: Stratified sampling for K-fold
    cv_stratified = settings.Setting(True)
    #: Number of repeats for ShuffleSplit sampling
    n_repeats = settings.Setting(3)
    #: ShuffleSplit sample size
    sample_size = settings.Setting(9)
    #: Stratified sampling for Random Sampling
    shuffle_stratified = settings.Setting(True)
    # CV where nr. of feature values determines nr. of folds
    fold_feature = settings.ContextSetting(None)
    fold_feature_selected = settings.ContextSetting(False)

    use_rope = settings.Setting(False)
    rope = settings.Setting(0.1)
    comparison_criterion = settings.Setting(0, schema_only=True)

    TARGET_AVERAGE = "(Average over classes)"
    class_selection = settings.ContextSetting(TARGET_AVERAGE)

    class Error(OWWidget.Error):
        train_data_empty = Msg("Train dataset is empty.")
        test_data_empty = Msg("Test dataset is empty.")
        class_required = Msg("Train data input requires a target variable.")
        too_many_classes = Msg("Too many target variables.")
        class_required_test = Msg("Test data input requires a target variable.")
        too_many_folds = Msg("Number of folds exceeds the data size")
        class_inconsistent = Msg("Test and train datasets "
                                 "have different target variables.")
        memory_error = Msg("Not enough memory.")
        no_class_values = Msg("Target variable has no values.")
        only_one_class_var_value = Msg("Target variable has only one value.")
        test_data_incompatible = Msg(
            "Test data may be incompatible with train data.")

    class Warning(OWWidget.Warning):
        missing_data = \
            Msg("Instances with unknown target values were removed from{}data.")
        test_data_missing = Msg("Missing separate test data input.")
        scores_not_computed = Msg("Some scores could not be computed.")
        test_data_unused = Msg("Test data is present but unused. "
                               "Select 'Test on test data' to use it.")

    class Information(OWWidget.Information):
        data_sampled = Msg("Train data has been sampled")
        test_data_sampled = Msg("Test data has been sampled")
        test_data_transformed = Msg(
            "Test data has been transformed to match the train data.")

    def __init__(self):
        super().__init__()

        self.data = None
        self.test_data = None
        self.preprocessor = None
        self.train_data_missing_vals = False
        self.test_data_missing_vals = False
        self.scorers = []
        self.__pending_comparison_criterion = self.comparison_criterion

        #: An Ordered dictionary with current inputs and their testing results.
        self.learners = OrderedDict()  # type: Dict[Any, Input]

        self.__state = State.Waiting
        # Do we need to [re]test any learners, set by _invalidate and
        # cleared by __update
        self.__needupdate = False
        self.__task = None  # type: Optional[TaskState]
        self.__executor = ThreadExecutor()

        self.info.set_input_summary(self.info.NoInput)
        self.info.set_output_summary(self.info.NoOutput)

        sbox = gui.vBox(self.controlArea, "Sampling")
        rbox = gui.radioButtons(
            sbox, self, "resampling", callback=self._param_changed)

        gui.appendRadioButton(rbox, "Cross validation")
        ibox = gui.indentedBox(rbox)
        gui.comboBox(
            ibox, self, "n_folds", label="Number of folds: ",
            items=[str(x) for x in self.NFolds],
            orientation=Qt.Horizontal, callback=self.kfold_changed)
        gui.checkBox(
            ibox, self, "cv_stratified", "Stratified",
            callback=self.kfold_changed)
        gui.appendRadioButton(rbox, "Cross validation by feature")
        ibox = gui.indentedBox(rbox)
        self.feature_model = DomainModel(
            order=DomainModel.METAS, valid_types=DiscreteVariable)
        self.features_combo = gui.comboBox(
            ibox, self, "fold_feature", model=self.feature_model,
            orientation=Qt.Horizontal, searchable=True,
            callback=self.fold_feature_changed)

        gui.appendRadioButton(rbox, "Random sampling")
        ibox = gui.indentedBox(rbox)
        gui.comboBox(
            ibox, self, "n_repeats", label="Repeat train/test: ",
            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,
            callback=self.shuffle_split_changed
        )
        gui.comboBox(
            ibox, self, "sample_size", label="Training set size: ",
            items=["{} %".format(x) for x in self.SampleSizes],
            orientation=Qt.Horizontal, callback=self.shuffle_split_changed
        )
        gui.checkBox(
            ibox, self, "shuffle_stratified", "Stratified",
            callback=self.shuffle_split_changed)

        gui.appendRadioButton(rbox, "Leave one out")

        gui.appendRadioButton(rbox, "Test on train data")
        gui.appendRadioButton(rbox, "Test on test data")

        self.cbox = gui.vBox(self.controlArea, "Target Class")
        self.class_selection_combo = gui.comboBox(
            self.cbox, self, "class_selection", items=[],
            sendSelectedValue=True, contentsLength=8, searchable=True,
            callback=self._on_target_class_changed
        )

        self.modcompbox = box = gui.vBox(self.controlArea, "Model Comparison")
        gui.comboBox(
            box, self, "comparison_criterion",
            callback=self.update_comparison_table)

        hbox = gui.hBox(box)
        gui.checkBox(hbox, self, "use_rope",
                     "Negligible difference: ",
                     callback=self._on_use_rope_changed)
        gui.lineEdit(hbox, self, "rope", validator=QDoubleValidator(),
                     controlWidth=70, callback=self.update_comparison_table,
                     alignment=Qt.AlignRight)
        self.controls.rope.setEnabled(self.use_rope)

        gui.rubber(self.controlArea)
        self.score_table = ScoreTable(self)
        self.score_table.shownScoresChanged.connect(self.update_stats_model)
        view = self.score_table.view
        view.setSizeAdjustPolicy(view.AdjustToContents)

        box = gui.vBox(self.mainArea, "Evaluation Results")
        box.layout().addWidget(self.score_table.view)

        self.compbox = box = gui.vBox(self.mainArea, box="Model comparison")
        table = self.comparison_table = QTableWidget(
            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,
            selectionMode=QTableWidget.NoSelection)
        table.setSizeAdjustPolicy(table.AdjustToContents)
        header = table.verticalHeader()
        header.setSectionResizeMode(QHeaderView.Fixed)
        header.setSectionsClickable(False)

        header = table.horizontalHeader()
        header.setTextElideMode(Qt.ElideRight)
        header.setDefaultAlignment(Qt.AlignCenter)
        header.setSectionsClickable(False)
        header.setStretchLastSection(False)
        header.setSectionResizeMode(QHeaderView.ResizeToContents)
        avg_width = self.fontMetrics().averageCharWidth()
        header.setMinimumSectionSize(8 * avg_width)
        header.setMaximumSectionSize(15 * avg_width)
        header.setDefaultSectionSize(15 * avg_width)
        box.layout().addWidget(table)
        box.layout().addWidget(QLabel(
            "<small>Table shows probabilities that the score for the model in "
            "the row is higher than that of the model in the column. "
            "Small numbers show the probability that the difference is "
            "negligible.</small>", wordWrap=True))

    @staticmethod
    def sizeHint():
        return QSize(780, 1)

    def _update_controls(self):
        self.fold_feature = None
        self.feature_model.set_domain(None)
        if self.data:
            self.feature_model.set_domain(self.data.domain)
            if self.fold_feature is None and self.feature_model:
                self.fold_feature = self.feature_model[0]
        enabled = bool(self.feature_model)
        self.controls.resampling.buttons[
            OWTestAndScore.FeatureFold].setEnabled(enabled)
        self.features_combo.setEnabled(enabled)
        if self.resampling == OWTestAndScore.FeatureFold and not enabled:
            self.resampling = OWTestAndScore.KFold

    @Inputs.learner
    def set_learner(self, learner, key):
        """
        Set the input `learner` for `key`.

        Parameters
        ----------
        learner : Optional[Orange.base.Learner]
        key : Any
        """
        if key in self.learners and learner is None:
            # Removed
            self._invalidate([key])
            del self.learners[key]
        elif learner is not None:
            self.learners[key] = InputLearner(learner, None, None)
            self._invalidate([key])

    @Inputs.train_data
    def set_train_data(self, data):
        """
        Set the input training dataset.

        Parameters
        ----------
        data : Optional[Orange.data.Table]
        """
        self.cancel()
        self.Information.data_sampled.clear()
        self.Error.train_data_empty.clear()
        self.Error.class_required.clear()
        self.Error.too_many_classes.clear()
        self.Error.no_class_values.clear()
        self.Error.only_one_class_var_value.clear()
        if data is not None and not data:
            self.Error.train_data_empty()
            data = None
        if data:
            conds = [not data.domain.class_vars,
                     len(data.domain.class_vars) > 1,
                     np.isnan(data.Y).all(),
                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]
            errors = [self.Error.class_required,
                      self.Error.too_many_classes,
                      self.Error.no_class_values,
                      self.Error.only_one_class_var_value]
            for cond, error in zip(conds, errors):
                if cond:
                    error()
                    data = None
                    break

        if isinstance(data, SqlTable):
            if data.approx_len() < AUTO_DL_LIMIT:
                data = Table(data)
            else:
                self.Information.data_sampled()
                data_sample = data.sample_time(1, no_cache=True)
                data_sample.download_data(AUTO_DL_LIMIT, partial=True)
                data = Table(data_sample)

        self.train_data_missing_vals = \
            data is not None and np.isnan(data.Y).any()
        if self.train_data_missing_vals or self.test_data_missing_vals:
            self.Warning.missing_data(self._which_missing_data())
            if data:
                data = HasClass()(data)
        else:
            self.Warning.missing_data.clear()

        self.data = data
        self.closeContext()
        self._update_scorers()
        self._update_controls()
        if data is not None:
            self._update_class_selection()
            self.openContext(data.domain)
            if self.fold_feature_selected and bool(self.feature_model):
                self.resampling = OWTestAndScore.FeatureFold
        self._invalidate()

    @Inputs.test_data
    def set_test_data(self, data):
        # type: (Orange.data.Table) -> None
        """
        Set the input separate testing dataset.

        Parameters
        ----------
        data : Optional[Orange.data.Table]
        """
        self.Information.test_data_sampled.clear()
        self.Error.test_data_empty.clear()
        if data is not None and not data:
            self.Error.test_data_empty()
            data = None
        if data and not data.domain.class_var:
            self.Error.class_required_test()
            data = None
        else:
            self.Error.class_required_test.clear()

        if isinstance(data, SqlTable):
            if data.approx_len() < AUTO_DL_LIMIT:
                data = Table(data)
            else:
                self.Information.test_data_sampled()
                data_sample = data.sample_time(1, no_cache=True)
                data_sample.download_data(AUTO_DL_LIMIT, partial=True)
                data = Table(data_sample)

        self.test_data_missing_vals = \
            data is not None and np.isnan(data.Y).any()
        if self.train_data_missing_vals or self.test_data_missing_vals:
            self.Warning.missing_data(self._which_missing_data())
            if data:
                data = HasClass()(data)
        else:
            self.Warning.missing_data.clear()

        self.test_data = data
        if self.resampling == OWTestAndScore.TestOnTest:
            self._invalidate()

    def _which_missing_data(self):
        return {(True, True): " ",  # both, don't specify
                (True, False): " train ",
                (False, True): " test "}[(self.train_data_missing_vals,
                                          self.test_data_missing_vals)]

    # List of scorers shouldn't be retrieved globally, when the module is
    # loading since add-ons could have registered additional scorers.
    # It could have been cached but
    # - we don't gain much with it
    # - it complicates the unit tests
    def _update_scorers(self):
        if self.data and self.data.domain.class_var:
            new_scorers = usable_scorers(self.data.domain.class_var)
        else:
            new_scorers = []
        # Don't unnecessarily reset the combo because this would always reset
        # comparison_criterion; we also set it explicitly, though, for clarity
        if new_scorers != self.scorers:
            self.scorers = new_scorers
            combo = self.controls.comparison_criterion
            combo.clear()
            combo.addItems([scorer.long_name or scorer.name
                            for scorer in self.scorers])
            self.comparison_criterion = 0
        if self.__pending_comparison_criterion is not None:
            # Check for the unlikely case that some scorers have been removed
            # from modules
            if self.__pending_comparison_criterion < len(self.scorers):
                self.comparison_criterion = self.__pending_comparison_criterion
            self.__pending_comparison_criterion = None
        self._update_compbox_title()

    def _update_compbox_title(self):
        criterion = self.comparison_criterion
        if criterion < len(self.scorers):
            scorer = self.scorers[criterion]()
            self.compbox.setTitle(f"Model Comparison by {scorer.name}")
        else:
            self.compbox.setTitle(f"Model Comparison")

    @Inputs.preprocessor
    def set_preprocessor(self, preproc):
        """
        Set the input preprocessor to apply on the training data.
        """
        self.preprocessor = preproc
        self._invalidate()

    def handleNewSignals(self):
        """Reimplemented from OWWidget.handleNewSignals."""
        self._update_class_selection()
        self.score_table.update_header(self.scorers)
        self._update_view_enabled()
        self.update_stats_model()
        self.set_input_summary()
        if self.__needupdate:
            self.__update()

    def set_input_summary(self):
        summary, details, kwargs = self.info.NoInput, "", {}
        if self.data and self.test_data:
            summary = f"{self.info.format_number(len(self.data))}," \
                      f" {self.info.format_number(len(self.test_data))}"
            details = format_multiple_summaries([
                ("Data", self.data), ("Test data", self.test_data)
            ])
            kwargs = {"format": Qt.RichText}
        elif self.data and not self.test_data:
            summary, details = len(self.data), format_summary_details(self.data)
        elif self.test_data and not self.data:
            summary = len(self.test_data)
            details = format_summary_details(self.test_data)
        self.info.set_input_summary(summary, details, **kwargs)

    def kfold_changed(self):
        self.resampling = OWTestAndScore.KFold
        self._param_changed()

    def fold_feature_changed(self):
        self.resampling = OWTestAndScore.FeatureFold
        self._param_changed()

    def shuffle_split_changed(self):
        self.resampling = OWTestAndScore.ShuffleSplit
        self._param_changed()

    def _param_changed(self):
        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)
        self._update_view_enabled()
        self._invalidate()
        self.__update()

    def _update_view_enabled(self):
        self.comparison_table.setEnabled(
            self.resampling == OWTestAndScore.KFold
            and len(self.learners) > 1
            and self.data is not None)
        self.score_table.view.setEnabled(
            self.data is not None)

    def update_stats_model(self):
        # Update the results_model with up to date scores.
        # Note: The target class specific scores (if requested) are
        # computed as needed in this method.
        model = self.score_table.model
        # clear the table model, but preserving the header labels
        for r in reversed(range(model.rowCount())):
            model.takeRow(r)

        target_index = None
        if self.data is not None:
            class_var = self.data.domain.class_var
            if self.data.domain.has_discrete_class and \
                            self.class_selection != self.TARGET_AVERAGE:
                target_index = class_var.values.index(self.class_selection)
        else:
            class_var = None

        errors = []
        has_missing_scores = False

        names = []
        for key, slot in self.learners.items():
            name = learner_name(slot.learner)
            names.append(name)
            head = QStandardItem(name)
            head.setData(key, Qt.UserRole)
            results = slot.results
            if results is not None and results.success:
                train = QStandardItem("{:.3f}".format(results.value.train_time))
                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
                train.setData(key, Qt.UserRole)
                test = QStandardItem("{:.3f}".format(results.value.test_time))
                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
                test.setData(key, Qt.UserRole)
                row = [head, train, test]
            else:
                row = [head]
            if isinstance(results, Try.Fail):
                head.setToolTip(str(results.exception))
                head.setText("{} (error)".format(name))
                head.setForeground(QtGui.QBrush(Qt.red))
                if isinstance(results.exception, DomainTransformationError) \
                        and self.resampling == self.TestOnTest:
                    self.Error.test_data_incompatible()
                    self.Information.test_data_transformed.clear()
                else:
                    errors.append("{name} failed with error:\n"
                                  "{exc.__class__.__name__}: {exc!s}"
                                  .format(name=name, exc=slot.results.exception)
                                  )

            if class_var is not None and class_var.is_discrete and \
                    target_index is not None:
                if slot.results is not None and slot.results.success:
                    ovr_results = results_one_vs_rest(
                        slot.results.value, target_index)

                    # Cell variable is used immediatelly, it's not stored
                    # pylint: disable=cell-var-from-loop
                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))
                             for scorer in self.scorers]
                else:
                    stats = None
            else:
                stats = slot.stats

            if stats is not None:
                for stat, scorer in zip(stats, self.scorers):
                    item = QStandardItem()
                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
                    if stat.success:
                        item.setData(float(stat.value[0]), Qt.DisplayRole)
                    else:
                        item.setToolTip(str(stat.exception))
                        if scorer.name in self.score_table.shown_scores:
                            has_missing_scores = True
                    row.append(item)

            model.appendRow(row)

        # Resort rows based on current sorting
        header = self.score_table.view.horizontalHeader()
        model.sort(
            header.sortIndicatorSection(),
            header.sortIndicatorOrder()
        )
        self._set_comparison_headers(names)

        self.error("\n".join(errors), shown=bool(errors))
        self.Warning.scores_not_computed(shown=has_missing_scores)

    def _on_use_rope_changed(self):
        self.controls.rope.setEnabled(self.use_rope)
        self.update_comparison_table()

    def update_comparison_table(self):
        self.comparison_table.clearContents()
        slots = self._successful_slots()
        if not (slots and self.scorers):
            return
        names = [learner_name(slot.learner) for slot in slots]
        self._set_comparison_headers(names)
        if self.resampling == OWTestAndScore.KFold:
            scores = self._scores_by_folds(slots)
            self._fill_table(names, scores)

    def _successful_slots(self):
        model = self.score_table.model
        proxy = self.score_table.sorted_model

        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)
                for row in range(proxy.rowCount()))
        slots = [slot for slot in (self.learners[key] for key in keys)
                 if slot.results is not None and slot.results.success]
        return slots

    def _set_comparison_headers(self, names):
        table = self.comparison_table
        try:
            # Prevent glitching during update
            table.setUpdatesEnabled(False)
            header = table.horizontalHeader()
            if len(names) > 2:
                header.setSectionResizeMode(QHeaderView.Stretch)
            else:
                header.setSectionResizeMode(QHeaderView.Fixed)
            table.setRowCount(len(names))
            table.setColumnCount(len(names))
            table.setVerticalHeaderLabels(names)
            table.setHorizontalHeaderLabels(names)
        finally:
            table.setUpdatesEnabled(True)

    def _scores_by_folds(self, slots):
        scorer = self.scorers[self.comparison_criterion]()
        self._update_compbox_title()
        if scorer.is_binary:
            if self.class_selection != self.TARGET_AVERAGE:
                class_var = self.data.domain.class_var
                target_index = class_var.values.index(self.class_selection)
                kw = dict(target=target_index)
            else:
                kw = dict(average='weighted')
        else:
            kw = {}

        def call_scorer(results):
            def thunked():
                return scorer.scores_by_folds(results.value, **kw).flatten()

            return thunked

        scores = [Try(call_scorer(slot.results)) for slot in slots]
        scores = [score.value if score.success else None for score in scores]
        # `None in scores doesn't work -- these are np.arrays)
        if any(score is None for score in scores):
            self.Warning.scores_not_computed()
        return scores

    def _fill_table(self, names, scores):
        table = self.comparison_table
        for row, row_name, row_scores in zip(count(), names, scores):
            for col, col_name, col_scores in zip(range(row), names, scores):
                if row_scores is None or col_scores is None:
                    continue
                if self.use_rope and self.rope:
                    p0, rope, p1 = baycomp.two_on_single(
                        row_scores, col_scores, self.rope)
                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):
                        self._set_cells_na(table, row, col)
                        continue
                    self._set_cell(table, row, col,
                                   f"{p0:.3f}<br/><small>{rope:.3f}</small>",
                                   f"p({row_name} > {col_name}) = {p0:.3f}\n"
                                   f"p({row_name} = {col_name}) = {rope:.3f}")
                    self._set_cell(table, col, row,
                                   f"{p1:.3f}<br/><small>{rope:.3f}</small>",
                                   f"p({col_name} > {row_name}) = {p1:.3f}\n"
                                   f"p({col_name} = {row_name}) = {rope:.3f}")
                else:
                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)
                    if np.isnan(p0) or np.isnan(p1):
                        self._set_cells_na(table, row, col)
                        continue
                    self._set_cell(table, row, col,
                                   f"{p0:.3f}",
                                   f"p({row_name} > {col_name}) = {p0:.3f}")
                    self._set_cell(table, col, row,
                                   f"{p1:.3f}",
                                   f"p({col_name} > {row_name}) = {p1:.3f}")

    @classmethod
    def _set_cells_na(cls, table, row, col):
        cls._set_cell(table, row, col, "NA", "comparison cannot be computed")
        cls._set_cell(table, col, row, "NA", "comparison cannot be computed")

    @staticmethod
    def _set_cell(table, row, col, label, tooltip):
        item = QLabel(label)
        item.setToolTip(tooltip)
        item.setAlignment(Qt.AlignCenter)
        table.setCellWidget(row, col, item)

    def _update_class_selection(self):
        self.class_selection_combo.setCurrentIndex(-1)
        self.class_selection_combo.clear()
        if not self.data:
            return

        if self.data.domain.has_discrete_class:
            self.cbox.setVisible(True)
            class_var = self.data.domain.class_var
            items = (self.TARGET_AVERAGE, ) + class_var.values
            self.class_selection_combo.addItems(items)

            class_index = 0
            if self.class_selection in class_var.values:
                class_index = class_var.values.index(self.class_selection) + 1

            self.class_selection_combo.setCurrentIndex(class_index)
            self.class_selection = items[class_index]
        else:
            self.cbox.setVisible(False)

    def _on_target_class_changed(self):
        self.update_stats_model()
        self.update_comparison_table()

    def _invalidate(self, which=None):
        self.cancel()
        self.fold_feature_selected = \
            self.resampling == OWTestAndScore.FeatureFold
        # Invalidate learner results for `which` input keys
        # (if None then all learner results are invalidated)
        if which is None:
            which = self.learners.keys()

        model = self.score_table.model
        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)
                         for row in range(model.rowCount())]

        for key in which:
            self.learners[key] = \
                self.learners[key]._replace(results=None, stats=None)

            if key in statmodelkeys:
                row = statmodelkeys.index(key)
                for c in range(1, model.columnCount()):
                    item = model.item(row, c)
                    if item is not None:
                        item.setData(None, Qt.DisplayRole)
                        item.setData(None, Qt.ToolTipRole)

        self.comparison_table.clearContents()

        self.__needupdate = True

    def commit(self):
        """
        Commit the results to output.
        """
        self.Error.memory_error.clear()
        valid = [slot for slot in self.learners.values()
                 if slot.results is not None and slot.results.success]
        combined = None
        predictions = None
        if valid:
            # Evaluation results
            combined = results_merge([slot.results.value for slot in valid])
            combined.learner_names = [learner_name(slot.learner)
                                      for slot in valid]

            # Predictions & Probabilities
            try:
                predictions = combined.get_augmented_data(combined.learner_names)
            except MemoryError:
                self.Error.memory_error()

        summary = len(predictions) if predictions else self.info.NoOutput
        details = format_summary_details(predictions) if predictions else ""
        self.info.set_output_summary(summary, details)

        self.Outputs.evaluations_results.send(combined)
        self.Outputs.predictions.send(predictions)

    def send_report(self):
        """Report on the testing schema and results"""
        if not self.data or not self.learners:
            return
        if self.resampling == self.KFold:
            stratified = 'Stratified ' if self.cv_stratified else ''
            items = [("Sampling type", "{}{}-fold Cross validation".
                      format(stratified, self.NFolds[self.n_folds]))]
        elif self.resampling == self.LeaveOneOut:
            items = [("Sampling type", "Leave one out")]
        elif self.resampling == self.ShuffleSplit:
            stratified = 'Stratified ' if self.shuffle_stratified else ''
            items = [("Sampling type",
                      "{}Shuffle split, {} random samples with {}% data "
                      .format(stratified, self.NRepeats[self.n_repeats],
                              self.SampleSizes[self.sample_size]))]
        elif self.resampling == self.TestOnTrain:
            items = [("Sampling type", "No sampling, test on training data")]
        elif self.resampling == self.TestOnTest:
            items = [("Sampling type", "No sampling, test on testing data")]
        else:
            items = []
        if self.data.domain.has_discrete_class:
            items += [("Target class", self.class_selection.strip("()"))]
        if items:
            self.report_items("Settings", items)
        self.report_table("Scores", self.score_table.view)

    @classmethod
    def migrate_settings(cls, settings_, version):
        if version < 2:
            if settings_["resampling"] > 0:
                settings_["resampling"] += 1
        if version < 3:
            # Older version used an incompatible context handler
            settings_["context_settings"] = [
                c for c in settings_.get("context_settings", ())
                if not hasattr(c, 'classes')]

    @Slot(float)
    def setProgressValue(self, value):
        self.progressBarSet(value)

    def __update(self):
        self.__needupdate = False

        assert self.__task is None or self.__state == State.Running
        if self.__state == State.Running:
            self.cancel()

        self.Warning.test_data_unused.clear()
        self.Error.test_data_incompatible.clear()
        self.Warning.test_data_missing.clear()
        self.Information.test_data_transformed(
            shown=self.resampling == self.TestOnTest
            and self.data is not None
            and self.test_data is not None
            and self.data.domain.attributes != self.test_data.domain.attributes)
        self.warning()
        self.Error.class_inconsistent.clear()
        self.Error.too_many_folds.clear()
        self.error()

        # check preconditions and return early
        if self.data is None:
            self.__state = State.Waiting
            self.commit()
            return
        if not self.learners:
            self.__state = State.Waiting
            self.commit()
            return
        if self.resampling == OWTestAndScore.KFold and \
                len(self.data) < self.NFolds[self.n_folds]:
            self.Error.too_many_folds()
            self.__state = State.Waiting
            self.commit()
            return

        elif self.resampling == OWTestAndScore.TestOnTest:
            if self.test_data is None:
                if not self.Error.test_data_empty.is_shown():
                    self.Warning.test_data_missing()
                self.__state = State.Waiting
                self.commit()
                return
            elif self.test_data.domain.class_var != self.data.domain.class_var:
                self.Error.class_inconsistent()
                self.__state = State.Waiting
                self.commit()
                return

        elif self.test_data is not None:
            self.Warning.test_data_unused()

        rstate = 42
        # items in need of an update
        items = [(key, slot) for key, slot in self.learners.items()
                 if slot.results is None]
        learners = [slot.learner for _, slot in items]

        # deepcopy all learners as they are not thread safe (by virtue of
        # the base API). These will be the effective learner objects tested
        # but will be replaced with the originals on return (see restore
        # learners bellow)
        learners_c = [copy.deepcopy(learner) for learner in learners]

        if self.resampling == OWTestAndScore.TestOnTest:
            test_f = partial(
                Orange.evaluation.TestOnTestData(
                    store_data=True, store_models=True),
                self.data, self.test_data, learners_c, self.preprocessor
            )
        else:
            if self.resampling == OWTestAndScore.KFold:
                sampler = Orange.evaluation.CrossValidation(
                    k=self.NFolds[self.n_folds],
                    random_state=rstate)
            elif self.resampling == OWTestAndScore.FeatureFold:
                sampler = Orange.evaluation.CrossValidationFeature(
                    feature=self.fold_feature)
            elif self.resampling == OWTestAndScore.LeaveOneOut:
                sampler = Orange.evaluation.LeaveOneOut()
            elif self.resampling == OWTestAndScore.ShuffleSplit:
                sampler = Orange.evaluation.ShuffleSplit(
                    n_resamples=self.NRepeats[self.n_repeats],
                    train_size=self.SampleSizes[self.sample_size] / 100,
                    test_size=None,
                    stratified=self.shuffle_stratified,
                    random_state=rstate)
            elif self.resampling == OWTestAndScore.TestOnTrain:
                sampler = Orange.evaluation.TestOnTrainingData(
                    store_models=True)
            else:
                assert False, "self.resampling %s" % self.resampling

            sampler.store_data = True
            test_f = partial(
                sampler, self.data, learners_c, self.preprocessor)

        def replace_learners(evalfunc, *args, **kwargs):
            res = evalfunc(*args, **kwargs)
            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))
            res.learners[:] = learners
            return res

        test_f = partial(replace_learners, test_f)

        self.__submit(test_f)

    def __submit(self, testfunc):
        # type: (Callable[[Callable[[float], None]], Results]) -> None
        """
        Submit a testing function for evaluation

        MUST not be called if an evaluation is already pending/running.
        Cancel the existing task first.

        Parameters
        ----------
        testfunc : Callable[[Callable[float]], Results])
            Must be a callable taking a single `callback` argument and
            returning a Results instance
        """
        assert self.__state != State.Running
        # Setup the task
        task = TaskState()

        def progress_callback(finished):
            if task.is_interruption_requested():
                raise UserInterrupt()
            task.set_progress_value(100 * finished)

        testfunc = partial(testfunc, callback=progress_callback)
        task.start(self.__executor, testfunc)

        task.progress_changed.connect(self.setProgressValue)
        task.watcher.finished.connect(self.__task_complete)

        self.Outputs.evaluations_results.invalidate()
        self.Outputs.predictions.invalidate()
        self.progressBarInit()
        self.setStatusMessage("Running")

        self.__state = State.Running
        self.__task = task

    @Slot(object)
    def __task_complete(self, f: 'Future[Results]'):
        # handle a completed task
        assert self.thread() is QThread.currentThread()
        assert self.__task is not None and self.__task.future is f
        self.progressBarFinished()
        self.setStatusMessage("")
        assert f.done()
        self.__task = None
        self.__state = State.Done
        try:
            results = f.result()    # type: Results
            learners = results.learners  # type: List[Learner]
        except Exception as er:  # pylint: disable=broad-except
            log.exception("testing error (in __task_complete):",
                          exc_info=True)
            self.error("\n".join(traceback.format_exception_only(type(er), er)))
            return

        learner_key = {slot.learner: key for key, slot in
                       self.learners.items()}
        assert all(learner in learner_key for learner in learners)

        # Update the results for individual learners
        class_var = results.domain.class_var
        for learner, result in zip(learners, results.split_by_model()):
            stats = None
            if class_var.is_primitive():
                ex = result.failed[0]
                if ex:
                    stats = [Try.Fail(ex)] * len(self.scorers)
                    result = Try.Fail(ex)
                else:
                    stats = [Try(scorer_caller(scorer, result))
                             for scorer in self.scorers]
                    result = Try.Success(result)
            key = learner_key.get(learner)
            self.learners[key] = \
                self.learners[key]._replace(results=result, stats=stats)

        self.score_table.update_header(self.scorers)
        self.update_stats_model()
        self.update_comparison_table()

        self.commit()

    def cancel(self):
        """
        Cancel the current/pending evaluation (if any).
        """
        if self.__task is not None:
            assert self.__state == State.Running
            self.__state = State.Cancelled
            task, self.__task = self.__task, None
            task.cancel()
            task.progress_changed.disconnect(self.setProgressValue)
            task.watcher.finished.disconnect(self.__task_complete)
            add_task_to_dispose_queue(task)

            self.progressBarFinished()
            self.setStatusMessage("")

    def onDeleteWidget(self):
        self.cancel()
        self.__executor.shutdown(True)
        super().onDeleteWidget()


def add_task_to_dispose_queue(task: TaskState):
    # transfer ownership of task to Qt, and delete it after completion
    # all other signals from task should be disconnected.
    assert task.parent() is None
    app = QApplication.instance()
    task.setParent(app)
    task.watcher.finished.connect(task.deleteLater)


class UserInterrupt(BaseException):
    """
    A BaseException subclass used for cooperative task/thread cancellation
    """


def results_add_by_model(x, y):
    def is_empty(res):
        return (getattr(res, "models", None) is None
                and getattr(res, "row_indices", None) is None)

    if is_empty(x):
        return y
    elif is_empty(y):
        return x

    assert (x.row_indices == y.row_indices).all()
    assert (x.actual == y.actual).all()

    res = Orange.evaluation.Results()
    res.data = x.data
    res.domain = x.domain
    res.row_indices = x.row_indices
    res.folds = x.folds
    res.actual = x.actual
    res.predicted = np.vstack((x.predicted, y.predicted))
    if getattr(x, "probabilities", None) is not None \
            and getattr(y, "probabilities") is not None:
        res.probabilities = np.vstack((x.probabilities, y.probabilities))

    if x.models is not None:
        res.models = np.hstack((x.models, y.models))
    return res


def results_merge(results):
    return reduce(results_add_by_model, results, Orange.evaluation.Results())


def results_one_vs_rest(results, pos_index):
    from Orange.preprocess.transformation import Indicator
    actual = results.actual == pos_index
    predicted = results.predicted == pos_index
    if results.probabilities is not None:
        c = results.probabilities.shape[2]
        assert c >= 2
        neg_indices = [i for i in range(c) if i != pos_index]
        pos_prob = results.probabilities[:, :, [pos_index]]
        neg_prob = np.sum(results.probabilities[:, :, neg_indices],
                          axis=2, keepdims=True)
        probabilities = np.dstack((neg_prob, pos_prob))
    else:
        probabilities = None

    res = Orange.evaluation.Results()
    res.actual = actual
    res.predicted = predicted
    res.folds = results.folds
    res.row_indices = results.row_indices
    res.probabilities = probabilities

    value = results.domain.class_var.values[pos_index]
    class_var = Orange.data.DiscreteVariable(
        "I({}=={})".format(results.domain.class_var.name, value),
        values=("False", "True"),
        compute_value=Indicator(results.domain.class_var, pos_index)
    )
    domain = Orange.data.Domain(
        results.domain.attributes,
        [class_var],
        results.domain.metas
    )
    res.data = None
    res.domain = domain
    return res


if __name__ == "__main__":  # pragma: no cover
    filename = "iris"
    preview_data = Table(filename)
    if preview_data.domain.class_var.is_discrete:
        prev_learners = [lambda data: 1 / 0,
                         Orange.classification.LogisticRegressionLearner(),
                         Orange.classification.MajorityLearner(),
                         Orange.classification.NaiveBayesLearner()]
    else:
        prev_learners = [lambda data: 1 / 0,
                         Orange.regression.MeanLearner(),
                         Orange.regression.KNNRegressionLearner(),
                         Orange.regression.RidgeRegressionLearner()]

    WidgetPreview(OWTestAndScore).run(
        set_train_data=preview_data,
        set_test_data=preview_data,
        set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]
    )

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
**Analysis**

- **Key Parts:**
  - **Commit Message:** `[FIX] Test and Score: Improve data errors.`
  - **Code Changes:** Consolidated multiple error messages into a single `train_data_error` with dynamic messages.
  - **Test Results:** Previously, tests `test_data_errors` and `test_one_class_value` failed with `AttributeError` because `train_data_error` did not exist.
  
- **Main Problem:** The `Error` class in `OWTestAndScore` lacks the `train_data_error` attribute, causing AttributeErrors when the widget tries to handle training data errors.

- **Expected Behavior:** The widget should handle and display training data errors using the `train_data_error` attribute without raising errors.

- **Actual Behavior:** Attempting to access `train_data_error` results in an `AttributeError`, preventing proper error handling.

**GitHub Issue**

[ISSUE]

**Title:** AttributeError: 'Error' object has no attribute 'train_data_error' when handling training data errors

**Description**

When the `OWTestAndScore` widget processes training data, it attempts to access the `train_data_error` attribute to display relevant error messages. However, this attribute does not exist in the `Error` class, leading to an AttributeError.

**Example Code**

```python
def test_handle_empty_train_data():
    data = None
    widget.Error.train_data_error.clear()
    if data is not None and not data:
        widget.Error.train_data_error("Train dataset is empty.")
```

**Expected Behavior**

The widget should successfully handle training data errors by accessing the `train_data_error` attribute and displaying the appropriate error message without raising any exceptions.

**Actual Behavior**

Accessing `train_data_error` results in the following error:

```
AttributeError: 'Error' object has no attribute 'train_data_error'
```

This prevents the widget from properly notifying users about issues with the training dataset.

[/ISSUE]
{
    "TestOWTestAndScore.test_CrossValidationByFeature": "PASSED",
    "TestOWTestAndScore.test_addon_scorers": "PASSED",
    "TestOWTestAndScore.test_basic": "PASSED",
    "TestOWTestAndScore.test_comparison_bad_scores": "PASSED",
    "TestOWTestAndScore.test_comparison_bad_slots": "PASSED",
    "TestOWTestAndScore.test_comparison_requires_cv": "PASSED",
    "TestOWTestAndScore.test_comparison_requires_multiple_models": "PASSED",
    "TestOWTestAndScore.test_data_errors": "PASSED",
    "TestOWTestAndScore.test_fill_table": "PASSED",
    "TestOWTestAndScore.test_image_export": "PASSED",
    "TestOWTestAndScore.test_memory_error": "PASSED",
    "TestOWTestAndScore.test_migrate_removes_invalid_contexts": "PASSED",
    "TestOWTestAndScore.test_minimum_size": "PASSED",
    "TestOWTestAndScore.test_more_learners": "PASSED",
    "TestOWTestAndScore.test_msg_base_class": "PASSED",
    "TestOWTestAndScore.test_nan_on_comparison": "PASSED",
    "TestOWTestAndScore.test_no_pregressbar_warning": "PASSED",
    "TestOWTestAndScore.test_one_class_value": "PASSED",
    "TestOWTestAndScore.test_resort_on_data_change": "PASSED",
    "TestOWTestAndScore.test_scores_constant": "PASSED",
    "TestOWTestAndScore.test_scores_cross_validation": "PASSED",
    "TestOWTestAndScore.test_scores_log_reg_advanced": "PASSED",
    "TestOWTestAndScore.test_scores_log_reg_bad": "PASSED",
    "TestOWTestAndScore.test_scores_log_reg_bad2": "PASSED",
    "TestOWTestAndScore.test_scores_log_reg_overfitted": "PASSED",
    "TestOWTestAndScore.test_testOnTest": "PASSED",
    "TestOWTestAndScore.test_testOnTest_incompatible_domain": "PASSED",
    "TestHelpers.test_results_one_vs_rest": "PASSED",
    "test_filename": "ERROR",
    "TestOWTestAndScore.test_comparison_binary_score": "FAILED",
    "TestOWTestAndScore.test_summary": "FAILED",
    "TestOWTestAndScore.test_target_changing": "FAILED"
}
diff --git a/Orange/widgets/evaluate/owtestandscore.py b/Orange/widgets/evaluate/owtestandscore.py
index 03fb23a3d..bffccdb6a 100644
--- a/Orange/widgets/evaluate/owtestandscore.py
+++ b/Orange/widgets/evaluate/owtestandscore.py
@@ -32,6 +32,7 @@ import Orange.evaluation
 from Orange.evaluation import Results
 from Orange.preprocess.preprocess import Preprocess
 import Orange.regression
+from Orange.statistics.util import unique
 from Orange.widgets import gui, settings, widget
 from Orange.widgets.evaluate.utils import \
     usable_scorers, ScoreTable, learner_name, scorer_caller
@@ -189,19 +190,15 @@ class OWTestAndScore(OWWidget):
     class_selection = settings.ContextSetting(TARGET_AVERAGE)
 
     class Error(OWWidget.Error):
-        train_data_empty = Msg("Train dataset is empty.")
         test_data_empty = Msg("Test dataset is empty.")
-        class_required = Msg("Train data input requires a target variable.")
-        too_many_classes = Msg("Too many target variables.")
         class_required_test = Msg("Test data input requires a target variable.")
         too_many_folds = Msg("Number of folds exceeds the data size")
         class_inconsistent = Msg("Test and train datasets "
                                  "have different target variables.")
         memory_error = Msg("Not enough memory.")
-        no_class_values = Msg("Target variable has no values.")
-        only_one_class_var_value = Msg("Target variable has only one value.")
         test_data_incompatible = Msg(
             "Test data may be incompatible with train data.")
+        train_data_error = Msg("{}")
 
     class Warning(OWWidget.Warning):
         missing_data = \
@@ -387,26 +384,27 @@ class OWTestAndScore(OWWidget):
         """
         self.cancel()
         self.Information.data_sampled.clear()
-        self.Error.train_data_empty.clear()
-        self.Error.class_required.clear()
-        self.Error.too_many_classes.clear()
-        self.Error.no_class_values.clear()
-        self.Error.only_one_class_var_value.clear()
-        if data is not None and not data:
-            self.Error.train_data_empty()
-            data = None
-        if data:
-            conds = [not data.domain.class_vars,
-                     len(data.domain.class_vars) > 1,
-                     np.isnan(data.Y).all(),
-                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]
-            errors = [self.Error.class_required,
-                      self.Error.too_many_classes,
-                      self.Error.no_class_values,
-                      self.Error.only_one_class_var_value]
-            for cond, error in zip(conds, errors):
+        self.Error.train_data_error.clear()
+
+        if data is not None:
+            data_errors = [
+                ("Train dataset is empty.", len(data) == 0),
+                (
+                    "Train data input requires a target variable.",
+                    not data.domain.class_vars
+                ),
+                ("Too many target variables.", len(data.domain.class_vars) > 1),
+                ("Target variable has no values.", np.isnan(data.Y).all()),
+                (
+                    "Target variable has only one value.",
+                    data.domain.has_discrete_class and len(unique(data.Y)) < 2
+                ),
+                ("Data has no features to learn from.", data.X.shape[1] == 0),
+            ]
+
+            for error_msg, cond in data_errors:
                 if cond:
-                    error()
+                    self.Error.train_data_error(error_msg)
                     data = None
                     break
 

{"Orange/widgets/evaluate/owtestandscore.py": "# pylint doesn't understand the Settings magic\n# pylint: disable=invalid-sequence-index\n# pylint: disable=too-many-lines,too-many-instance-attributes\nimport abc\nimport enum\nimport logging\nimport traceback\nimport copy\nfrom functools import partial, reduce\n\nfrom concurrent.futures import Future\nfrom collections import OrderedDict, namedtuple\nfrom itertools import count\nfrom typing import Any, Optional, List, Dict, Callable\n\nimport numpy as np\nimport baycomp\n\nfrom AnyQt import QtGui\nfrom AnyQt.QtCore import Qt, QSize, QThread\nfrom AnyQt.QtCore import pyqtSlot as Slot\nfrom AnyQt.QtGui import QStandardItem, QDoubleValidator\nfrom AnyQt.QtWidgets import QHeaderView, QTableWidget, QLabel, QApplication\n\nfrom Orange.base import Learner\nimport Orange.classification\nfrom Orange.data import Table, DiscreteVariable\nfrom Orange.data.table import DomainTransformationError\nfrom Orange.data.filter import HasClass\nfrom Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT\nimport Orange.evaluation\nfrom Orange.evaluation import Results\nfrom Orange.preprocess.preprocess import Preprocess\nimport Orange.regression\nfrom Orange.widgets import gui, settings, widget\nfrom Orange.widgets.evaluate.utils import \\\n    usable_scorers, ScoreTable, learner_name, scorer_caller\nfrom Orange.widgets.utils.itemmodels import DomainModel\nfrom Orange.widgets.utils.widgetpreview import WidgetPreview\nfrom Orange.widgets.utils.concurrent import ThreadExecutor, TaskState\nfrom Orange.widgets.utils.state_summary import (format_multiple_summaries,\n                                                format_summary_details)\nfrom Orange.widgets.widget import OWWidget, Msg, Input, Output\n\nlog = logging.getLogger(__name__)\n\nInputLearner = namedtuple(\n    \"InputLearner\",\n    [\"learner\",  # :: Orange.base.Learner\n     \"results\",  # :: Option[Try[Orange.evaluation.Results]]\n     \"stats\"]    # :: Option[Sequence[Try[float]]]\n)\n\n\nclass Try(abc.ABC):\n    \"\"\"Try to walk in a Turing tar pit.\"\"\"\n\n    class Success:\n        \"\"\"Data type for instance constructed on success\"\"\"\n        __slots__ = (\"__value\",)\n#         __bool__ = lambda self: True\n        success = property(lambda self: True)\n        value = property(lambda self: self.__value)\n\n        def __init__(self, value):\n            self.__value = value\n\n        def __getnewargs__(self):\n            return (self.value,)\n\n        def __repr__(self):\n            return \"{}({!r})\".format(self.__class__.__qualname__,\n                                     self.value)\n\n        def map(self, fn):\n            return Try(lambda: fn(self.value))\n\n    class Fail:\n        \"\"\"Data type for instance constructed on fail\"\"\"\n        __slots__ = (\"__exception\", )\n#         __bool__ = lambda self: False\n        success = property(lambda self: False)\n        exception = property(lambda self: self.__exception)\n\n        def __init__(self, exception):\n            self.__exception = exception\n\n        def __getnewargs__(self):\n            return (self.exception, )\n\n        def __repr__(self):\n            return \"{}({!r})\".format(self.__class__.__qualname__,\n                                     self.exception)\n\n        def map(self, _fn):\n            return self\n\n    def __new__(cls, f, *args, **kwargs):\n        try:\n            rval = f(*args, **kwargs)\n        except BaseException as ex:\n            # IMPORANT: Will capture traceback in ex.__traceback__\n            return Try.Fail(ex)\n        else:\n            return Try.Success(rval)\n\nTry.register(Try.Success)\nTry.register(Try.Fail)\n\n\ndef raise_(exc):\n    raise exc\n\nTry.register = lambda cls: raise_(TypeError())\n\n\nclass State(enum.Enum):\n    \"\"\"\n    OWTestAndScore's runtime state.\n    \"\"\"\n    #: No or insufficient input (i.e. no data or no learners)\n    Waiting = \"Waiting\"\n    #: Executing/running the evaluations\n    Running = \"Running\"\n    #: Finished running evaluations (success or error)\n    Done = \"Done\"\n    #: Evaluation cancelled\n    Cancelled = \"Cancelled\"\n\n\nclass OWTestAndScore(OWWidget):\n    name = \"Test and Score\"\n    description = \"Cross-validation accuracy estimation.\"\n    icon = \"icons/TestLearners1.svg\"\n    priority = 100\n    keywords = ['Cross Validation', 'CV']\n    replaces = [\"Orange.widgets.evaluate.owtestlearners.OWTestLearners\"]\n\n    class Inputs:\n        train_data = Input(\"Data\", Table, default=True)\n        test_data = Input(\"Test Data\", Table)\n        learner = Input(\"Learner\", Learner, multiple=True)\n        preprocessor = Input(\"Preprocessor\", Preprocess)\n\n    class Outputs:\n        predictions = Output(\"Predictions\", Table)\n        evaluations_results = Output(\"Evaluation Results\", Results)\n\n    settings_version = 3\n    UserAdviceMessages = [\n        widget.Message(\n            \"Click on the table header to select shown columns\",\n            \"click_header\")]\n\n    settingsHandler = settings.PerfectDomainContextHandler()\n    score_table = settings.SettingProvider(ScoreTable)\n\n    #: Resampling/testing types\n    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \\\n        = 0, 1, 2, 3, 4, 5\n    #: Numbers of folds\n    NFolds = [2, 3, 5, 10, 20]\n    #: Number of repetitions\n    NRepeats = [2, 3, 5, 10, 20, 50, 100]\n    #: Sample sizes\n    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]\n\n    #: Selected resampling type\n    resampling = settings.Setting(0)\n    #: Number of folds for K-fold cross validation\n    n_folds = settings.Setting(3)\n    #: Stratified sampling for K-fold\n    cv_stratified = settings.Setting(True)\n    #: Number of repeats for ShuffleSplit sampling\n    n_repeats = settings.Setting(3)\n    #: ShuffleSplit sample size\n    sample_size = settings.Setting(9)\n    #: Stratified sampling for Random Sampling\n    shuffle_stratified = settings.Setting(True)\n    # CV where nr. of feature values determines nr. of folds\n    fold_feature = settings.ContextSetting(None)\n    fold_feature_selected = settings.ContextSetting(False)\n\n    use_rope = settings.Setting(False)\n    rope = settings.Setting(0.1)\n    comparison_criterion = settings.Setting(0, schema_only=True)\n\n    TARGET_AVERAGE = \"(Average over classes)\"\n    class_selection = settings.ContextSetting(TARGET_AVERAGE)\n\n    class Error(OWWidget.Error):\n        train_data_empty = Msg(\"Train dataset is empty.\")\n        test_data_empty = Msg(\"Test dataset is empty.\")\n        class_required = Msg(\"Train data input requires a target variable.\")\n        too_many_classes = Msg(\"Too many target variables.\")\n        class_required_test = Msg(\"Test data input requires a target variable.\")\n        too_many_folds = Msg(\"Number of folds exceeds the data size\")\n        class_inconsistent = Msg(\"Test and train datasets \"\n                                 \"have different target variables.\")\n        memory_error = Msg(\"Not enough memory.\")\n        no_class_values = Msg(\"Target variable has no values.\")\n        only_one_class_var_value = Msg(\"Target variable has only one value.\")\n        test_data_incompatible = Msg(\n            \"Test data may be incompatible with train data.\")\n\n    class Warning(OWWidget.Warning):\n        missing_data = \\\n            Msg(\"Instances with unknown target values were removed from{}data.\")\n        test_data_missing = Msg(\"Missing separate test data input.\")\n        scores_not_computed = Msg(\"Some scores could not be computed.\")\n        test_data_unused = Msg(\"Test data is present but unused. \"\n                               \"Select 'Test on test data' to use it.\")\n\n    class Information(OWWidget.Information):\n        data_sampled = Msg(\"Train data has been sampled\")\n        test_data_sampled = Msg(\"Test data has been sampled\")\n        test_data_transformed = Msg(\n            \"Test data has been transformed to match the train data.\")\n\n    def __init__(self):\n        super().__init__()\n\n        self.data = None\n        self.test_data = None\n        self.preprocessor = None\n        self.train_data_missing_vals = False\n        self.test_data_missing_vals = False\n        self.scorers = []\n        self.__pending_comparison_criterion = self.comparison_criterion\n\n        #: An Ordered dictionary with current inputs and their testing results.\n        self.learners = OrderedDict()  # type: Dict[Any, Input]\n\n        self.__state = State.Waiting\n        # Do we need to [re]test any learners, set by _invalidate and\n        # cleared by __update\n        self.__needupdate = False\n        self.__task = None  # type: Optional[TaskState]\n        self.__executor = ThreadExecutor()\n\n        self.info.set_input_summary(self.info.NoInput)\n        self.info.set_output_summary(self.info.NoOutput)\n\n        sbox = gui.vBox(self.controlArea, \"Sampling\")\n        rbox = gui.radioButtons(\n            sbox, self, \"resampling\", callback=self._param_changed)\n\n        gui.appendRadioButton(rbox, \"Cross validation\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_folds\", label=\"Number of folds: \",\n            items=[str(x) for x in self.NFolds],\n            orientation=Qt.Horizontal, callback=self.kfold_changed)\n        gui.checkBox(\n            ibox, self, \"cv_stratified\", \"Stratified\",\n            callback=self.kfold_changed)\n        gui.appendRadioButton(rbox, \"Cross validation by feature\")\n        ibox = gui.indentedBox(rbox)\n        self.feature_model = DomainModel(\n            order=DomainModel.METAS, valid_types=DiscreteVariable)\n        self.features_combo = gui.comboBox(\n            ibox, self, \"fold_feature\", model=self.feature_model,\n            orientation=Qt.Horizontal, searchable=True,\n            callback=self.fold_feature_changed)\n\n        gui.appendRadioButton(rbox, \"Random sampling\")\n        ibox = gui.indentedBox(rbox)\n        gui.comboBox(\n            ibox, self, \"n_repeats\", label=\"Repeat train/test: \",\n            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,\n            callback=self.shuffle_split_changed\n        )\n        gui.comboBox(\n            ibox, self, \"sample_size\", label=\"Training set size: \",\n            items=[\"{} %\".format(x) for x in self.SampleSizes],\n            orientation=Qt.Horizontal, callback=self.shuffle_split_changed\n        )\n        gui.checkBox(\n            ibox, self, \"shuffle_stratified\", \"Stratified\",\n            callback=self.shuffle_split_changed)\n\n        gui.appendRadioButton(rbox, \"Leave one out\")\n\n        gui.appendRadioButton(rbox, \"Test on train data\")\n        gui.appendRadioButton(rbox, \"Test on test data\")\n\n        self.cbox = gui.vBox(self.controlArea, \"Target Class\")\n        self.class_selection_combo = gui.comboBox(\n            self.cbox, self, \"class_selection\", items=[],\n            sendSelectedValue=True, contentsLength=8, searchable=True,\n            callback=self._on_target_class_changed\n        )\n\n        self.modcompbox = box = gui.vBox(self.controlArea, \"Model Comparison\")\n        gui.comboBox(\n            box, self, \"comparison_criterion\",\n            callback=self.update_comparison_table)\n\n        hbox = gui.hBox(box)\n        gui.checkBox(hbox, self, \"use_rope\",\n                     \"Negligible difference: \",\n                     callback=self._on_use_rope_changed)\n        gui.lineEdit(hbox, self, \"rope\", validator=QDoubleValidator(),\n                     controlWidth=70, callback=self.update_comparison_table,\n                     alignment=Qt.AlignRight)\n        self.controls.rope.setEnabled(self.use_rope)\n\n        gui.rubber(self.controlArea)\n        self.score_table = ScoreTable(self)\n        self.score_table.shownScoresChanged.connect(self.update_stats_model)\n        view = self.score_table.view\n        view.setSizeAdjustPolicy(view.AdjustToContents)\n\n        box = gui.vBox(self.mainArea, \"Evaluation Results\")\n        box.layout().addWidget(self.score_table.view)\n\n        self.compbox = box = gui.vBox(self.mainArea, box=\"Model comparison\")\n        table = self.comparison_table = QTableWidget(\n            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,\n            selectionMode=QTableWidget.NoSelection)\n        table.setSizeAdjustPolicy(table.AdjustToContents)\n        header = table.verticalHeader()\n        header.setSectionResizeMode(QHeaderView.Fixed)\n        header.setSectionsClickable(False)\n\n        header = table.horizontalHeader()\n        header.setTextElideMode(Qt.ElideRight)\n        header.setDefaultAlignment(Qt.AlignCenter)\n        header.setSectionsClickable(False)\n        header.setStretchLastSection(False)\n        header.setSectionResizeMode(QHeaderView.ResizeToContents)\n        avg_width = self.fontMetrics().averageCharWidth()\n        header.setMinimumSectionSize(8 * avg_width)\n        header.setMaximumSectionSize(15 * avg_width)\n        header.setDefaultSectionSize(15 * avg_width)\n        box.layout().addWidget(table)\n        box.layout().addWidget(QLabel(\n            \"<small>Table shows probabilities that the score for the model in \"\n            \"the row is higher than that of the model in the column. \"\n            \"Small numbers show the probability that the difference is \"\n            \"negligible.</small>\", wordWrap=True))\n\n    @staticmethod\n    def sizeHint():\n        return QSize(780, 1)\n\n    def _update_controls(self):\n        self.fold_feature = None\n        self.feature_model.set_domain(None)\n        if self.data:\n            self.feature_model.set_domain(self.data.domain)\n            if self.fold_feature is None and self.feature_model:\n                self.fold_feature = self.feature_model[0]\n        enabled = bool(self.feature_model)\n        self.controls.resampling.buttons[\n            OWTestAndScore.FeatureFold].setEnabled(enabled)\n        self.features_combo.setEnabled(enabled)\n        if self.resampling == OWTestAndScore.FeatureFold and not enabled:\n            self.resampling = OWTestAndScore.KFold\n\n    @Inputs.learner\n    def set_learner(self, learner, key):\n        \"\"\"\n        Set the input `learner` for `key`.\n\n        Parameters\n        ----------\n        learner : Optional[Orange.base.Learner]\n        key : Any\n        \"\"\"\n        if key in self.learners and learner is None:\n            # Removed\n            self._invalidate([key])\n            del self.learners[key]\n        elif learner is not None:\n            self.learners[key] = InputLearner(learner, None, None)\n            self._invalidate([key])\n\n    @Inputs.train_data\n    def set_train_data(self, data):\n        \"\"\"\n        Set the input training dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.cancel()\n        self.Information.data_sampled.clear()\n        self.Error.train_data_empty.clear()\n        self.Error.class_required.clear()\n        self.Error.too_many_classes.clear()\n        self.Error.no_class_values.clear()\n        self.Error.only_one_class_var_value.clear()\n        if data is not None and not data:\n            self.Error.train_data_empty()\n            data = None\n        if data:\n            conds = [not data.domain.class_vars,\n                     len(data.domain.class_vars) > 1,\n                     np.isnan(data.Y).all(),\n                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]\n            errors = [self.Error.class_required,\n                      self.Error.too_many_classes,\n                      self.Error.no_class_values,\n                      self.Error.only_one_class_var_value]\n            for cond, error in zip(conds, errors):\n                if cond:\n                    error()\n                    data = None\n                    break\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.train_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.data = data\n        self.closeContext()\n        self._update_scorers()\n        self._update_controls()\n        if data is not None:\n            self._update_class_selection()\n            self.openContext(data.domain)\n            if self.fold_feature_selected and bool(self.feature_model):\n                self.resampling = OWTestAndScore.FeatureFold\n        self._invalidate()\n\n    @Inputs.test_data\n    def set_test_data(self, data):\n        # type: (Orange.data.Table) -> None\n        \"\"\"\n        Set the input separate testing dataset.\n\n        Parameters\n        ----------\n        data : Optional[Orange.data.Table]\n        \"\"\"\n        self.Information.test_data_sampled.clear()\n        self.Error.test_data_empty.clear()\n        if data is not None and not data:\n            self.Error.test_data_empty()\n            data = None\n        if data and not data.domain.class_var:\n            self.Error.class_required_test()\n            data = None\n        else:\n            self.Error.class_required_test.clear()\n\n        if isinstance(data, SqlTable):\n            if data.approx_len() < AUTO_DL_LIMIT:\n                data = Table(data)\n            else:\n                self.Information.test_data_sampled()\n                data_sample = data.sample_time(1, no_cache=True)\n                data_sample.download_data(AUTO_DL_LIMIT, partial=True)\n                data = Table(data_sample)\n\n        self.test_data_missing_vals = \\\n            data is not None and np.isnan(data.Y).any()\n        if self.train_data_missing_vals or self.test_data_missing_vals:\n            self.Warning.missing_data(self._which_missing_data())\n            if data:\n                data = HasClass()(data)\n        else:\n            self.Warning.missing_data.clear()\n\n        self.test_data = data\n        if self.resampling == OWTestAndScore.TestOnTest:\n            self._invalidate()\n\n    def _which_missing_data(self):\n        return {(True, True): \" \",  # both, don't specify\n                (True, False): \" train \",\n                (False, True): \" test \"}[(self.train_data_missing_vals,\n                                          self.test_data_missing_vals)]\n\n    # List of scorers shouldn't be retrieved globally, when the module is\n    # loading since add-ons could have registered additional scorers.\n    # It could have been cached but\n    # - we don't gain much with it\n    # - it complicates the unit tests\n    def _update_scorers(self):\n        if self.data and self.data.domain.class_var:\n            new_scorers = usable_scorers(self.data.domain.class_var)\n        else:\n            new_scorers = []\n        # Don't unnecessarily reset the combo because this would always reset\n        # comparison_criterion; we also set it explicitly, though, for clarity\n        if new_scorers != self.scorers:\n            self.scorers = new_scorers\n            combo = self.controls.comparison_criterion\n            combo.clear()\n            combo.addItems([scorer.long_name or scorer.name\n                            for scorer in self.scorers])\n            self.comparison_criterion = 0\n        if self.__pending_comparison_criterion is not None:\n            # Check for the unlikely case that some scorers have been removed\n            # from modules\n            if self.__pending_comparison_criterion < len(self.scorers):\n                self.comparison_criterion = self.__pending_comparison_criterion\n            self.__pending_comparison_criterion = None\n        self._update_compbox_title()\n\n    def _update_compbox_title(self):\n        criterion = self.comparison_criterion\n        if criterion < len(self.scorers):\n            scorer = self.scorers[criterion]()\n            self.compbox.setTitle(f\"Model Comparison by {scorer.name}\")\n        else:\n            self.compbox.setTitle(f\"Model Comparison\")\n\n    @Inputs.preprocessor\n    def set_preprocessor(self, preproc):\n        \"\"\"\n        Set the input preprocessor to apply on the training data.\n        \"\"\"\n        self.preprocessor = preproc\n        self._invalidate()\n\n    def handleNewSignals(self):\n        \"\"\"Reimplemented from OWWidget.handleNewSignals.\"\"\"\n        self._update_class_selection()\n        self.score_table.update_header(self.scorers)\n        self._update_view_enabled()\n        self.update_stats_model()\n        self.set_input_summary()\n        if self.__needupdate:\n            self.__update()\n\n    def set_input_summary(self):\n        summary, details, kwargs = self.info.NoInput, \"\", {}\n        if self.data and self.test_data:\n            summary = f\"{self.info.format_number(len(self.data))},\" \\\n                      f\" {self.info.format_number(len(self.test_data))}\"\n            details = format_multiple_summaries([\n                (\"Data\", self.data), (\"Test data\", self.test_data)\n            ])\n            kwargs = {\"format\": Qt.RichText}\n        elif self.data and not self.test_data:\n            summary, details = len(self.data), format_summary_details(self.data)\n        elif self.test_data and not self.data:\n            summary = len(self.test_data)\n            details = format_summary_details(self.test_data)\n        self.info.set_input_summary(summary, details, **kwargs)\n\n    def kfold_changed(self):\n        self.resampling = OWTestAndScore.KFold\n        self._param_changed()\n\n    def fold_feature_changed(self):\n        self.resampling = OWTestAndScore.FeatureFold\n        self._param_changed()\n\n    def shuffle_split_changed(self):\n        self.resampling = OWTestAndScore.ShuffleSplit\n        self._param_changed()\n\n    def _param_changed(self):\n        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)\n        self._update_view_enabled()\n        self._invalidate()\n        self.__update()\n\n    def _update_view_enabled(self):\n        self.comparison_table.setEnabled(\n            self.resampling == OWTestAndScore.KFold\n            and len(self.learners) > 1\n            and self.data is not None)\n        self.score_table.view.setEnabled(\n            self.data is not None)\n\n    def update_stats_model(self):\n        # Update the results_model with up to date scores.\n        # Note: The target class specific scores (if requested) are\n        # computed as needed in this method.\n        model = self.score_table.model\n        # clear the table model, but preserving the header labels\n        for r in reversed(range(model.rowCount())):\n            model.takeRow(r)\n\n        target_index = None\n        if self.data is not None:\n            class_var = self.data.domain.class_var\n            if self.data.domain.has_discrete_class and \\\n                            self.class_selection != self.TARGET_AVERAGE:\n                target_index = class_var.values.index(self.class_selection)\n        else:\n            class_var = None\n\n        errors = []\n        has_missing_scores = False\n\n        names = []\n        for key, slot in self.learners.items():\n            name = learner_name(slot.learner)\n            names.append(name)\n            head = QStandardItem(name)\n            head.setData(key, Qt.UserRole)\n            results = slot.results\n            if results is not None and results.success:\n                train = QStandardItem(\"{:.3f}\".format(results.value.train_time))\n                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                train.setData(key, Qt.UserRole)\n                test = QStandardItem(\"{:.3f}\".format(results.value.test_time))\n                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                test.setData(key, Qt.UserRole)\n                row = [head, train, test]\n            else:\n                row = [head]\n            if isinstance(results, Try.Fail):\n                head.setToolTip(str(results.exception))\n                head.setText(\"{} (error)\".format(name))\n                head.setForeground(QtGui.QBrush(Qt.red))\n                if isinstance(results.exception, DomainTransformationError) \\\n                        and self.resampling == self.TestOnTest:\n                    self.Error.test_data_incompatible()\n                    self.Information.test_data_transformed.clear()\n                else:\n                    errors.append(\"{name} failed with error:\\n\"\n                                  \"{exc.__class__.__name__}: {exc!s}\"\n                                  .format(name=name, exc=slot.results.exception)\n                                  )\n\n            if class_var is not None and class_var.is_discrete and \\\n                    target_index is not None:\n                if slot.results is not None and slot.results.success:\n                    ovr_results = results_one_vs_rest(\n                        slot.results.value, target_index)\n\n                    # Cell variable is used immediatelly, it's not stored\n                    # pylint: disable=cell-var-from-loop\n                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))\n                             for scorer in self.scorers]\n                else:\n                    stats = None\n            else:\n                stats = slot.stats\n\n            if stats is not None:\n                for stat, scorer in zip(stats, self.scorers):\n                    item = QStandardItem()\n                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)\n                    if stat.success:\n                        item.setData(float(stat.value[0]), Qt.DisplayRole)\n                    else:\n                        item.setToolTip(str(stat.exception))\n                        if scorer.name in self.score_table.shown_scores:\n                            has_missing_scores = True\n                    row.append(item)\n\n            model.appendRow(row)\n\n        # Resort rows based on current sorting\n        header = self.score_table.view.horizontalHeader()\n        model.sort(\n            header.sortIndicatorSection(),\n            header.sortIndicatorOrder()\n        )\n        self._set_comparison_headers(names)\n\n        self.error(\"\\n\".join(errors), shown=bool(errors))\n        self.Warning.scores_not_computed(shown=has_missing_scores)\n\n    def _on_use_rope_changed(self):\n        self.controls.rope.setEnabled(self.use_rope)\n        self.update_comparison_table()\n\n    def update_comparison_table(self):\n        self.comparison_table.clearContents()\n        slots = self._successful_slots()\n        if not (slots and self.scorers):\n            return\n        names = [learner_name(slot.learner) for slot in slots]\n        self._set_comparison_headers(names)\n        if self.resampling == OWTestAndScore.KFold:\n            scores = self._scores_by_folds(slots)\n            self._fill_table(names, scores)\n\n    def _successful_slots(self):\n        model = self.score_table.model\n        proxy = self.score_table.sorted_model\n\n        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)\n                for row in range(proxy.rowCount()))\n        slots = [slot for slot in (self.learners[key] for key in keys)\n                 if slot.results is not None and slot.results.success]\n        return slots\n\n    def _set_comparison_headers(self, names):\n        table = self.comparison_table\n        try:\n            # Prevent glitching during update\n            table.setUpdatesEnabled(False)\n            header = table.horizontalHeader()\n            if len(names) > 2:\n                header.setSectionResizeMode(QHeaderView.Stretch)\n            else:\n                header.setSectionResizeMode(QHeaderView.Fixed)\n            table.setRowCount(len(names))\n            table.setColumnCount(len(names))\n            table.setVerticalHeaderLabels(names)\n            table.setHorizontalHeaderLabels(names)\n        finally:\n            table.setUpdatesEnabled(True)\n\n    def _scores_by_folds(self, slots):\n        scorer = self.scorers[self.comparison_criterion]()\n        self._update_compbox_title()\n        if scorer.is_binary:\n            if self.class_selection != self.TARGET_AVERAGE:\n                class_var = self.data.domain.class_var\n                target_index = class_var.values.index(self.class_selection)\n                kw = dict(target=target_index)\n            else:\n                kw = dict(average='weighted')\n        else:\n            kw = {}\n\n        def call_scorer(results):\n            def thunked():\n                return scorer.scores_by_folds(results.value, **kw).flatten()\n\n            return thunked\n\n        scores = [Try(call_scorer(slot.results)) for slot in slots]\n        scores = [score.value if score.success else None for score in scores]\n        # `None in scores doesn't work -- these are np.arrays)\n        if any(score is None for score in scores):\n            self.Warning.scores_not_computed()\n        return scores\n\n    def _fill_table(self, names, scores):\n        table = self.comparison_table\n        for row, row_name, row_scores in zip(count(), names, scores):\n            for col, col_name, col_scores in zip(range(row), names, scores):\n                if row_scores is None or col_scores is None:\n                    continue\n                if self.use_rope and self.rope:\n                    p0, rope, p1 = baycomp.two_on_single(\n                        row_scores, col_scores, self.rope)\n                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\\n\"\n                                   f\"p({row_name} = {col_name}) = {rope:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}<br/><small>{rope:.3f}</small>\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\\n\"\n                                   f\"p({col_name} = {row_name}) = {rope:.3f}\")\n                else:\n                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)\n                    if np.isnan(p0) or np.isnan(p1):\n                        self._set_cells_na(table, row, col)\n                        continue\n                    self._set_cell(table, row, col,\n                                   f\"{p0:.3f}\",\n                                   f\"p({row_name} > {col_name}) = {p0:.3f}\")\n                    self._set_cell(table, col, row,\n                                   f\"{p1:.3f}\",\n                                   f\"p({col_name} > {row_name}) = {p1:.3f}\")\n\n    @classmethod\n    def _set_cells_na(cls, table, row, col):\n        cls._set_cell(table, row, col, \"NA\", \"comparison cannot be computed\")\n        cls._set_cell(table, col, row, \"NA\", \"comparison cannot be computed\")\n\n    @staticmethod\n    def _set_cell(table, row, col, label, tooltip):\n        item = QLabel(label)\n        item.setToolTip(tooltip)\n        item.setAlignment(Qt.AlignCenter)\n        table.setCellWidget(row, col, item)\n\n    def _update_class_selection(self):\n        self.class_selection_combo.setCurrentIndex(-1)\n        self.class_selection_combo.clear()\n        if not self.data:\n            return\n\n        if self.data.domain.has_discrete_class:\n            self.cbox.setVisible(True)\n            class_var = self.data.domain.class_var\n            items = (self.TARGET_AVERAGE, ) + class_var.values\n            self.class_selection_combo.addItems(items)\n\n            class_index = 0\n            if self.class_selection in class_var.values:\n                class_index = class_var.values.index(self.class_selection) + 1\n\n            self.class_selection_combo.setCurrentIndex(class_index)\n            self.class_selection = items[class_index]\n        else:\n            self.cbox.setVisible(False)\n\n    def _on_target_class_changed(self):\n        self.update_stats_model()\n        self.update_comparison_table()\n\n    def _invalidate(self, which=None):\n        self.cancel()\n        self.fold_feature_selected = \\\n            self.resampling == OWTestAndScore.FeatureFold\n        # Invalidate learner results for `which` input keys\n        # (if None then all learner results are invalidated)\n        if which is None:\n            which = self.learners.keys()\n\n        model = self.score_table.model\n        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)\n                         for row in range(model.rowCount())]\n\n        for key in which:\n            self.learners[key] = \\\n                self.learners[key]._replace(results=None, stats=None)\n\n            if key in statmodelkeys:\n                row = statmodelkeys.index(key)\n                for c in range(1, model.columnCount()):\n                    item = model.item(row, c)\n                    if item is not None:\n                        item.setData(None, Qt.DisplayRole)\n                        item.setData(None, Qt.ToolTipRole)\n\n        self.comparison_table.clearContents()\n\n        self.__needupdate = True\n\n    def commit(self):\n        \"\"\"\n        Commit the results to output.\n        \"\"\"\n        self.Error.memory_error.clear()\n        valid = [slot for slot in self.learners.values()\n                 if slot.results is not None and slot.results.success]\n        combined = None\n        predictions = None\n        if valid:\n            # Evaluation results\n            combined = results_merge([slot.results.value for slot in valid])\n            combined.learner_names = [learner_name(slot.learner)\n                                      for slot in valid]\n\n            # Predictions & Probabilities\n            try:\n                predictions = combined.get_augmented_data(combined.learner_names)\n            except MemoryError:\n                self.Error.memory_error()\n\n        summary = len(predictions) if predictions else self.info.NoOutput\n        details = format_summary_details(predictions) if predictions else \"\"\n        self.info.set_output_summary(summary, details)\n\n        self.Outputs.evaluations_results.send(combined)\n        self.Outputs.predictions.send(predictions)\n\n    def send_report(self):\n        \"\"\"Report on the testing schema and results\"\"\"\n        if not self.data or not self.learners:\n            return\n        if self.resampling == self.KFold:\n            stratified = 'Stratified ' if self.cv_stratified else ''\n            items = [(\"Sampling type\", \"{}{}-fold Cross validation\".\n                      format(stratified, self.NFolds[self.n_folds]))]\n        elif self.resampling == self.LeaveOneOut:\n            items = [(\"Sampling type\", \"Leave one out\")]\n        elif self.resampling == self.ShuffleSplit:\n            stratified = 'Stratified ' if self.shuffle_stratified else ''\n            items = [(\"Sampling type\",\n                      \"{}Shuffle split, {} random samples with {}% data \"\n                      .format(stratified, self.NRepeats[self.n_repeats],\n                              self.SampleSizes[self.sample_size]))]\n        elif self.resampling == self.TestOnTrain:\n            items = [(\"Sampling type\", \"No sampling, test on training data\")]\n        elif self.resampling == self.TestOnTest:\n            items = [(\"Sampling type\", \"No sampling, test on testing data\")]\n        else:\n            items = []\n        if self.data.domain.has_discrete_class:\n            items += [(\"Target class\", self.class_selection.strip(\"()\"))]\n        if items:\n            self.report_items(\"Settings\", items)\n        self.report_table(\"Scores\", self.score_table.view)\n\n    @classmethod\n    def migrate_settings(cls, settings_, version):\n        if version < 2:\n            if settings_[\"resampling\"] > 0:\n                settings_[\"resampling\"] += 1\n        if version < 3:\n            # Older version used an incompatible context handler\n            settings_[\"context_settings\"] = [\n                c for c in settings_.get(\"context_settings\", ())\n                if not hasattr(c, 'classes')]\n\n    @Slot(float)\n    def setProgressValue(self, value):\n        self.progressBarSet(value)\n\n    def __update(self):\n        self.__needupdate = False\n\n        assert self.__task is None or self.__state == State.Running\n        if self.__state == State.Running:\n            self.cancel()\n\n        self.Warning.test_data_unused.clear()\n        self.Error.test_data_incompatible.clear()\n        self.Warning.test_data_missing.clear()\n        self.Information.test_data_transformed(\n            shown=self.resampling == self.TestOnTest\n            and self.data is not None\n            and self.test_data is not None\n            and self.data.domain.attributes != self.test_data.domain.attributes)\n        self.warning()\n        self.Error.class_inconsistent.clear()\n        self.Error.too_many_folds.clear()\n        self.error()\n\n        # check preconditions and return early\n        if self.data is None:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if not self.learners:\n            self.__state = State.Waiting\n            self.commit()\n            return\n        if self.resampling == OWTestAndScore.KFold and \\\n                len(self.data) < self.NFolds[self.n_folds]:\n            self.Error.too_many_folds()\n            self.__state = State.Waiting\n            self.commit()\n            return\n\n        elif self.resampling == OWTestAndScore.TestOnTest:\n            if self.test_data is None:\n                if not self.Error.test_data_empty.is_shown():\n                    self.Warning.test_data_missing()\n                self.__state = State.Waiting\n                self.commit()\n                return\n            elif self.test_data.domain.class_var != self.data.domain.class_var:\n                self.Error.class_inconsistent()\n                self.__state = State.Waiting\n                self.commit()\n                return\n\n        elif self.test_data is not None:\n            self.Warning.test_data_unused()\n\n        rstate = 42\n        # items in need of an update\n        items = [(key, slot) for key, slot in self.learners.items()\n                 if slot.results is None]\n        learners = [slot.learner for _, slot in items]\n\n        # deepcopy all learners as they are not thread safe (by virtue of\n        # the base API). These will be the effective learner objects tested\n        # but will be replaced with the originals on return (see restore\n        # learners bellow)\n        learners_c = [copy.deepcopy(learner) for learner in learners]\n\n        if self.resampling == OWTestAndScore.TestOnTest:\n            test_f = partial(\n                Orange.evaluation.TestOnTestData(\n                    store_data=True, store_models=True),\n                self.data, self.test_data, learners_c, self.preprocessor\n            )\n        else:\n            if self.resampling == OWTestAndScore.KFold:\n                sampler = Orange.evaluation.CrossValidation(\n                    k=self.NFolds[self.n_folds],\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.FeatureFold:\n                sampler = Orange.evaluation.CrossValidationFeature(\n                    feature=self.fold_feature)\n            elif self.resampling == OWTestAndScore.LeaveOneOut:\n                sampler = Orange.evaluation.LeaveOneOut()\n            elif self.resampling == OWTestAndScore.ShuffleSplit:\n                sampler = Orange.evaluation.ShuffleSplit(\n                    n_resamples=self.NRepeats[self.n_repeats],\n                    train_size=self.SampleSizes[self.sample_size] / 100,\n                    test_size=None,\n                    stratified=self.shuffle_stratified,\n                    random_state=rstate)\n            elif self.resampling == OWTestAndScore.TestOnTrain:\n                sampler = Orange.evaluation.TestOnTrainingData(\n                    store_models=True)\n            else:\n                assert False, \"self.resampling %s\" % self.resampling\n\n            sampler.store_data = True\n            test_f = partial(\n                sampler, self.data, learners_c, self.preprocessor)\n\n        def replace_learners(evalfunc, *args, **kwargs):\n            res = evalfunc(*args, **kwargs)\n            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))\n            res.learners[:] = learners\n            return res\n\n        test_f = partial(replace_learners, test_f)\n\n        self.__submit(test_f)\n\n    def __submit(self, testfunc):\n        # type: (Callable[[Callable[[float], None]], Results]) -> None\n        \"\"\"\n        Submit a testing function for evaluation\n\n        MUST not be called if an evaluation is already pending/running.\n        Cancel the existing task first.\n\n        Parameters\n        ----------\n        testfunc : Callable[[Callable[float]], Results])\n            Must be a callable taking a single `callback` argument and\n            returning a Results instance\n        \"\"\"\n        assert self.__state != State.Running\n        # Setup the task\n        task = TaskState()\n\n        def progress_callback(finished):\n            if task.is_interruption_requested():\n                raise UserInterrupt()\n            task.set_progress_value(100 * finished)\n\n        testfunc = partial(testfunc, callback=progress_callback)\n        task.start(self.__executor, testfunc)\n\n        task.progress_changed.connect(self.setProgressValue)\n        task.watcher.finished.connect(self.__task_complete)\n\n        self.Outputs.evaluations_results.invalidate()\n        self.Outputs.predictions.invalidate()\n        self.progressBarInit()\n        self.setStatusMessage(\"Running\")\n\n        self.__state = State.Running\n        self.__task = task\n\n    @Slot(object)\n    def __task_complete(self, f: 'Future[Results]'):\n        # handle a completed task\n        assert self.thread() is QThread.currentThread()\n        assert self.__task is not None and self.__task.future is f\n        self.progressBarFinished()\n        self.setStatusMessage(\"\")\n        assert f.done()\n        self.__task = None\n        self.__state = State.Done\n        try:\n            results = f.result()    # type: Results\n            learners = results.learners  # type: List[Learner]\n        except Exception as er:  # pylint: disable=broad-except\n            log.exception(\"testing error (in __task_complete):\",\n                          exc_info=True)\n            self.error(\"\\n\".join(traceback.format_exception_only(type(er), er)))\n            return\n\n        learner_key = {slot.learner: key for key, slot in\n                       self.learners.items()}\n        assert all(learner in learner_key for learner in learners)\n\n        # Update the results for individual learners\n        class_var = results.domain.class_var\n        for learner, result in zip(learners, results.split_by_model()):\n            stats = None\n            if class_var.is_primitive():\n                ex = result.failed[0]\n                if ex:\n                    stats = [Try.Fail(ex)] * len(self.scorers)\n                    result = Try.Fail(ex)\n                else:\n                    stats = [Try(scorer_caller(scorer, result))\n                             for scorer in self.scorers]\n                    result = Try.Success(result)\n            key = learner_key.get(learner)\n            self.learners[key] = \\\n                self.learners[key]._replace(results=result, stats=stats)\n\n        self.score_table.update_header(self.scorers)\n        self.update_stats_model()\n        self.update_comparison_table()\n\n        self.commit()\n\n    def cancel(self):\n        \"\"\"\n        Cancel the current/pending evaluation (if any).\n        \"\"\"\n        if self.__task is not None:\n            assert self.__state == State.Running\n            self.__state = State.Cancelled\n            task, self.__task = self.__task, None\n            task.cancel()\n            task.progress_changed.disconnect(self.setProgressValue)\n            task.watcher.finished.disconnect(self.__task_complete)\n            add_task_to_dispose_queue(task)\n\n            self.progressBarFinished()\n            self.setStatusMessage(\"\")\n\n    def onDeleteWidget(self):\n        self.cancel()\n        self.__executor.shutdown(True)\n        super().onDeleteWidget()\n\n\ndef add_task_to_dispose_queue(task: TaskState):\n    # transfer ownership of task to Qt, and delete it after completion\n    # all other signals from task should be disconnected.\n    assert task.parent() is None\n    app = QApplication.instance()\n    task.setParent(app)\n    task.watcher.finished.connect(task.deleteLater)\n\n\nclass UserInterrupt(BaseException):\n    \"\"\"\n    A BaseException subclass used for cooperative task/thread cancellation\n    \"\"\"\n\n\ndef results_add_by_model(x, y):\n    def is_empty(res):\n        return (getattr(res, \"models\", None) is None\n                and getattr(res, \"row_indices\", None) is None)\n\n    if is_empty(x):\n        return y\n    elif is_empty(y):\n        return x\n\n    assert (x.row_indices == y.row_indices).all()\n    assert (x.actual == y.actual).all()\n\n    res = Orange.evaluation.Results()\n    res.data = x.data\n    res.domain = x.domain\n    res.row_indices = x.row_indices\n    res.folds = x.folds\n    res.actual = x.actual\n    res.predicted = np.vstack((x.predicted, y.predicted))\n    if getattr(x, \"probabilities\", None) is not None \\\n            and getattr(y, \"probabilities\") is not None:\n        res.probabilities = np.vstack((x.probabilities, y.probabilities))\n\n    if x.models is not None:\n        res.models = np.hstack((x.models, y.models))\n    return res\n\n\ndef results_merge(results):\n    return reduce(results_add_by_model, results, Orange.evaluation.Results())\n\n\ndef results_one_vs_rest(results, pos_index):\n    from Orange.preprocess.transformation import Indicator\n    actual = results.actual == pos_index\n    predicted = results.predicted == pos_index\n    if results.probabilities is not None:\n        c = results.probabilities.shape[2]\n        assert c >= 2\n        neg_indices = [i for i in range(c) if i != pos_index]\n        pos_prob = results.probabilities[:, :, [pos_index]]\n        neg_prob = np.sum(results.probabilities[:, :, neg_indices],\n                          axis=2, keepdims=True)\n        probabilities = np.dstack((neg_prob, pos_prob))\n    else:\n        probabilities = None\n\n    res = Orange.evaluation.Results()\n    res.actual = actual\n    res.predicted = predicted\n    res.folds = results.folds\n    res.row_indices = results.row_indices\n    res.probabilities = probabilities\n\n    value = results.domain.class_var.values[pos_index]\n    class_var = Orange.data.DiscreteVariable(\n        \"I({}=={})\".format(results.domain.class_var.name, value),\n        values=(\"False\", \"True\"),\n        compute_value=Indicator(results.domain.class_var, pos_index)\n    )\n    domain = Orange.data.Domain(\n        results.domain.attributes,\n        [class_var],\n        results.domain.metas\n    )\n    res.data = None\n    res.domain = domain\n    return res\n\n\nif __name__ == \"__main__\":  # pragma: no cover\n    filename = \"iris\"\n    preview_data = Table(filename)\n    if preview_data.domain.class_var.is_discrete:\n        prev_learners = [lambda data: 1 / 0,\n                         Orange.classification.LogisticRegressionLearner(),\n                         Orange.classification.MajorityLearner(),\n                         Orange.classification.NaiveBayesLearner()]\n    else:\n        prev_learners = [lambda data: 1 / 0,\n                         Orange.regression.MeanLearner(),\n                         Orange.regression.KNNRegressionLearner(),\n                         Orange.regression.RidgeRegressionLearner()]\n\n    WidgetPreview(OWTestAndScore).run(\n        set_train_data=preview_data,\n        set_test_data=preview_data,\n        set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]\n    )\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
**Analysis**

- **Key Parts:**
  - **Commit Message:** `[FIX] Test and Score: Improve data errors.`
  - **Code Changes:** Consolidated multiple error messages into a single `train_data_error` with dynamic messages.
  - **Test Results:** Previously, tests `test_data_errors` and `test_one_class_value` failed with `AttributeError` because `train_data_error` did not exist.
  
- **Main Problem:** The `Error` class in `OWTestAndScore` lacks the `train_data_error` attribute, causing AttributeErrors when the widget tries to handle training data errors.

- **Expected Behavior:** The widget should handle and display training data errors using the `train_data_error` attribute without raising errors.

- **Actual Behavior:** Attempting to access `train_data_error` results in an `AttributeError`, preventing proper error handling.

**GitHub Issue**

[ISSUE]

**Title:** AttributeError: 'Error' object has no attribute 'train_data_error' when handling training data errors

**Description**

When the `OWTestAndScore` widget processes training data, it attempts to access the `train_data_error` attribute to display relevant error messages. However, this attribute does not exist in the `Error` class, leading to an AttributeError.

**Example Code**

```python
def test_handle_empty_train_data():
    data = None
    widget.Error.train_data_error.clear()
    if data is not None and not data:
        widget.Error.train_data_error("Train dataset is empty.")
```

**Expected Behavior**

The widget should successfully handle training data errors by accessing the `train_data_error` attribute and displaying the appropriate error message without raising any exceptions.

**Actual Behavior**

Accessing `train_data_error` results in the following error:

```
AttributeError: 'Error' object has no attribute 'train_data_error'
```

This prevents the widget from properly notifying users about issues with the training dataset.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### Orange/widgets/evaluate/owtestandscore.py
# pylint doesn't understand the Settings magic
# pylint: disable=invalid-sequence-index
# pylint: disable=too-many-lines,too-many-instance-attributes
import abc
import enum
import logging
import traceback
import copy
from functools import partial, reduce

from concurrent.futures import Future
from collections import OrderedDict, namedtuple
from itertools import count
from typing import Any, Optional, List, Dict, Callable

import numpy as np
import baycomp

from AnyQt import QtGui
from AnyQt.QtCore import Qt, QSize, QThread
from AnyQt.QtCore import pyqtSlot as Slot
from AnyQt.QtGui import QStandardItem, QDoubleValidator
from AnyQt.QtWidgets import QHeaderView, QTableWidget, QLabel, QApplication

from Orange.base import Learner
import Orange.classification
from Orange.data import Table, DiscreteVariable
from Orange.data.table import DomainTransformationError
from Orange.data.filter import HasClass
from Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT
import Orange.evaluation
from Orange.evaluation import Results
from Orange.preprocess.preprocess import Preprocess
import Orange.regression
from Orange.widgets import gui, settings, widget
from Orange.widgets.evaluate.utils import \
    usable_scorers, ScoreTable, learner_name, scorer_caller
from Orange.widgets.utils.itemmodels import DomainModel
from Orange.widgets.utils.widgetpreview import WidgetPreview
from Orange.widgets.utils.concurrent import ThreadExecutor, TaskState
from Orange.widgets.utils.state_summary import (format_multiple_summaries,
                                                format_summary_details)
from Orange.widgets.widget import OWWidget, Msg, Input, Output

log = logging.getLogger(__name__)

InputLearner = namedtuple(
    "InputLearner",
    ["learner",  # :: Orange.base.Learner
     "results",  # :: Option[Try[Orange.evaluation.Results]]
     "stats"]    # :: Option[Sequence[Try[float]]]
)


class Try(abc.ABC):
    """Try to walk in a Turing tar pit."""

    class Success:
        """Data type for instance constructed on success"""
        __slots__ = ("__value",)
#         __bool__ = lambda self: True
        success = property(lambda self: True)
        value = property(lambda self: self.__value)

        def __init__(self, value):
            self.__value = value

        def __getnewargs__(self):
            return (self.value,)

        def __repr__(self):
            return "{}({!r})".format(self.__class__.__qualname__,
                                     self.value)

        def map(self, fn):
            return Try(lambda: fn(self.value))

    class Fail:
        """Data type for instance constructed on fail"""
        __slots__ = ("__exception", )
#         __bool__ = lambda self: False
        success = property(lambda self: False)
        exception = property(lambda self: self.__exception)

        def __init__(self, exception):
            self.__exception = exception

        def __getnewargs__(self):
            return (self.exception, )

        def __repr__(self):
            return "{}({!r})".format(self.__class__.__qualname__,
                                     self.exception)

        def map(self, _fn):
            return self

    def __new__(cls, f, *args, **kwargs):
        try:
            rval = f(*args, **kwargs)
        except BaseException as ex:
            # IMPORANT: Will capture traceback in ex.__traceback__
            return Try.Fail(ex)
        else:
            return Try.Success(rval)

Try.register(Try.Success)
Try.register(Try.Fail)


def raise_(exc):
    raise exc

Try.register = lambda cls: raise_(TypeError())


class State(enum.Enum):
    """
    OWTestAndScore's runtime state.
    """
    #: No or insufficient input (i.e. no data or no learners)
    Waiting = "Waiting"
    #: Executing/running the evaluations
    Running = "Running"
    #: Finished running evaluations (success or error)
    Done = "Done"
    #: Evaluation cancelled
    Cancelled = "Cancelled"


class OWTestAndScore(OWWidget):
    name = "Test and Score"
    description = "Cross-validation accuracy estimation."
    icon = "icons/TestLearners1.svg"
    priority = 100
    keywords = ['Cross Validation', 'CV']
    replaces = ["Orange.widgets.evaluate.owtestlearners.OWTestLearners"]

    class Inputs:
        train_data = Input("Data", Table, default=True)
        test_data = Input("Test Data", Table)
        learner = Input("Learner", Learner, multiple=True)
        preprocessor = Input("Preprocessor", Preprocess)

    class Outputs:
        predictions = Output("Predictions", Table)
        evaluations_results = Output("Evaluation Results", Results)

    settings_version = 3
    UserAdviceMessages = [
        widget.Message(
            "Click on the table header to select shown columns",
            "click_header")]

    settingsHandler = settings.PerfectDomainContextHandler()
    score_table = settings.SettingProvider(ScoreTable)

    #: Resampling/testing types
    KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \
        = 0, 1, 2, 3, 4, 5
    #: Numbers of folds
    NFolds = [2, 3, 5, 10, 20]
    #: Number of repetitions
    NRepeats = [2, 3, 5, 10, 20, 50, 100]
    #: Sample sizes
    SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95]

    #: Selected resampling type
    resampling = settings.Setting(0)
    #: Number of folds for K-fold cross validation
    n_folds = settings.Setting(3)
    #: Stratified sampling for K-fold
    cv_stratified = settings.Setting(True)
    #: Number of repeats for ShuffleSplit sampling
    n_repeats = settings.Setting(3)
    #: ShuffleSplit sample size
    sample_size = settings.Setting(9)
    #: Stratified sampling for Random Sampling
    shuffle_stratified = settings.Setting(True)
    # CV where nr. of feature values determines nr. of folds
    fold_feature = settings.ContextSetting(None)
    fold_feature_selected = settings.ContextSetting(False)

    use_rope = settings.Setting(False)
    rope = settings.Setting(0.1)
    comparison_criterion = settings.Setting(0, schema_only=True)

    TARGET_AVERAGE = "(Average over classes)"
    class_selection = settings.ContextSetting(TARGET_AVERAGE)

    class Error(OWWidget.Error):
        train_data_empty = Msg("Train dataset is empty.")
        test_data_empty = Msg("Test dataset is empty.")
        class_required = Msg("Train data input requires a target variable.")
        too_many_classes = Msg("Too many target variables.")
        class_required_test = Msg("Test data input requires a target variable.")
        too_many_folds = Msg("Number of folds exceeds the data size")
        class_inconsistent = Msg("Test and train datasets "
                                 "have different target variables.")
        memory_error = Msg("Not enough memory.")
        no_class_values = Msg("Target variable has no values.")
        only_one_class_var_value = Msg("Target variable has only one value.")
        test_data_incompatible = Msg(
            "Test data may be incompatible with train data.")

    class Warning(OWWidget.Warning):
        missing_data = \
            Msg("Instances with unknown target values were removed from{}data.")
        test_data_missing = Msg("Missing separate test data input.")
        scores_not_computed = Msg("Some scores could not be computed.")
        test_data_unused = Msg("Test data is present but unused. "
                               "Select 'Test on test data' to use it.")

    class Information(OWWidget.Information):
        data_sampled = Msg("Train data has been sampled")
        test_data_sampled = Msg("Test data has been sampled")
        test_data_transformed = Msg(
            "Test data has been transformed to match the train data.")

    def __init__(self):
        super().__init__()

        self.data = None
        self.test_data = None
        self.preprocessor = None
        self.train_data_missing_vals = False
        self.test_data_missing_vals = False
        self.scorers = []
        self.__pending_comparison_criterion = self.comparison_criterion

        #: An Ordered dictionary with current inputs and their testing results.
        self.learners = OrderedDict()  # type: Dict[Any, Input]

        self.__state = State.Waiting
        # Do we need to [re]test any learners, set by _invalidate and
        # cleared by __update
        self.__needupdate = False
        self.__task = None  # type: Optional[TaskState]
        self.__executor = ThreadExecutor()

        self.info.set_input_summary(self.info.NoInput)
        self.info.set_output_summary(self.info.NoOutput)

        sbox = gui.vBox(self.controlArea, "Sampling")
        rbox = gui.radioButtons(
            sbox, self, "resampling", callback=self._param_changed)

        gui.appendRadioButton(rbox, "Cross validation")
        ibox = gui.indentedBox(rbox)
        gui.comboBox(
            ibox, self, "n_folds", label="Number of folds: ",
            items=[str(x) for x in self.NFolds],
            orientation=Qt.Horizontal, callback=self.kfold_changed)
        gui.checkBox(
            ibox, self, "cv_stratified", "Stratified",
            callback=self.kfold_changed)
        gui.appendRadioButton(rbox, "Cross validation by feature")
        ibox = gui.indentedBox(rbox)
        self.feature_model = DomainModel(
            order=DomainModel.METAS, valid_types=DiscreteVariable)
        self.features_combo = gui.comboBox(
            ibox, self, "fold_feature", model=self.feature_model,
            orientation=Qt.Horizontal, searchable=True,
            callback=self.fold_feature_changed)

        gui.appendRadioButton(rbox, "Random sampling")
        ibox = gui.indentedBox(rbox)
        gui.comboBox(
            ibox, self, "n_repeats", label="Repeat train/test: ",
            items=[str(x) for x in self.NRepeats], orientation=Qt.Horizontal,
            callback=self.shuffle_split_changed
        )
        gui.comboBox(
            ibox, self, "sample_size", label="Training set size: ",
            items=["{} %".format(x) for x in self.SampleSizes],
            orientation=Qt.Horizontal, callback=self.shuffle_split_changed
        )
        gui.checkBox(
            ibox, self, "shuffle_stratified", "Stratified",
            callback=self.shuffle_split_changed)

        gui.appendRadioButton(rbox, "Leave one out")

        gui.appendRadioButton(rbox, "Test on train data")
        gui.appendRadioButton(rbox, "Test on test data")

        self.cbox = gui.vBox(self.controlArea, "Target Class")
        self.class_selection_combo = gui.comboBox(
            self.cbox, self, "class_selection", items=[],
            sendSelectedValue=True, contentsLength=8, searchable=True,
            callback=self._on_target_class_changed
        )

        self.modcompbox = box = gui.vBox(self.controlArea, "Model Comparison")
        gui.comboBox(
            box, self, "comparison_criterion",
            callback=self.update_comparison_table)

        hbox = gui.hBox(box)
        gui.checkBox(hbox, self, "use_rope",
                     "Negligible difference: ",
                     callback=self._on_use_rope_changed)
        gui.lineEdit(hbox, self, "rope", validator=QDoubleValidator(),
                     controlWidth=70, callback=self.update_comparison_table,
                     alignment=Qt.AlignRight)
        self.controls.rope.setEnabled(self.use_rope)

        gui.rubber(self.controlArea)
        self.score_table = ScoreTable(self)
        self.score_table.shownScoresChanged.connect(self.update_stats_model)
        view = self.score_table.view
        view.setSizeAdjustPolicy(view.AdjustToContents)

        box = gui.vBox(self.mainArea, "Evaluation Results")
        box.layout().addWidget(self.score_table.view)

        self.compbox = box = gui.vBox(self.mainArea, box="Model comparison")
        table = self.comparison_table = QTableWidget(
            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,
            selectionMode=QTableWidget.NoSelection)
        table.setSizeAdjustPolicy(table.AdjustToContents)
        header = table.verticalHeader()
        header.setSectionResizeMode(QHeaderView.Fixed)
        header.setSectionsClickable(False)

        header = table.horizontalHeader()
        header.setTextElideMode(Qt.ElideRight)
        header.setDefaultAlignment(Qt.AlignCenter)
        header.setSectionsClickable(False)
        header.setStretchLastSection(False)
        header.setSectionResizeMode(QHeaderView.ResizeToContents)
        avg_width = self.fontMetrics().averageCharWidth()
        header.setMinimumSectionSize(8 * avg_width)
        header.setMaximumSectionSize(15 * avg_width)
        header.setDefaultSectionSize(15 * avg_width)
        box.layout().addWidget(table)
        box.layout().addWidget(QLabel(
            "<small>Table shows probabilities that the score for the model in "
            "the row is higher than that of the model in the column. "
            "Small numbers show the probability that the difference is "
            "negligible.</small>", wordWrap=True))

    @staticmethod
    def sizeHint():
        return QSize(780, 1)

    def _update_controls(self):
        self.fold_feature = None
        self.feature_model.set_domain(None)
        if self.data:
            self.feature_model.set_domain(self.data.domain)
            if self.fold_feature is None and self.feature_model:
                self.fold_feature = self.feature_model[0]
        enabled = bool(self.feature_model)
        self.controls.resampling.buttons[
            OWTestAndScore.FeatureFold].setEnabled(enabled)
        self.features_combo.setEnabled(enabled)
        if self.resampling == OWTestAndScore.FeatureFold and not enabled:
            self.resampling = OWTestAndScore.KFold

    @Inputs.learner
    def set_learner(self, learner, key):
        """
        Set the input `learner` for `key`.

        Parameters
        ----------
        learner : Optional[Orange.base.Learner]
        key : Any
        """
        if key in self.learners and learner is None:
            # Removed
            self._invalidate([key])
            del self.learners[key]
        elif learner is not None:
            self.learners[key] = InputLearner(learner, None, None)
            self._invalidate([key])

    @Inputs.train_data
    def set_train_data(self, data):
        """
        Set the input training dataset.

        Parameters
        ----------
        data : Optional[Orange.data.Table]
        """
        self.cancel()
        self.Information.data_sampled.clear()
        self.Error.train_data_empty.clear()
        self.Error.class_required.clear()
        self.Error.too_many_classes.clear()
        self.Error.no_class_values.clear()
        self.Error.only_one_class_var_value.clear()
        if data is not None and not data:
            self.Error.train_data_empty()
            data = None
        if data:
            conds = [not data.domain.class_vars,
                     len(data.domain.class_vars) > 1,
                     np.isnan(data.Y).all(),
                     data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]
            errors = [self.Error.class_required,
                      self.Error.too_many_classes,
                      self.Error.no_class_values,
                      self.Error.only_one_class_var_value]
            for cond, error in zip(conds, errors):
                if cond:
                    error()
                    data = None
                    break

        if isinstance(data, SqlTable):
            if data.approx_len() < AUTO_DL_LIMIT:
                data = Table(data)
            else:
                self.Information.data_sampled()
                data_sample = data.sample_time(1, no_cache=True)
                data_sample.download_data(AUTO_DL_LIMIT, partial=True)
                data = Table(data_sample)

        self.train_data_missing_vals = \
            data is not None and np.isnan(data.Y).any()
        if self.train_data_missing_vals or self.test_data_missing_vals:
            self.Warning.missing_data(self._which_missing_data())
            if data:
                data = HasClass()(data)
        else:
            self.Warning.missing_data.clear()

        self.data = data
        self.closeContext()
        self._update_scorers()
        self._update_controls()
        if data is not None:
            self._update_class_selection()
            self.openContext(data.domain)
            if self.fold_feature_selected and bool(self.feature_model):
                self.resampling = OWTestAndScore.FeatureFold
        self._invalidate()

    @Inputs.test_data
    def set_test_data(self, data):
        # type: (Orange.data.Table) -> None
        """
        Set the input separate testing dataset.

        Parameters
        ----------
        data : Optional[Orange.data.Table]
        """
        self.Information.test_data_sampled.clear()
        self.Error.test_data_empty.clear()
        if data is not None and not data:
            self.Error.test_data_empty()
            data = None
        if data and not data.domain.class_var:
            self.Error.class_required_test()
            data = None
        else:
            self.Error.class_required_test.clear()

        if isinstance(data, SqlTable):
            if data.approx_len() < AUTO_DL_LIMIT:
                data = Table(data)
            else:
                self.Information.test_data_sampled()
                data_sample = data.sample_time(1, no_cache=True)
                data_sample.download_data(AUTO_DL_LIMIT, partial=True)
                data = Table(data_sample)

        self.test_data_missing_vals = \
            data is not None and np.isnan(data.Y).any()
        if self.train_data_missing_vals or self.test_data_missing_vals:
            self.Warning.missing_data(self._which_missing_data())
            if data:
                data = HasClass()(data)
        else:
            self.Warning.missing_data.clear()

        self.test_data = data
        if self.resampling == OWTestAndScore.TestOnTest:
            self._invalidate()

    def _which_missing_data(self):
        return {(True, True): " ",  # both, don't specify
                (True, False): " train ",
                (False, True): " test "}[(self.train_data_missing_vals,
                                          self.test_data_missing_vals)]

    # List of scorers shouldn't be retrieved globally, when the module is
    # loading since add-ons could have registered additional scorers.
    # It could have been cached but
    # - we don't gain much with it
    # - it complicates the unit tests
    def _update_scorers(self):
        if self.data and self.data.domain.class_var:
            new_scorers = usable_scorers(self.data.domain.class_var)
        else:
            new_scorers = []
        # Don't unnecessarily reset the combo because this would always reset
        # comparison_criterion; we also set it explicitly, though, for clarity
        if new_scorers != self.scorers:
            self.scorers = new_scorers
            combo = self.controls.comparison_criterion
            combo.clear()
            combo.addItems([scorer.long_name or scorer.name
                            for scorer in self.scorers])
            self.comparison_criterion = 0
        if self.__pending_comparison_criterion is not None:
            # Check for the unlikely case that some scorers have been removed
            # from modules
            if self.__pending_comparison_criterion < len(self.scorers):
                self.comparison_criterion = self.__pending_comparison_criterion
            self.__pending_comparison_criterion = None
        self._update_compbox_title()

    def _update_compbox_title(self):
        criterion = self.comparison_criterion
        if criterion < len(self.scorers):
            scorer = self.scorers[criterion]()
            self.compbox.setTitle(f"Model Comparison by {scorer.name}")
        else:
            self.compbox.setTitle(f"Model Comparison")

    @Inputs.preprocessor
    def set_preprocessor(self, preproc):
        """
        Set the input preprocessor to apply on the training data.
        """
        self.preprocessor = preproc
        self._invalidate()

    def handleNewSignals(self):
        """Reimplemented from OWWidget.handleNewSignals."""
        self._update_class_selection()
        self.score_table.update_header(self.scorers)
        self._update_view_enabled()
        self.update_stats_model()
        self.set_input_summary()
        if self.__needupdate:
            self.__update()

    def set_input_summary(self):
        summary, details, kwargs = self.info.NoInput, "", {}
        if self.data and self.test_data:
            summary = f"{self.info.format_number(len(self.data))}," \
                      f" {self.info.format_number(len(self.test_data))}"
            details = format_multiple_summaries([
                ("Data", self.data), ("Test data", self.test_data)
            ])
            kwargs = {"format": Qt.RichText}
        elif self.data and not self.test_data:
            summary, details = len(self.data), format_summary_details(self.data)
        elif self.test_data and not self.data:
            summary = len(self.test_data)
            details = format_summary_details(self.test_data)
        self.info.set_input_summary(summary, details, **kwargs)

    def kfold_changed(self):
        self.resampling = OWTestAndScore.KFold
        self._param_changed()

    def fold_feature_changed(self):
        self.resampling = OWTestAndScore.FeatureFold
        self._param_changed()

    def shuffle_split_changed(self):
        self.resampling = OWTestAndScore.ShuffleSplit
        self._param_changed()

    def _param_changed(self):
        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)
        self._update_view_enabled()
        self._invalidate()
        self.__update()

    def _update_view_enabled(self):
        self.comparison_table.setEnabled(
            self.resampling == OWTestAndScore.KFold
            and len(self.learners) > 1
            and self.data is not None)
        self.score_table.view.setEnabled(
            self.data is not None)

    def update_stats_model(self):
        # Update the results_model with up to date scores.
        # Note: The target class specific scores (if requested) are
        # computed as needed in this method.
        model = self.score_table.model
        # clear the table model, but preserving the header labels
        for r in reversed(range(model.rowCount())):
            model.takeRow(r)

        target_index = None
        if self.data is not None:
            class_var = self.data.domain.class_var
            if self.data.domain.has_discrete_class and \
                            self.class_selection != self.TARGET_AVERAGE:
                target_index = class_var.values.index(self.class_selection)
        else:
            class_var = None

        errors = []
        has_missing_scores = False

        names = []
        for key, slot in self.learners.items():
            name = learner_name(slot.learner)
            names.append(name)
            head = QStandardItem(name)
            head.setData(key, Qt.UserRole)
            results = slot.results
            if results is not None and results.success:
                train = QStandardItem("{:.3f}".format(results.value.train_time))
                train.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
                train.setData(key, Qt.UserRole)
                test = QStandardItem("{:.3f}".format(results.value.test_time))
                test.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
                test.setData(key, Qt.UserRole)
                row = [head, train, test]
            else:
                row = [head]
            if isinstance(results, Try.Fail):
                head.setToolTip(str(results.exception))
                head.setText("{} (error)".format(name))
                head.setForeground(QtGui.QBrush(Qt.red))
                if isinstance(results.exception, DomainTransformationError) \
                        and self.resampling == self.TestOnTest:
                    self.Error.test_data_incompatible()
                    self.Information.test_data_transformed.clear()
                else:
                    errors.append("{name} failed with error:\n"
                                  "{exc.__class__.__name__}: {exc!s}"
                                  .format(name=name, exc=slot.results.exception)
                                  )

            if class_var is not None and class_var.is_discrete and \
                    target_index is not None:
                if slot.results is not None and slot.results.success:
                    ovr_results = results_one_vs_rest(
                        slot.results.value, target_index)

                    # Cell variable is used immediatelly, it's not stored
                    # pylint: disable=cell-var-from-loop
                    stats = [Try(scorer_caller(scorer, ovr_results, target=1))
                             for scorer in self.scorers]
                else:
                    stats = None
            else:
                stats = slot.stats

            if stats is not None:
                for stat, scorer in zip(stats, self.scorers):
                    item = QStandardItem()
                    item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
                    if stat.success:
                        item.setData(float(stat.value[0]), Qt.DisplayRole)
                    else:
                        item.setToolTip(str(stat.exception))
                        if scorer.name in self.score_table.shown_scores:
                            has_missing_scores = True
                    row.append(item)

            model.appendRow(row)

        # Resort rows based on current sorting
        header = self.score_table.view.horizontalHeader()
        model.sort(
            header.sortIndicatorSection(),
            header.sortIndicatorOrder()
        )
        self._set_comparison_headers(names)

        self.error("\n".join(errors), shown=bool(errors))
        self.Warning.scores_not_computed(shown=has_missing_scores)

    def _on_use_rope_changed(self):
        self.controls.rope.setEnabled(self.use_rope)
        self.update_comparison_table()

    def update_comparison_table(self):
        self.comparison_table.clearContents()
        slots = self._successful_slots()
        if not (slots and self.scorers):
            return
        names = [learner_name(slot.learner) for slot in slots]
        self._set_comparison_headers(names)
        if self.resampling == OWTestAndScore.KFold:
            scores = self._scores_by_folds(slots)
            self._fill_table(names, scores)

    def _successful_slots(self):
        model = self.score_table.model
        proxy = self.score_table.sorted_model

        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)
                for row in range(proxy.rowCount()))
        slots = [slot for slot in (self.learners[key] for key in keys)
                 if slot.results is not None and slot.results.success]
        return slots

    def _set_comparison_headers(self, names):
        table = self.comparison_table
        try:
            # Prevent glitching during update
            table.setUpdatesEnabled(False)
            header = table.horizontalHeader()
            if len(names) > 2:
                header.setSectionResizeMode(QHeaderView.Stretch)
            else:
                header.setSectionResizeMode(QHeaderView.Fixed)
            table.setRowCount(len(names))
            table.setColumnCount(len(names))
            table.setVerticalHeaderLabels(names)
            table.setHorizontalHeaderLabels(names)
        finally:
            table.setUpdatesEnabled(True)

    def _scores_by_folds(self, slots):
        scorer = self.scorers[self.comparison_criterion]()
        self._update_compbox_title()
        if scorer.is_binary:
            if self.class_selection != self.TARGET_AVERAGE:
                class_var = self.data.domain.class_var
                target_index = class_var.values.index(self.class_selection)
                kw = dict(target=target_index)
            else:
                kw = dict(average='weighted')
        else:
            kw = {}

        def call_scorer(results):
            def thunked():
                return scorer.scores_by_folds(results.value, **kw).flatten()

            return thunked

        scores = [Try(call_scorer(slot.results)) for slot in slots]
        scores = [score.value if score.success else None for score in scores]
        # `None in scores doesn't work -- these are np.arrays)
        if any(score is None for score in scores):
            self.Warning.scores_not_computed()
        return scores

    def _fill_table(self, names, scores):
        table = self.comparison_table
        for row, row_name, row_scores in zip(count(), names, scores):
            for col, col_name, col_scores in zip(range(row), names, scores):
                if row_scores is None or col_scores is None:
                    continue
                if self.use_rope and self.rope:
                    p0, rope, p1 = baycomp.two_on_single(
                        row_scores, col_scores, self.rope)
                    if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):
                        self._set_cells_na(table, row, col)
                        continue
                    self._set_cell(table, row, col,
                                   f"{p0:.3f}<br/><small>{rope:.3f}</small>",
                                   f"p({row_name} > {col_name}) = {p0:.3f}\n"
                                   f"p({row_name} = {col_name}) = {rope:.3f}")
                    self._set_cell(table, col, row,
                                   f"{p1:.3f}<br/><small>{rope:.3f}</small>",
                                   f"p({col_name} > {row_name}) = {p1:.3f}\n"
                                   f"p({col_name} = {row_name}) = {rope:.3f}")
                else:
                    p0, p1 = baycomp.two_on_single(row_scores, col_scores)
                    if np.isnan(p0) or np.isnan(p1):
                        self._set_cells_na(table, row, col)
                        continue
                    self._set_cell(table, row, col,
                                   f"{p0:.3f}",
                                   f"p({row_name} > {col_name}) = {p0:.3f}")
                    self._set_cell(table, col, row,
                                   f"{p1:.3f}",
                                   f"p({col_name} > {row_name}) = {p1:.3f}")

    @classmethod
    def _set_cells_na(cls, table, row, col):
        cls._set_cell(table, row, col, "NA", "comparison cannot be computed")
        cls._set_cell(table, col, row, "NA", "comparison cannot be computed")

    @staticmethod
    def _set_cell(table, row, col, label, tooltip):
        item = QLabel(label)
        item.setToolTip(tooltip)
        item.setAlignment(Qt.AlignCenter)
        table.setCellWidget(row, col, item)

    def _update_class_selection(self):
        self.class_selection_combo.setCurrentIndex(-1)
        self.class_selection_combo.clear()
        if not self.data:
            return

        if self.data.domain.has_discrete_class:
            self.cbox.setVisible(True)
            class_var = self.data.domain.class_var
            items = (self.TARGET_AVERAGE, ) + class_var.values
            self.class_selection_combo.addItems(items)

            class_index = 0
            if self.class_selection in class_var.values:
                class_index = class_var.values.index(self.class_selection) + 1

            self.class_selection_combo.setCurrentIndex(class_index)
            self.class_selection = items[class_index]
        else:
            self.cbox.setVisible(False)

    def _on_target_class_changed(self):
        self.update_stats_model()
        self.update_comparison_table()

    def _invalidate(self, which=None):
        self.cancel()
        self.fold_feature_selected = \
            self.resampling == OWTestAndScore.FeatureFold
        # Invalidate learner results for `which` input keys
        # (if None then all learner results are invalidated)
        if which is None:
            which = self.learners.keys()

        model = self.score_table.model
        statmodelkeys = [model.item(row, 0).data(Qt.UserRole)
                         for row in range(model.rowCount())]

        for key in which:
            self.learners[key] = \
                self.learners[key]._replace(results=None, stats=None)

            if key in statmodelkeys:
                row = statmodelkeys.index(key)
                for c in range(1, model.columnCount()):
                    item = model.item(row, c)
                    if item is not None:
                        item.setData(None, Qt.DisplayRole)
                        item.setData(None, Qt.ToolTipRole)

        self.comparison_table.clearContents()

        self.__needupdate = True

    def commit(self):
        """
        Commit the results to output.
        """
        self.Error.memory_error.clear()
        valid = [slot for slot in self.learners.values()
                 if slot.results is not None and slot.results.success]
        combined = None
        predictions = None
        if valid:
            # Evaluation results
            combined = results_merge([slot.results.value for slot in valid])
            combined.learner_names = [learner_name(slot.learner)
                                      for slot in valid]

            # Predictions & Probabilities
            try:
                predictions = combined.get_augmented_data(combined.learner_names)
            except MemoryError:
                self.Error.memory_error()

        summary = len(predictions) if predictions else self.info.NoOutput
        details = format_summary_details(predictions) if predictions else ""
        self.info.set_output_summary(summary, details)

        self.Outputs.evaluations_results.send(combined)
        self.Outputs.predictions.send(predictions)

    def send_report(self):
        """Report on the testing schema and results"""
        if not self.data or not self.learners:
            return
        if self.resampling == self.KFold:
            stratified = 'Stratified ' if self.cv_stratified else ''
            items = [("Sampling type", "{}{}-fold Cross validation".
                      format(stratified, self.NFolds[self.n_folds]))]
        elif self.resampling == self.LeaveOneOut:
            items = [("Sampling type", "Leave one out")]
        elif self.resampling == self.ShuffleSplit:
            stratified = 'Stratified ' if self.shuffle_stratified else ''
            items = [("Sampling type",
                      "{}Shuffle split, {} random samples with {}% data "
                      .format(stratified, self.NRepeats[self.n_repeats],
                              self.SampleSizes[self.sample_size]))]
        elif self.resampling == self.TestOnTrain:
            items = [("Sampling type", "No sampling, test on training data")]
        elif self.resampling == self.TestOnTest:
            items = [("Sampling type", "No sampling, test on testing data")]
        else:
            items = []
        if self.data.domain.has_discrete_class:
            items += [("Target class", self.class_selection.strip("()"))]
        if items:
            self.report_items("Settings", items)
        self.report_table("Scores", self.score_table.view)

    @classmethod
    def migrate_settings(cls, settings_, version):
        if version < 2:
            if settings_["resampling"] > 0:
                settings_["resampling"] += 1
        if version < 3:
            # Older version used an incompatible context handler
            settings_["context_settings"] = [
                c for c in settings_.get("context_settings", ())
                if not hasattr(c, 'classes')]

    @Slot(float)
    def setProgressValue(self, value):
        self.progressBarSet(value)

    def __update(self):
        self.__needupdate = False

        assert self.__task is None or self.__state == State.Running
        if self.__state == State.Running:
            self.cancel()

        self.Warning.test_data_unused.clear()
        self.Error.test_data_incompatible.clear()
        self.Warning.test_data_missing.clear()
        self.Information.test_data_transformed(
            shown=self.resampling == self.TestOnTest
            and self.data is not None
            and self.test_data is not None
            and self.data.domain.attributes != self.test_data.domain.attributes)
        self.warning()
        self.Error.class_inconsistent.clear()
        self.Error.too_many_folds.clear()
        self.error()

        # check preconditions and return early
        if self.data is None:
            self.__state = State.Waiting
            self.commit()
            return
        if not self.learners:
            self.__state = State.Waiting
            self.commit()
            return
        if self.resampling == OWTestAndScore.KFold and \
                len(self.data) < self.NFolds[self.n_folds]:
            self.Error.too_many_folds()
            self.__state = State.Waiting
            self.commit()
            return

        elif self.resampling == OWTestAndScore.TestOnTest:
            if self.test_data is None:
                if not self.Error.test_data_empty.is_shown():
                    self.Warning.test_data_missing()
                self.__state = State.Waiting
                self.commit()
                return
            elif self.test_data.domain.class_var != self.data.domain.class_var:
                self.Error.class_inconsistent()
                self.__state = State.Waiting
                self.commit()
                return

        elif self.test_data is not None:
            self.Warning.test_data_unused()

        rstate = 42
        # items in need of an update
        items = [(key, slot) for key, slot in self.learners.items()
                 if slot.results is None]
        learners = [slot.learner for _, slot in items]

        # deepcopy all learners as they are not thread safe (by virtue of
        # the base API). These will be the effective learner objects tested
        # but will be replaced with the originals on return (see restore
        # learners bellow)
        learners_c = [copy.deepcopy(learner) for learner in learners]

        if self.resampling == OWTestAndScore.TestOnTest:
            test_f = partial(
                Orange.evaluation.TestOnTestData(
                    store_data=True, store_models=True),
                self.data, self.test_data, learners_c, self.preprocessor
            )
        else:
            if self.resampling == OWTestAndScore.KFold:
                sampler = Orange.evaluation.CrossValidation(
                    k=self.NFolds[self.n_folds],
                    random_state=rstate)
            elif self.resampling == OWTestAndScore.FeatureFold:
                sampler = Orange.evaluation.CrossValidationFeature(
                    feature=self.fold_feature)
            elif self.resampling == OWTestAndScore.LeaveOneOut:
                sampler = Orange.evaluation.LeaveOneOut()
            elif self.resampling == OWTestAndScore.ShuffleSplit:
                sampler = Orange.evaluation.ShuffleSplit(
                    n_resamples=self.NRepeats[self.n_repeats],
                    train_size=self.SampleSizes[self.sample_size] / 100,
                    test_size=None,
                    stratified=self.shuffle_stratified,
                    random_state=rstate)
            elif self.resampling == OWTestAndScore.TestOnTrain:
                sampler = Orange.evaluation.TestOnTrainingData(
                    store_models=True)
            else:
                assert False, "self.resampling %s" % self.resampling

            sampler.store_data = True
            test_f = partial(
                sampler, self.data, learners_c, self.preprocessor)

        def replace_learners(evalfunc, *args, **kwargs):
            res = evalfunc(*args, **kwargs)
            assert all(lc is lo for lc, lo in zip(learners_c, res.learners))
            res.learners[:] = learners
            return res

        test_f = partial(replace_learners, test_f)

        self.__submit(test_f)

    def __submit(self, testfunc):
        # type: (Callable[[Callable[[float], None]], Results]) -> None
        """
        Submit a testing function for evaluation

        MUST not be called if an evaluation is already pending/running.
        Cancel the existing task first.

        Parameters
        ----------
        testfunc : Callable[[Callable[float]], Results])
            Must be a callable taking a single `callback` argument and
            returning a Results instance
        """
        assert self.__state != State.Running
        # Setup the task
        task = TaskState()

        def progress_callback(finished):
            if task.is_interruption_requested():
                raise UserInterrupt()
            task.set_progress_value(100 * finished)

        testfunc = partial(testfunc, callback=progress_callback)
        task.start(self.__executor, testfunc)

        task.progress_changed.connect(self.setProgressValue)
        task.watcher.finished.connect(self.__task_complete)

        self.Outputs.evaluations_results.invalidate()
        self.Outputs.predictions.invalidate()
        self.progressBarInit()
        self.setStatusMessage("Running")

        self.__state = State.Running
        self.__task = task

    @Slot(object)
    def __task_complete(self, f: 'Future[Results]'):
        # handle a completed task
        assert self.thread() is QThread.currentThread()
        assert self.__task is not None and self.__task.future is f
        self.progressBarFinished()
        self.setStatusMessage("")
        assert f.done()
        self.__task = None
        self.__state = State.Done
        try:
            results = f.result()    # type: Results
            learners = results.learners  # type: List[Learner]
        except Exception as er:  # pylint: disable=broad-except
            log.exception("testing error (in __task_complete):",
                          exc_info=True)
            self.error("\n".join(traceback.format_exception_only(type(er), er)))
            return

        learner_key = {slot.learner: key for key, slot in
                       self.learners.items()}
        assert all(learner in learner_key for learner in learners)

        # Update the results for individual learners
        class_var = results.domain.class_var
        for learner, result in zip(learners, results.split_by_model()):
            stats = None
            if class_var.is_primitive():
                ex = result.failed[0]
                if ex:
                    stats = [Try.Fail(ex)] * len(self.scorers)
                    result = Try.Fail(ex)
                else:
                    stats = [Try(scorer_caller(scorer, result))
                             for scorer in self.scorers]
                    result = Try.Success(result)
            key = learner_key.get(learner)
            self.learners[key] = \
                self.learners[key]._replace(results=result, stats=stats)

        self.score_table.update_header(self.scorers)
        self.update_stats_model()
        self.update_comparison_table()

        self.commit()

    def cancel(self):
        """
        Cancel the current/pending evaluation (if any).
        """
        if self.__task is not None:
            assert self.__state == State.Running
            self.__state = State.Cancelled
            task, self.__task = self.__task, None
            task.cancel()
            task.progress_changed.disconnect(self.setProgressValue)
            task.watcher.finished.disconnect(self.__task_complete)
            add_task_to_dispose_queue(task)

            self.progressBarFinished()
            self.setStatusMessage("")

    def onDeleteWidget(self):
        self.cancel()
        self.__executor.shutdown(True)
        super().onDeleteWidget()


def add_task_to_dispose_queue(task: TaskState):
    # transfer ownership of task to Qt, and delete it after completion
    # all other signals from task should be disconnected.
    assert task.parent() is None
    app = QApplication.instance()
    task.setParent(app)
    task.watcher.finished.connect(task.deleteLater)


class UserInterrupt(BaseException):
    """
    A BaseException subclass used for cooperative task/thread cancellation
    """


def results_add_by_model(x, y):
    def is_empty(res):
        return (getattr(res, "models", None) is None
                and getattr(res, "row_indices", None) is None)

    if is_empty(x):
        return y
    elif is_empty(y):
        return x

    assert (x.row_indices == y.row_indices).all()
    assert (x.actual == y.actual).all()

    res = Orange.evaluation.Results()
    res.data = x.data
    res.domain = x.domain
    res.row_indices = x.row_indices
    res.folds = x.folds
    res.actual = x.actual
    res.predicted = np.vstack((x.predicted, y.predicted))
    if getattr(x, "probabilities", None) is not None \
            and getattr(y, "probabilities") is not None:
        res.probabilities = np.vstack((x.probabilities, y.probabilities))

    if x.models is not None:
        res.models = np.hstack((x.models, y.models))
    return res


def results_merge(results):
    return reduce(results_add_by_model, results, Orange.evaluation.Results())


def results_one_vs_rest(results, pos_index):
    from Orange.preprocess.transformation import Indicator
    actual = results.actual == pos_index
    predicted = results.predicted == pos_index
    if results.probabilities is not None:
        c = results.probabilities.shape[2]
        assert c >= 2
        neg_indices = [i for i in range(c) if i != pos_index]
        pos_prob = results.probabilities[:, :, [pos_index]]
        neg_prob = np.sum(results.probabilities[:, :, neg_indices],
                          axis=2, keepdims=True)
        probabilities = np.dstack((neg_prob, pos_prob))
    else:
        probabilities = None

    res = Orange.evaluation.Results()
    res.actual = actual
    res.predicted = predicted
    res.folds = results.folds
    res.row_indices = results.row_indices
    res.probabilities = probabilities

    value = results.domain.class_var.values[pos_index]
    class_var = Orange.data.DiscreteVariable(
        "I({}=={})".format(results.domain.class_var.name, value),
        values=("False", "True"),
        compute_value=Indicator(results.domain.class_var, pos_index)
    )
    domain = Orange.data.Domain(
        results.domain.attributes,
        [class_var],
        results.domain.metas
    )
    res.data = None
    res.domain = domain
    return res


if __name__ == "__main__":  # pragma: no cover
    filename = "iris"
    preview_data = Table(filename)
    if preview_data.domain.class_var.is_discrete:
        prev_learners = [lambda data: 1 / 0,
                         Orange.classification.LogisticRegressionLearner(),
                         Orange.classification.MajorityLearner(),
                         Orange.classification.NaiveBayesLearner()]
    else:
        prev_learners = [lambda data: 1 / 0,
                         Orange.regression.MeanLearner(),
                         Orange.regression.KNNRegressionLearner(),
                         Orange.regression.RidgeRegressionLearner()]

    WidgetPreview(OWTestAndScore).run(
        set_train_data=preview_data,
        set_test_data=preview_data,
        set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]
    )

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user