{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\n\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom ..base import Interface\nfrom ..utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with swallow_outputs() as cmo:\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom ..base import Interface\nfrom ..utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "c2a43ea1c",
                "new_commit_hash": "ebb42e630",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 10,
                            "length": 7
                        },
                        "new_range": {
                            "start": 10,
                            "length": 7
                        },
                        "section": ""
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "from contextlib import contextmanager",
                                "type": "added"
                            },
                            {
                                "content": "import logging",
                                "type": "context"
                            },
                            {
                                "content": "from os.path import (",
                                "type": "context"
                            },
                            {
                                "content": "    exists,",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "statement",
                            "name": "",
                            "content": "from contextlib import contextmanager",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 13,
                            "end_lineno": 13,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 381,
                            "length": 6
                        },
                        "new_range": {
                            "start": 381,
                            "length": 14
                        },
                        "section": "def test_discover_ds_trace(path, otherdir):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        assert_dict_equal(spec, goal)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@contextmanager",
                                "type": "added"
                            },
                            {
                                "content": "def _swallow_outputs(isatty=True):",
                                "type": "added"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        stdout = cmo.handles[0]",
                                "type": "added"
                            },
                            {
                                "content": "        stdout.isatty = lambda: isatty",
                                "type": "added"
                            },
                            {
                                "content": "        yield cmo",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def test_utils_suppress_similar():",
                                "type": "context"
                            },
                            {
                                "content": "    tu = TestUtils()",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "function",
                            "name": "_swallow_outputs",
                            "content": "def _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 385,
                            "end_lineno": 389,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 393,
                            "length": 21
                        },
                        "new_range": {
                            "start": 401,
                            "length": 34
                        },
                        "section": "def test_utils_suppress_similar():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                       status=\"ok\",",
                                "type": "context"
                            },
                            {
                                "content": "                       path=\"path{}\".format(i))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "    with _swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        cmo.isatty = lambda: True",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))",
                                "type": "context"
                            },
                            {
                                "content": "        assert_in(\"path8\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_not_in(\"suppressed\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "    with _swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))",
                                "type": "context"
                            },
                            {
                                "content": "        assert_in(\"path9\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_not_in(\"suppressed\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "    with _swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))",
                                "type": "context"
                            },
                            {
                                "content": "        assert_not_in(\"path10\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with _swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))",
                                "type": "added"
                            },
                            {
                                "content": "        assert_not_in(\"path10\", cmo.out)",
                                "type": "added"
                            },
                            {
                                "content": "        # We see an update for each result.",
                                "type": "added"
                            },
                            {
                                "content": "        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)",
                                "type": "added"
                            },
                            {
                                "content": "        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)",
                                "type": "added"
                            },
                            {
                                "content": "        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    with _swallow_outputs(isatty=False) as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))",
                                "type": "added"
                            },
                            {
                                "content": "        assert_in(\"path10\", cmo.out)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Check a chain of similar messages, split in half by a distinct one.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def n_foo_split_by_a_bar(number):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "function",
                            "name": "test_utils_suppress_similar",
                            "content": "def test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with swallow_outputs() as cmo:\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 384,
                            "end_lineno": 434,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 421,
                            "length": 13
                        },
                        "new_range": {
                            "start": 442,
                            "length": 13
                        },
                        "section": "def test_utils_suppress_similar():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                           status=\"ok\",",
                                "type": "context"
                            },
                            {
                                "content": "                           path=\"path\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "    with _swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))",
                                "type": "context"
                            },
                            {
                                "content": "        assert_in(\"path10\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_in(\"path19\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_not_in(\"suppressed\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "    with _swallow_outputs() as cmo:",
                                "type": "added"
                            },
                            {
                                "content": "        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))",
                                "type": "context"
                            },
                            {
                                "content": "        assert_in(\"path10\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_not_in(\"path20\", cmo.out)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "function",
                            "name": "test_utils_suppress_similar",
                            "content": "def test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with swallow_outputs() as cmo:\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 384,
                            "end_lineno": 434,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport inspect\nimport logging\nimport wrapt\nimport sys\nimport re\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nimport os.path as op\nfrom os.path import join as opj\nfrom os.path import isdir\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\nimport json\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import (\n    path_startswith,\n    path_is_subpath,\n    assure_unicode,\n    getargspec,\n    get_wrapped_class,\n)\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad import cfg as dlcfg\nfrom datalad.dochelpers import (\n    exc_str,\n    single_or_plural,\n)\n\nfrom datalad.ui import ui\nimport datalad.support.ansi_colors as ac\n\nfrom datalad.interface.base import default_logchannels\nfrom datalad.interface.base import get_allargs_as_kwargs\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom .results import known_result_xfms\nfrom datalad.config import ConfigManager\nfrom datalad.core.local.resulthooks import (\n    get_jsonhooks_from_config,\n    match_jsonhook2result,\n    run_jsonhook,\n)\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef cls2cmdlinename(cls):\n    \"Return the cmdline command name from an Interface class\"\n    r = re.compile(r'([a-z0-9])([A-Z])')\n    return r.sub('\\\\1-\\\\2', cls.__name__).lower()\n\n\n# TODO remove\n# only `drop` and `uninstall` are still using this\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.dirty:\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.core.local.save import Save\n        Save.__call__(dataset=ds, message=msg, updated=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\n# TODO remove\n# only `remove` and `uninstall` use this, the uses path `path_is_subpath`\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        path = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    path_drive, _ = op.splitdrive(path)\n    for p in values:\n        p_drive, _ = op.splitdrive(p)\n        # need to protect against unsupported use of relpath() with\n        # abspaths on windows from different drives (gh-3724)\n        if path_drive != p_drive:\n        # different drives, enough evidence for \"not under\"\n            continue\n        rpath = relpath(p, start=path)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,\n                                      spec, includeds=None):\n    \"\"\"Discover the edges and nodes in a dataset tree to given target paths\n\n    Parameters\n    ----------\n    basepath : path\n      Path to a start or top-level dataset. Really has to be a path to a\n      dataset!\n    targetpaths : list(path)\n      Any non-zero number of paths that are termination points for the\n      search algorithm. Can be paths to datasets, directories, or files\n      (and any combination thereof).\n    current_trace : list\n      For a top-level call this should probably always be `[]`\n    spec : dict\n      `content_by_ds`-style dictionary that will receive information about the\n      discovered datasets. Specifically, for each discovered dataset there\n      will be in item with its path under the key (path) of the respective\n      superdataset.\n    includeds : sequence, optional\n      Any paths given are treated as existing subdatasets, regardless of\n      whether they can be found in the filesystem. Such subdatasets will appear\n      under the key of the closest existing dataset in the `spec`.\n\n    Returns\n    -------\n    None\n      Function calls itself recursively and populates `spec` dict in-place.\n      Keys are dataset paths, values are sets of subdataset paths\n    \"\"\"\n    # convert to set for faster lookup\n    includeds = includeds if isinstance(includeds, set) else \\\n        set() if includeds is None else set(includeds)\n    # this beast walks the directory tree from a given `basepath` until\n    # it discovers any of the given `targetpaths`\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(basepath)\n    if valid_repo:\n        # we are passing into a new dataset, extend the dataset trace\n        current_trace = current_trace + [basepath]\n    # this edge is not done, we need to try to reach any downstream\n    # dataset\n    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)\n    # whether anything in this directory matched a targetpath\n    filematch = False\n    if isdir(basepath):\n        for p in listdir(basepath):\n            p = assure_unicode(opj(basepath, p))\n            if not isdir(p):\n                if p in targetpaths:\n                    filematch = True\n                # we cannot have anything below this one\n                continue\n            # OPT listdir might be large and we could have only few items\n            # in `targetpaths` -- so traverse only those in spec which have\n            # leading dir basepath\n            # filter targets matching this downward path\n            downward_targets = set(\n                t for t in targetpaths if path_startswith(t, p))\n            if not downward_targets:\n                continue\n            # remove the matching ones from the \"todo\" list\n            undiscovered_ds.difference_update(downward_targets)\n            # go one deeper\n            discover_dataset_trace_to_targets(\n                p, downward_targets, current_trace, spec,\n                includeds=includeds if not includeds else includeds.intersection(\n                    downward_targets))\n    undiscovered_ds = [t for t in undiscovered_ds\n                       if includeds and\n                          path_is_subpath(t, current_trace[-1]) and\n                          t in includeds]\n    if filematch or basepath in targetpaths or undiscovered_ds:\n        for i, p in enumerate(current_trace[:-1]):\n            # TODO RF prepare proper annotated path dicts\n            subds = spec.get(p, set())\n            subds.add(current_trace[i + 1])\n            spec[p] = subds\n        if undiscovered_ds:\n            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(\n                undiscovered_ds)\n\n\ndef get_result_filter(fx):\n    \"\"\"Wrap a filter into a helper to be able to accept additional\n    arguments, if the filter doesn't support it already\"\"\"\n    _fx = fx\n    if fx and not getargspec(fx).keywords:\n        def _fx(res, **kwargs):\n            return fx(res)\n    return _fx\n\n\ndef eval_results(func):\n    \"\"\"Decorator for return value evaluation of datalad commands.\n\n    Note, this decorator is only compatible with commands that return\n    status dict sequences!\n\n    Two basic modes of operation are supported: 1) \"generator mode\" that\n    `yields` individual results, and 2) \"list mode\" that returns a sequence of\n    results. The behavior can be selected via the kwarg `return_type`.\n    Default is \"list mode\".\n\n    This decorator implements common functionality for result rendering/output,\n    error detection/handling, and logging.\n\n    Result rendering/output can be triggered via the\n    `datalad.api.result-renderer` configuration variable, or the\n    `result_renderer` keyword argument of each decorated command. Supported\n    modes are: 'default' (one line per result with action, status, path,\n    and an optional message); 'json' (one object per result, like git-annex),\n    'json_pp' (like 'json', but pretty-printed spanning multiple lines),\n    'tailored' custom output formatting provided by each command\n    class (if any).\n\n    Error detection works by inspecting the `status` item of all result\n    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'\n    will cause an IncompleteResultsError exception to be raised that carries\n    the failed actions' status dictionaries in its `failed` attribute.\n\n    Status messages will be logged automatically, by default the following\n    association of result status and log channel will be used: 'ok' (debug),\n    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger\n    instances included in the results are used to capture the origin of a\n    status report.\n\n    Parameters\n    ----------\n    func: function\n      __call__ method of a subclass of Interface,\n      i.e. a datalad command definition\n    \"\"\"\n\n    @wrapt.decorator\n    def eval_func(wrapped, instance, args, kwargs):\n        lgr.log(2, \"Entered eval_func for %s\", func)\n        # for result filters\n        # we need to produce a dict with argname/argvalue pairs for all args\n        # incl. defaults and args given as positionals\n        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)\n\n        # determine the command class associated with `wrapped`\n        wrapped_class = get_wrapped_class(wrapped)\n\n        # retrieve common options from kwargs, and fall back on the command\n        # class attributes, or general defaults if needed\n        kwargs = kwargs.copy()  # we will pop, which might cause side-effect\n        common_params = {\n            p_name: kwargs.pop(\n                # go with any explicitly given default\n                p_name,\n                # otherwise determine the command class and pull any\n                # default set in that class\n                getattr(\n                    wrapped_class,\n                    p_name,\n                    # or the common default\n                    eval_defaults[p_name]))\n            for p_name in eval_params}\n\n        # short cuts and configured setup for common options\n        return_type = common_params['return_type']\n        result_filter = get_result_filter(common_params['result_filter'])\n        # resolve string labels for transformers too\n        result_xfm = known_result_xfms.get(\n            common_params['result_xfm'],\n            # use verbatim, if not a known label\n            common_params['result_xfm'])\n        result_renderer = common_params['result_renderer']\n        # TODO remove this conditional branch entirely, done outside\n        if not result_renderer:\n            result_renderer = dlcfg.get('datalad.api.result-renderer', None)\n        # look for potential override of logging behavior\n        result_log_level = dlcfg.get('datalad.log.result-level', None)\n\n        # query cfg for defaults\n        # .is_installed and .config can be costly, so ensure we do\n        # it only once. See https://github.com/datalad/datalad/issues/3575\n        dataset_arg = allkwargs.get('dataset', None)\n        from datalad.distribution.dataset import Dataset\n        ds = dataset_arg if isinstance(dataset_arg, Dataset) \\\n            else Dataset(dataset_arg) if dataset_arg else None\n        # do not reuse a dataset's existing config manager here\n        # they are configured to read the committed dataset configuration\n        # too. That means a datalad update can silently bring in new\n        # procedure definitions from the outside, and in some sense enable\n        # remote code execution by a 3rd-party\n        # To avoid that, create a new config manager that only reads local\n        # config (system and .git/config), plus any overrides given to this\n        # datalad session\n        proc_cfg = ConfigManager(\n            ds, source='local', overrides=dlcfg.overrides\n        ) if ds and ds.is_installed() else dlcfg\n\n        # look for hooks\n        hooks = get_jsonhooks_from_config(proc_cfg)\n\n        # this internal helper function actually drives the command\n        # generator-style, it may generate an exception if desired,\n        # on incomplete results\n        def generator_func(*_args, **_kwargs):\n            # flag whether to raise an exception\n            incomplete_results = []\n            # track what actions were performed how many times\n            action_summary = {}\n\n            # if a custom summary is to be provided, collect the results\n            # of the command execution\n            results = []\n            do_custom_result_summary = result_renderer in ('tailored', 'default') \\\n                and hasattr(wrapped_class, 'custom_result_summary_renderer')\n\n            # process main results\n            for r in _process_results(\n                    # execution\n                    wrapped(*_args, **_kwargs),\n                    wrapped_class,\n                    common_params['on_failure'],\n                    # bookkeeping\n                    action_summary,\n                    incomplete_results,\n                    # communication\n                    result_renderer,\n                    result_log_level,\n                    # let renderers get to see how a command was called\n                    allkwargs):\n                for hook, spec in hooks.items():\n                    # run the hooks before we yield the result\n                    # this ensures that they are executed before\n                    # a potentially wrapper command gets to act\n                    # on them\n                    if match_jsonhook2result(hook, r, spec['match']):\n                        lgr.debug('Result %s matches hook %s', r, hook)\n                        # a hook is also a command that yields results\n                        # so yield them outside too\n                        # users need to pay attention to void infinite\n                        # loops, i.e. when a hook yields a result that\n                        # triggers that same hook again\n                        for hr in run_jsonhook(hook, spec, r, dataset_arg):\n                            # apply same logic as for main results, otherwise\n                            # any filters would only tackle the primary results\n                            # and a mixture of return values could happen\n                            if not keep_result(hr, result_filter, **allkwargs):\n                                continue\n                            hr = xfm_result(hr, result_xfm)\n                            # rationale for conditional is a few lines down\n                            if hr:\n                                yield hr\n                if not keep_result(r, result_filter, **allkwargs):\n                    continue\n                r = xfm_result(r, result_xfm)\n                # in case the result_xfm decided to not give us anything\n                # exclude it from the results. There is no particular reason\n                # to do so other than that it was established behavior when\n                # this comment was written. This will not affect any real\n                # result record\n                if r:\n                    yield r\n\n                # collect if summary is desired\n                if do_custom_result_summary:\n                    results.append(r)\n\n            # result summary before a potential exception\n            # custom first\n            if do_custom_result_summary:\n                wrapped_class.custom_result_summary_renderer(results)\n            elif result_renderer == 'default' and action_summary and \\\n                    sum(sum(s.values()) for s in action_summary.values()) > 1:\n                # give a summary in default mode, when there was more than one\n                # action performed\n                ui.message(\"action summary:\\n  {}\".format(\n                    '\\n  '.join('{} ({})'.format(\n                        act,\n                        ', '.join('{}: {}'.format(status, action_summary[act][status])\n                                  for status in sorted(action_summary[act])))\n                                for act in sorted(action_summary))))\n\n            if incomplete_results:\n                raise IncompleteResultsError(\n                    failed=incomplete_results,\n                    msg=\"Command did not complete successfully\")\n\n        if return_type == 'generator':\n            # hand over the generator\n            lgr.log(2, \"Returning generator_func from eval_func for %s\", wrapped_class)\n            return generator_func(*args, **kwargs)\n        else:\n            @wrapt.decorator\n            def return_func(wrapped_, instance_, args_, kwargs_):\n                results = wrapped_(*args_, **kwargs_)\n                if inspect.isgenerator(results):\n                    # unwind generator if there is one, this actually runs\n                    # any processing\n                    results = list(results)\n                # render summaries\n                if not result_xfm and result_renderer in ('tailored', 'default'):\n                    # cannot render transformed results\n                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):\n                        wrapped_class.custom_result_summary_renderer(results)\n                if return_type == 'item-or-list' and \\\n                        len(results) < 2:\n                    return results[0] if results else None\n                else:\n                    return results\n            lgr.log(2, \"Returning return_func from eval_func for %s\", wrapped_class)\n            return return_func(generator_func)(*args, **kwargs)\n\n    return eval_func(func)\n\n\ndef default_result_renderer(res):\n    if res.get('status', None) != 'notneeded':\n        path = res['path']\n        path = str(path)\n        ui.message('{action}({status}): {path}{type}{msg}'.format(\n                action=ac.color_word(res['action'], ac.BOLD),\n                status=ac.color_status(res['status']),\n                path=relpath(path, res['refds']) if res.get('refds') else path,\n                type=' ({})'.format(\n                        ac.color_word(res['type'], ac.MAGENTA)\n                ) if 'type' in res else '',\n                msg=' [{}]'.format(\n                        res['message'][0] % res['message'][1:]\n                        if isinstance(res['message'], tuple) else res[\n                            'message'])\n                if res.get('message', None) else ''))\n\n\ndef _display_suppressed_message(nsimilar, ndisplayed):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)))\n\n\ndef _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions)\n\n\ndef keep_result(res, rfilter, **kwargs):\n    if not rfilter:\n        return True\n    try:\n        if not rfilter(res, **kwargs):\n            # give the slightest indication which filter was employed\n            raise ValueError(\n                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))\n    except ValueError as e:\n        # make sure to report the excluded result to massively improve\n        # debugging experience\n        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)\n        return False\n    return True\n\n\ndef xfm_result(res, xfm):\n    if not xfm:\n        return res\n\n    return xfm(res)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport inspect\nimport logging\nimport wrapt\nimport sys\nimport re\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nimport os.path as op\nfrom os.path import join as opj\nfrom os.path import isdir\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\nimport json\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import (\n    path_startswith,\n    path_is_subpath,\n    assure_unicode,\n    getargspec,\n    get_wrapped_class,\n)\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad import cfg as dlcfg\nfrom datalad.dochelpers import (\n    exc_str,\n    single_or_plural,\n)\n\nfrom datalad.ui import ui\nimport datalad.support.ansi_colors as ac\n\nfrom datalad.interface.base import default_logchannels\nfrom datalad.interface.base import get_allargs_as_kwargs\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom .results import known_result_xfms\nfrom datalad.config import ConfigManager\nfrom datalad.core.local.resulthooks import (\n    get_jsonhooks_from_config,\n    match_jsonhook2result,\n    run_jsonhook,\n)\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef cls2cmdlinename(cls):\n    \"Return the cmdline command name from an Interface class\"\n    r = re.compile(r'([a-z0-9])([A-Z])')\n    return r.sub('\\\\1-\\\\2', cls.__name__).lower()\n\n\n# TODO remove\n# only `drop` and `uninstall` are still using this\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.dirty:\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.core.local.save import Save\n        Save.__call__(dataset=ds, message=msg, updated=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\n# TODO remove\n# only `remove` and `uninstall` use this, the uses path `path_is_subpath`\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        path = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    path_drive, _ = op.splitdrive(path)\n    for p in values:\n        p_drive, _ = op.splitdrive(p)\n        # need to protect against unsupported use of relpath() with\n        # abspaths on windows from different drives (gh-3724)\n        if path_drive != p_drive:\n        # different drives, enough evidence for \"not under\"\n            continue\n        rpath = relpath(p, start=path)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,\n                                      spec, includeds=None):\n    \"\"\"Discover the edges and nodes in a dataset tree to given target paths\n\n    Parameters\n    ----------\n    basepath : path\n      Path to a start or top-level dataset. Really has to be a path to a\n      dataset!\n    targetpaths : list(path)\n      Any non-zero number of paths that are termination points for the\n      search algorithm. Can be paths to datasets, directories, or files\n      (and any combination thereof).\n    current_trace : list\n      For a top-level call this should probably always be `[]`\n    spec : dict\n      `content_by_ds`-style dictionary that will receive information about the\n      discovered datasets. Specifically, for each discovered dataset there\n      will be in item with its path under the key (path) of the respective\n      superdataset.\n    includeds : sequence, optional\n      Any paths given are treated as existing subdatasets, regardless of\n      whether they can be found in the filesystem. Such subdatasets will appear\n      under the key of the closest existing dataset in the `spec`.\n\n    Returns\n    -------\n    None\n      Function calls itself recursively and populates `spec` dict in-place.\n      Keys are dataset paths, values are sets of subdataset paths\n    \"\"\"\n    # convert to set for faster lookup\n    includeds = includeds if isinstance(includeds, set) else \\\n        set() if includeds is None else set(includeds)\n    # this beast walks the directory tree from a given `basepath` until\n    # it discovers any of the given `targetpaths`\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(basepath)\n    if valid_repo:\n        # we are passing into a new dataset, extend the dataset trace\n        current_trace = current_trace + [basepath]\n    # this edge is not done, we need to try to reach any downstream\n    # dataset\n    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)\n    # whether anything in this directory matched a targetpath\n    filematch = False\n    if isdir(basepath):\n        for p in listdir(basepath):\n            p = assure_unicode(opj(basepath, p))\n            if not isdir(p):\n                if p in targetpaths:\n                    filematch = True\n                # we cannot have anything below this one\n                continue\n            # OPT listdir might be large and we could have only few items\n            # in `targetpaths` -- so traverse only those in spec which have\n            # leading dir basepath\n            # filter targets matching this downward path\n            downward_targets = set(\n                t for t in targetpaths if path_startswith(t, p))\n            if not downward_targets:\n                continue\n            # remove the matching ones from the \"todo\" list\n            undiscovered_ds.difference_update(downward_targets)\n            # go one deeper\n            discover_dataset_trace_to_targets(\n                p, downward_targets, current_trace, spec,\n                includeds=includeds if not includeds else includeds.intersection(\n                    downward_targets))\n    undiscovered_ds = [t for t in undiscovered_ds\n                       if includeds and\n                          path_is_subpath(t, current_trace[-1]) and\n                          t in includeds]\n    if filematch or basepath in targetpaths or undiscovered_ds:\n        for i, p in enumerate(current_trace[:-1]):\n            # TODO RF prepare proper annotated path dicts\n            subds = spec.get(p, set())\n            subds.add(current_trace[i + 1])\n            spec[p] = subds\n        if undiscovered_ds:\n            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(\n                undiscovered_ds)\n\n\ndef get_result_filter(fx):\n    \"\"\"Wrap a filter into a helper to be able to accept additional\n    arguments, if the filter doesn't support it already\"\"\"\n    _fx = fx\n    if fx and not getargspec(fx).keywords:\n        def _fx(res, **kwargs):\n            return fx(res)\n    return _fx\n\n\ndef eval_results(func):\n    \"\"\"Decorator for return value evaluation of datalad commands.\n\n    Note, this decorator is only compatible with commands that return\n    status dict sequences!\n\n    Two basic modes of operation are supported: 1) \"generator mode\" that\n    `yields` individual results, and 2) \"list mode\" that returns a sequence of\n    results. The behavior can be selected via the kwarg `return_type`.\n    Default is \"list mode\".\n\n    This decorator implements common functionality for result rendering/output,\n    error detection/handling, and logging.\n\n    Result rendering/output can be triggered via the\n    `datalad.api.result-renderer` configuration variable, or the\n    `result_renderer` keyword argument of each decorated command. Supported\n    modes are: 'default' (one line per result with action, status, path,\n    and an optional message); 'json' (one object per result, like git-annex),\n    'json_pp' (like 'json', but pretty-printed spanning multiple lines),\n    'tailored' custom output formatting provided by each command\n    class (if any).\n\n    Error detection works by inspecting the `status` item of all result\n    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'\n    will cause an IncompleteResultsError exception to be raised that carries\n    the failed actions' status dictionaries in its `failed` attribute.\n\n    Status messages will be logged automatically, by default the following\n    association of result status and log channel will be used: 'ok' (debug),\n    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger\n    instances included in the results are used to capture the origin of a\n    status report.\n\n    Parameters\n    ----------\n    func: function\n      __call__ method of a subclass of Interface,\n      i.e. a datalad command definition\n    \"\"\"\n\n    @wrapt.decorator\n    def eval_func(wrapped, instance, args, kwargs):\n        lgr.log(2, \"Entered eval_func for %s\", func)\n        # for result filters\n        # we need to produce a dict with argname/argvalue pairs for all args\n        # incl. defaults and args given as positionals\n        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)\n\n        # determine the command class associated with `wrapped`\n        wrapped_class = get_wrapped_class(wrapped)\n\n        # retrieve common options from kwargs, and fall back on the command\n        # class attributes, or general defaults if needed\n        kwargs = kwargs.copy()  # we will pop, which might cause side-effect\n        common_params = {\n            p_name: kwargs.pop(\n                # go with any explicitly given default\n                p_name,\n                # otherwise determine the command class and pull any\n                # default set in that class\n                getattr(\n                    wrapped_class,\n                    p_name,\n                    # or the common default\n                    eval_defaults[p_name]))\n            for p_name in eval_params}\n\n        # short cuts and configured setup for common options\n        return_type = common_params['return_type']\n        result_filter = get_result_filter(common_params['result_filter'])\n        # resolve string labels for transformers too\n        result_xfm = known_result_xfms.get(\n            common_params['result_xfm'],\n            # use verbatim, if not a known label\n            common_params['result_xfm'])\n        result_renderer = common_params['result_renderer']\n        # TODO remove this conditional branch entirely, done outside\n        if not result_renderer:\n            result_renderer = dlcfg.get('datalad.api.result-renderer', None)\n        # look for potential override of logging behavior\n        result_log_level = dlcfg.get('datalad.log.result-level', None)\n\n        # query cfg for defaults\n        # .is_installed and .config can be costly, so ensure we do\n        # it only once. See https://github.com/datalad/datalad/issues/3575\n        dataset_arg = allkwargs.get('dataset', None)\n        from datalad.distribution.dataset import Dataset\n        ds = dataset_arg if isinstance(dataset_arg, Dataset) \\\n            else Dataset(dataset_arg) if dataset_arg else None\n        # do not reuse a dataset's existing config manager here\n        # they are configured to read the committed dataset configuration\n        # too. That means a datalad update can silently bring in new\n        # procedure definitions from the outside, and in some sense enable\n        # remote code execution by a 3rd-party\n        # To avoid that, create a new config manager that only reads local\n        # config (system and .git/config), plus any overrides given to this\n        # datalad session\n        proc_cfg = ConfigManager(\n            ds, source='local', overrides=dlcfg.overrides\n        ) if ds and ds.is_installed() else dlcfg\n\n        # look for hooks\n        hooks = get_jsonhooks_from_config(proc_cfg)\n\n        # this internal helper function actually drives the command\n        # generator-style, it may generate an exception if desired,\n        # on incomplete results\n        def generator_func(*_args, **_kwargs):\n            # flag whether to raise an exception\n            incomplete_results = []\n            # track what actions were performed how many times\n            action_summary = {}\n\n            # if a custom summary is to be provided, collect the results\n            # of the command execution\n            results = []\n            do_custom_result_summary = result_renderer in ('tailored', 'default') \\\n                and hasattr(wrapped_class, 'custom_result_summary_renderer')\n\n            # process main results\n            for r in _process_results(\n                    # execution\n                    wrapped(*_args, **_kwargs),\n                    wrapped_class,\n                    common_params['on_failure'],\n                    # bookkeeping\n                    action_summary,\n                    incomplete_results,\n                    # communication\n                    result_renderer,\n                    result_log_level,\n                    # let renderers get to see how a command was called\n                    allkwargs):\n                for hook, spec in hooks.items():\n                    # run the hooks before we yield the result\n                    # this ensures that they are executed before\n                    # a potentially wrapper command gets to act\n                    # on them\n                    if match_jsonhook2result(hook, r, spec['match']):\n                        lgr.debug('Result %s matches hook %s', r, hook)\n                        # a hook is also a command that yields results\n                        # so yield them outside too\n                        # users need to pay attention to void infinite\n                        # loops, i.e. when a hook yields a result that\n                        # triggers that same hook again\n                        for hr in run_jsonhook(hook, spec, r, dataset_arg):\n                            # apply same logic as for main results, otherwise\n                            # any filters would only tackle the primary results\n                            # and a mixture of return values could happen\n                            if not keep_result(hr, result_filter, **allkwargs):\n                                continue\n                            hr = xfm_result(hr, result_xfm)\n                            # rationale for conditional is a few lines down\n                            if hr:\n                                yield hr\n                if not keep_result(r, result_filter, **allkwargs):\n                    continue\n                r = xfm_result(r, result_xfm)\n                # in case the result_xfm decided to not give us anything\n                # exclude it from the results. There is no particular reason\n                # to do so other than that it was established behavior when\n                # this comment was written. This will not affect any real\n                # result record\n                if r:\n                    yield r\n\n                # collect if summary is desired\n                if do_custom_result_summary:\n                    results.append(r)\n\n            # result summary before a potential exception\n            # custom first\n            if do_custom_result_summary:\n                wrapped_class.custom_result_summary_renderer(results)\n            elif result_renderer == 'default' and action_summary and \\\n                    sum(sum(s.values()) for s in action_summary.values()) > 1:\n                # give a summary in default mode, when there was more than one\n                # action performed\n                ui.message(\"action summary:\\n  {}\".format(\n                    '\\n  '.join('{} ({})'.format(\n                        act,\n                        ', '.join('{}: {}'.format(status, action_summary[act][status])\n                                  for status in sorted(action_summary[act])))\n                                for act in sorted(action_summary))))\n\n            if incomplete_results:\n                raise IncompleteResultsError(\n                    failed=incomplete_results,\n                    msg=\"Command did not complete successfully\")\n\n        if return_type == 'generator':\n            # hand over the generator\n            lgr.log(2, \"Returning generator_func from eval_func for %s\", wrapped_class)\n            return generator_func(*args, **kwargs)\n        else:\n            @wrapt.decorator\n            def return_func(wrapped_, instance_, args_, kwargs_):\n                results = wrapped_(*args_, **kwargs_)\n                if inspect.isgenerator(results):\n                    # unwind generator if there is one, this actually runs\n                    # any processing\n                    results = list(results)\n                # render summaries\n                if not result_xfm and result_renderer in ('tailored', 'default'):\n                    # cannot render transformed results\n                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):\n                        wrapped_class.custom_result_summary_renderer(results)\n                if return_type == 'item-or-list' and \\\n                        len(results) < 2:\n                    return results[0] if results else None\n                else:\n                    return results\n            lgr.log(2, \"Returning return_func from eval_func for %s\", wrapped_class)\n            return return_func(generator_func)(*args, **kwargs)\n\n    return eval_func(func)\n\n\ndef default_result_renderer(res):\n    if res.get('status', None) != 'notneeded':\n        path = res['path']\n        path = str(path)\n        ui.message('{action}({status}): {path}{type}{msg}'.format(\n                action=ac.color_word(res['action'], ac.BOLD),\n                status=ac.color_status(res['status']),\n                path=relpath(path, res['refds']) if res.get('refds') else path,\n                type=' ({})'.format(\n                        ac.color_word(res['type'], ac.MAGENTA)\n                ) if 'type' in res else '',\n                msg=' [{}]'.format(\n                        res['message'][0] % res['message'][1:]\n                        if isinstance(res['message'], tuple) else res[\n                            'message'])\n                if res.get('message', None) else ''))\n\n\ndef _display_suppressed_message(nsimilar, ndisplayed, final=False):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)),\n                   cr=\"\\n\" if final else \"\\r\")\n\n\ndef _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10 if sys.stdout.isatty() else float(\"inf\")\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n                else:\n                    _display_suppressed_message(\n                        result_repetitions, render_n_repetitions)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions,\n                    final=True)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions, final=True)\n\n\ndef keep_result(res, rfilter, **kwargs):\n    if not rfilter:\n        return True\n    try:\n        if not rfilter(res, **kwargs):\n            # give the slightest indication which filter was employed\n            raise ValueError(\n                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))\n    except ValueError as e:\n        # make sure to report the excluded result to massively improve\n        # debugging experience\n        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)\n        return False\n    return True\n\n\ndef xfm_result(res, xfm):\n    if not xfm:\n        return res\n\n    return xfm(res)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "db03e38dc",
                "new_commit_hash": "67d62f588",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 514,
                            "length": 7
                        },
                        "new_range": {
                            "start": 514,
                            "length": 7
                        },
                        "section": "def default_result_renderer(res):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                if res.get('message', None) else ''))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def _display_suppressed_message(nsimilar, ndisplayed):",
                                "type": "deleted"
                            },
                            {
                                "content": "def _display_suppressed_message(nsimilar, ndisplayed, final=False):",
                                "type": "added"
                            },
                            {
                                "content": "    # +1 because there was the original result + nsimilar displayed.",
                                "type": "context"
                            },
                            {
                                "content": "    n_suppressed = nsimilar - ndisplayed + 1",
                                "type": "context"
                            },
                            {
                                "content": "    if n_suppressed > 0:",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "_display_suppressed_message",
                            "content": "def _display_suppressed_message(nsimilar, ndisplayed):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 517,
                            "end_lineno": 525,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 522,
                            "length": 7
                        },
                        "new_range": {
                            "start": 522,
                            "length": 8
                        },
                        "section": "def _display_suppressed_message(nsimilar, ndisplayed):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                   .format(n_suppressed,",
                                "type": "context"
                            },
                            {
                                "content": "                           single_or_plural(\"message has\",",
                                "type": "context"
                            },
                            {
                                "content": "                                            \"messages have\",",
                                "type": "context"
                            },
                            {
                                "content": "                                            n_suppressed, False)))",
                                "type": "deleted"
                            },
                            {
                                "content": "                                            n_suppressed, False)),",
                                "type": "added"
                            },
                            {
                                "content": "                   cr=\"\\n\" if final else \"\\r\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def _process_results(",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "_display_suppressed_message",
                            "content": "def _display_suppressed_message(nsimilar, ndisplayed):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 517,
                            "end_lineno": 525,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 546,
                            "length": 7
                        },
                        "new_range": {
                            "start": 547,
                            "length": 7
                        },
                        "section": "def _process_results("
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    # counter for detected repetitions",
                                "type": "context"
                            },
                            {
                                "content": "    result_repetitions = 0",
                                "type": "context"
                            },
                            {
                                "content": "    # how many repetitions to show, before suppression kicks in",
                                "type": "context"
                            },
                            {
                                "content": "    render_n_repetitions = 10",
                                "type": "deleted"
                            },
                            {
                                "content": "    render_n_repetitions = 10 if sys.stdout.isatty() else float(\"inf\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    for res in results:",
                                "type": "context"
                            },
                            {
                                "content": "        if not res or 'action' not in res:",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "_process_results",
                            "content": "def _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 528,
                            "end_lineno": 641,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 598,
                            "length": 11
                        },
                        "new_range": {
                            "start": 599,
                            "length": 15
                        },
                        "section": "def _process_results("
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                result_repetitions += 1",
                                "type": "context"
                            },
                            {
                                "content": "                if result_repetitions < render_n_repetitions:",
                                "type": "context"
                            },
                            {
                                "content": "                    default_result_renderer(res)",
                                "type": "context"
                            },
                            {
                                "content": "                else:",
                                "type": "added"
                            },
                            {
                                "content": "                    _display_suppressed_message(",
                                "type": "added"
                            },
                            {
                                "content": "                        result_repetitions, render_n_repetitions)",
                                "type": "added"
                            },
                            {
                                "content": "            else:",
                                "type": "context"
                            },
                            {
                                "content": "                # this one is new, first report on any prev. suppressed results",
                                "type": "context"
                            },
                            {
                                "content": "                # by number, and then render this fresh one",
                                "type": "context"
                            },
                            {
                                "content": "                _display_suppressed_message(",
                                "type": "context"
                            },
                            {
                                "content": "                    result_repetitions, render_n_repetitions)",
                                "type": "deleted"
                            },
                            {
                                "content": "                    result_repetitions, render_n_repetitions,",
                                "type": "added"
                            },
                            {
                                "content": "                    final=True)",
                                "type": "added"
                            },
                            {
                                "content": "                default_result_renderer(res)",
                                "type": "context"
                            },
                            {
                                "content": "                result_repetitions = 0",
                                "type": "context"
                            },
                            {
                                "content": "            last_result = trimmed_result",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "_process_results",
                            "content": "def _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 528,
                            "end_lineno": 641,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 638,
                            "length": 7
                        },
                        "new_range": {
                            "start": 643,
                            "length": 7
                        },
                        "section": "def _process_results("
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        yield res",
                                "type": "context"
                            },
                            {
                                "content": "    # make sure to report on any issues that we had suppressed",
                                "type": "context"
                            },
                            {
                                "content": "    _display_suppressed_message(",
                                "type": "context"
                            },
                            {
                                "content": "        result_repetitions, render_n_repetitions)",
                                "type": "deleted"
                            },
                            {
                                "content": "        result_repetitions, render_n_repetitions, final=True)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def keep_result(res, rfilter, **kwargs):",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "_process_results",
                            "content": "def _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 528,
                            "end_lineno": 641,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "d182b92981c7519e011959ecb09d12a0373605b7^",
    "new_commit_hash": "d182b92981c7519e011959ecb09d12a0373605b7",
    "commit_message": "Merge pull request #4422 from kyleam/suppression-feedback\n\ninterface: Provide more timely feedback when suppressing results",
    "commit_date": "2020-04-18T09:53:51+02:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "d182b92981c7519e011959ecb09d12a0373605b7",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom datalad.interface.base import Interface\nfrom datalad.interface.utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.12.5.dev628\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 74ms\nInstalled 25 packages in 4.82s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 63 packages in 5.19s\nPrepared 1 package in 5.28s\nInstalled 57 packages in 516ms\n + annexremote==1.6.6\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.12.5 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.1.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 97 packages in 379ms\nInstalled 32 packages in 746ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7\nplugins: cov-6.0.0\ncollected 6 items\n\nr2e_tests/test_1.py FF..E.                                               [100%]\n\n==================================== ERRORS ====================================\n___________________ ERROR at setup of test_discover_ds_trace ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7/r2e_tests/test_1.py, line 329\n  @with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n  @with_tempfile(mkdir=True)\n  def test_discover_ds_trace(path, otherdir):\nE       fixture 'otherdir' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7/r2e_tests/test_1.py:329\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex30z30wv8'}\nfilename = '/tmp/datalad_temp_test_dirtyp50kkvrg'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:691: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexx_rfxvl3'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchyzkghz_g4'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:518: TypeError\n==================================== PASSES ====================================\n_______________________ test_eval_results_plus_build_doc _______________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.utils:utils.py:2484 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\nDEBUG    datalad.utils:utils.py:2484 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_eval_results_plus_build_doc\nPASSED r2e_tests/test_1.py::test_result_filter\nPASSED r2e_tests/test_1.py::test_utils_suppress_similar\nERROR r2e_tests/test_1.py::test_discover_ds_trace\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\n===================== 2 failed, 3 passed, 1 error in 2.03s =====================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7\nplugins: cov-6.0.0\ncollected 6 items\n\nr2e_tests/test_1.py FF..EF                                               [100%]\n\n==================================== ERRORS ====================================\n___________________ ERROR at setup of test_discover_ds_trace ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7/r2e_tests/test_1.py, line 329\n  @with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n  @with_tempfile(mkdir=True)\n  def test_discover_ds_trace(path, otherdir):\nE       fixture 'otherdir' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_d182b92981c7519e011959ecb09d12a0373605b7/r2e_tests/test_1.py:329\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexwza588ji'}\nfilename = '/tmp/datalad_temp_test_dirtyvf3hp9t0'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:691: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexhydb3k82'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchyqwxvh2sj'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:518: TypeError\n_________________________ test_utils_suppress_similar __________________________\n\n    def test_utils_suppress_similar():\n        tu = TestUtils()\n    \n        # Check suppression boundary for straight chain of similar\n        # messages.\n    \n        def n_foo(number):\n            for i in range(number):\n                yield dict(action=\"foo\",\n                           status=\"ok\",\n                           path=\"path{}\".format(i))\n    \n        with _swallow_outputs() as cmo:\n            cmo.isatty = lambda: True\n            list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n            assert_in(\"path8\", cmo.out)\n            assert_not_in(\"suppressed\", cmo.out)\n    \n        with _swallow_outputs() as cmo:\n            list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n            assert_in(\"path9\", cmo.out)\n            assert_not_in(\"suppressed\", cmo.out)\n    \n        with _swallow_outputs() as cmo:\n            list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n            assert_not_in(\"path10\", cmo.out)\n            assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n    \n        with _swallow_outputs() as cmo:\n            list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n            assert_not_in(\"path10\", cmo.out)\n            # We see an update for each result.\n>           assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n\nr2e_tests/test_1.py:424: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nregex = '1 .* suppressed'\nc = ['foo(ok): path0\\nfoo(ok): path1\\nfoo(ok): path2\\nfoo(ok): path3\\nfoo(ok): path4\\nfoo(ok): path5\\nfoo(ok): path6\\nfoo(...path7\\nfoo(ok): path8\\nfoo(ok): path9\\n  [3 similar messages have been suppressed]\\naction summary:\\n  foo (ok: 13)\\n']\nflags = 0, match = False, msg = None\n\n    def assert_re_in(regex, c, flags=0, match=True, msg=None):\n        \"\"\"Assert that container (list, str, etc) contains entry matching the regex\n        \"\"\"\n        if not isinstance(c, (list, tuple)):\n            c = [c]\n        for e in c:\n            if (re.match if match else re.search)(regex, e, flags=flags):\n                return\n>       raise AssertionError(\n            msg or \"Not a single entry matched %r in %r\" % (regex, c)\n        )\nE       AssertionError: Not a single entry matched '1 .* suppressed' in ['foo(ok): path0\\nfoo(ok): path1\\nfoo(ok): path2\\nfoo(ok): path3\\nfoo(ok): path4\\nfoo(ok): path5\\nfoo(ok): path6\\nfoo(ok): path7\\nfoo(ok): path8\\nfoo(ok): path9\\n  [3 similar messages have been suppressed]\\naction summary:\\n  foo (ok: 13)\\n']\n\ndatalad/tests/utils.py:1183: AssertionError\n==================================== PASSES ====================================\n_______________________ test_eval_results_plus_build_doc _______________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.utils:utils.py:2484 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\nDEBUG    datalad.utils:utils.py:2484 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_eval_results_plus_build_doc\nPASSED r2e_tests/test_1.py::test_result_filter\nERROR r2e_tests/test_1.py::test_discover_ds_trace\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\nFAILED r2e_tests/test_1.py::test_utils_suppress_similar - AssertionError: Not...\n===================== 3 failed, 2 passed, 1 error in 1.13s =====================\n",
    "old_commit_res_stderr": ""
}
datalad/interface/tests/test_utils.py datalad/interface/utils.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Suppression Feedback Not Displayed When Handling Multiple Similar Results

**Description:**
When processing a large number of similar results, the interface is supposed to suppress redundant messages and provide a summary indicating how many messages were suppressed. Currently, the suppression feedback is either missing or not timely, resulting in all messages being displayed without any summary. This cluttered output makes it difficult to identify the actual number of unique messages and understand the extent of suppression.

**Example Code:**
```python
tu = TestUtils()

def generate_similar_results(count):
    for i in range(count):
        yield dict(action='process', status='ok', path=f'item{i}')

# Attempt to process 11 similar results
list(tu(11, result_fn=generate_similar_results, result_renderer='default'))
```

**Expected Behavior:**
After processing a set number of similar messages (e.g., 10), additional similar messages should be suppressed. A summary message such as "[1 similar message has been suppressed]" should be displayed to indicate that redundant messages are not being shown individually.

**Actual Behavior:**
All 11 messages are displayed individually without any suppression summary. This leads to a cluttered output where it's hard to distinguish between unique messages and suppressed duplicates.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/utils.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface utility functions

"""

__docformat__ = 'restructuredtext'

import inspect
import logging
import wrapt
import sys
import re
from os import curdir
from os import pardir
from os import listdir
import os.path as op
from os.path import join as opj
from os.path import isdir
from os.path import relpath
from os.path import sep
from os.path import split as psplit
from itertools import chain

import json

# avoid import from API to not get into circular imports
from datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us
from datalad.utils import (
    path_startswith,
    path_is_subpath,
    assure_unicode,
    getargspec,
    get_wrapped_class,
)
from datalad.support.gitrepo import GitRepo
from datalad.support.exceptions import IncompleteResultsError
from datalad import cfg as dlcfg
from datalad.dochelpers import (
    exc_str,
    single_or_plural,
)

from datalad.ui import ui
import datalad.support.ansi_colors as ac

from datalad.interface.base import default_logchannels
from datalad.interface.base import get_allargs_as_kwargs
from datalad.interface.common_opts import eval_params
from datalad.interface.common_opts import eval_defaults
from .results import known_result_xfms
from datalad.config import ConfigManager
from datalad.core.local.resulthooks import (
    get_jsonhooks_from_config,
    match_jsonhook2result,
    run_jsonhook,
)


lgr = logging.getLogger('datalad.interface.utils')


def cls2cmdlinename(cls):
    "Return the cmdline command name from an Interface class"
    r = re.compile(r'([a-z0-9])([A-Z])')
    return r.sub('\\1-\\2', cls.__name__).lower()


# TODO remove
# only `drop` and `uninstall` are still using this
def handle_dirty_dataset(ds, mode, msg=None):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    ds : Dataset or None
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    msg : str or None
      Custom message to use for a potential commit.

    Returns
    -------
    None
    """
    if ds is None:
        # nothing to be handled
        return
    if msg is None:
        msg = '[DATALAD] auto-saved changes'

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    if ds.repo:
        ds.repo.precommit()

    if mode == 'ignore':
        return
    elif mode == 'fail':
        if not ds.repo or ds.repo.dirty:
            raise RuntimeError('dataset {} has unsaved changes'.format(ds))
    elif mode == 'save-before':
        if not ds.is_installed():
            raise RuntimeError('dataset {} is not yet installed'.format(ds))
        from datalad.core.local.save import Save
        Save.__call__(dataset=ds, message=msg, updated=True)
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def get_tree_roots(paths):
    """Return common root paths for a set of paths

    This function determines the smallest set of common root
    paths and sorts all given paths under the respective
    root.

    Returns
    -------
    dict
      paths by root
    """
    paths_ws = [_with_sep(p) for p in paths]
    # sort all paths under their potential roots
    roots = {}
    # start from the top to get all paths down the line
    # and collate them into as few roots as possible
    for s in sorted(paths_ws):
        if any([s.startswith(r) for r in roots]):
            # this path is already covered by a known root
            continue
        # find all sub paths
        subs = [p for p in paths if p.startswith(s)]
        roots[s.rstrip(sep)] = subs
    return roots


# TODO remove
# only `remove` and `uninstall` use this, the uses path `path_is_subpath`
def path_is_under(values, path=None):
    """Whether a given path is a subdirectory of any of the given test values

    Parameters
    ----------
    values : sequence or dict
      Paths to be tested against. This can be a dictionary in which case
      all values from all keys will be tested against.
    path : path or None
      Test path. If None is given, the process' working directory is
      used.

    Returns
    -------
    bool
    """
    if path is None:
        from datalad.utils import getpwd
        path = getpwd()
    if isinstance(values, dict):
        values = chain(*values.values())
    path_drive, _ = op.splitdrive(path)
    for p in values:
        p_drive, _ = op.splitdrive(p)
        # need to protect against unsupported use of relpath() with
        # abspaths on windows from different drives (gh-3724)
        if path_drive != p_drive:
        # different drives, enough evidence for "not under"
            continue
        rpath = relpath(p, start=path)
        if rpath == curdir \
                or rpath == pardir \
                or set(psplit(rpath)) == {pardir}:
            # first match is enough
            return True
    return False


def discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,
                                      spec, includeds=None):
    """Discover the edges and nodes in a dataset tree to given target paths

    Parameters
    ----------
    basepath : path
      Path to a start or top-level dataset. Really has to be a path to a
      dataset!
    targetpaths : list(path)
      Any non-zero number of paths that are termination points for the
      search algorithm. Can be paths to datasets, directories, or files
      (and any combination thereof).
    current_trace : list
      For a top-level call this should probably always be `[]`
    spec : dict
      `content_by_ds`-style dictionary that will receive information about the
      discovered datasets. Specifically, for each discovered dataset there
      will be in item with its path under the key (path) of the respective
      superdataset.
    includeds : sequence, optional
      Any paths given are treated as existing subdatasets, regardless of
      whether they can be found in the filesystem. Such subdatasets will appear
      under the key of the closest existing dataset in the `spec`.

    Returns
    -------
    None
      Function calls itself recursively and populates `spec` dict in-place.
      Keys are dataset paths, values are sets of subdataset paths
    """
    # convert to set for faster lookup
    includeds = includeds if isinstance(includeds, set) else \
        set() if includeds is None else set(includeds)
    # this beast walks the directory tree from a given `basepath` until
    # it discovers any of the given `targetpaths`
    # if it finds one, it commits any accummulated trace of visited
    # datasets on this edge to the spec
    valid_repo = GitRepo.is_valid_repo(basepath)
    if valid_repo:
        # we are passing into a new dataset, extend the dataset trace
        current_trace = current_trace + [basepath]
    # this edge is not done, we need to try to reach any downstream
    # dataset
    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)
    # whether anything in this directory matched a targetpath
    filematch = False
    if isdir(basepath):
        for p in listdir(basepath):
            p = assure_unicode(opj(basepath, p))
            if not isdir(p):
                if p in targetpaths:
                    filematch = True
                # we cannot have anything below this one
                continue
            # OPT listdir might be large and we could have only few items
            # in `targetpaths` -- so traverse only those in spec which have
            # leading dir basepath
            # filter targets matching this downward path
            downward_targets = set(
                t for t in targetpaths if path_startswith(t, p))
            if not downward_targets:
                continue
            # remove the matching ones from the "todo" list
            undiscovered_ds.difference_update(downward_targets)
            # go one deeper
            discover_dataset_trace_to_targets(
                p, downward_targets, current_trace, spec,
                includeds=includeds if not includeds else includeds.intersection(
                    downward_targets))
    undiscovered_ds = [t for t in undiscovered_ds
                       if includeds and
                          path_is_subpath(t, current_trace[-1]) and
                          t in includeds]
    if filematch or basepath in targetpaths or undiscovered_ds:
        for i, p in enumerate(current_trace[:-1]):
            # TODO RF prepare proper annotated path dicts
            subds = spec.get(p, set())
            subds.add(current_trace[i + 1])
            spec[p] = subds
        if undiscovered_ds:
            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(
                undiscovered_ds)


def get_result_filter(fx):
    """Wrap a filter into a helper to be able to accept additional
    arguments, if the filter doesn't support it already"""
    _fx = fx
    if fx and not getargspec(fx).keywords:
        def _fx(res, **kwargs):
            return fx(res)
    return _fx


def eval_results(func):
    """Decorator for return value evaluation of datalad commands.

    Note, this decorator is only compatible with commands that return
    status dict sequences!

    Two basic modes of operation are supported: 1) "generator mode" that
    `yields` individual results, and 2) "list mode" that returns a sequence of
    results. The behavior can be selected via the kwarg `return_type`.
    Default is "list mode".

    This decorator implements common functionality for result rendering/output,
    error detection/handling, and logging.

    Result rendering/output can be triggered via the
    `datalad.api.result-renderer` configuration variable, or the
    `result_renderer` keyword argument of each decorated command. Supported
    modes are: 'default' (one line per result with action, status, path,
    and an optional message); 'json' (one object per result, like git-annex),
    'json_pp' (like 'json', but pretty-printed spanning multiple lines),
    'tailored' custom output formatting provided by each command
    class (if any).

    Error detection works by inspecting the `status` item of all result
    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'
    will cause an IncompleteResultsError exception to be raised that carries
    the failed actions' status dictionaries in its `failed` attribute.

    Status messages will be logged automatically, by default the following
    association of result status and log channel will be used: 'ok' (debug),
    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger
    instances included in the results are used to capture the origin of a
    status report.

    Parameters
    ----------
    func: function
      __call__ method of a subclass of Interface,
      i.e. a datalad command definition
    """

    @wrapt.decorator
    def eval_func(wrapped, instance, args, kwargs):
        lgr.log(2, "Entered eval_func for %s", func)
        # for result filters
        # we need to produce a dict with argname/argvalue pairs for all args
        # incl. defaults and args given as positionals
        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)

        # determine the command class associated with `wrapped`
        wrapped_class = get_wrapped_class(wrapped)

        # retrieve common options from kwargs, and fall back on the command
        # class attributes, or general defaults if needed
        kwargs = kwargs.copy()  # we will pop, which might cause side-effect
        common_params = {
            p_name: kwargs.pop(
                # go with any explicitly given default
                p_name,
                # otherwise determine the command class and pull any
                # default set in that class
                getattr(
                    wrapped_class,
                    p_name,
                    # or the common default
                    eval_defaults[p_name]))
            for p_name in eval_params}

        # short cuts and configured setup for common options
        return_type = common_params['return_type']
        result_filter = get_result_filter(common_params['result_filter'])
        # resolve string labels for transformers too
        result_xfm = known_result_xfms.get(
            common_params['result_xfm'],
            # use verbatim, if not a known label
            common_params['result_xfm'])
        result_renderer = common_params['result_renderer']
        # TODO remove this conditional branch entirely, done outside
        if not result_renderer:
            result_renderer = dlcfg.get('datalad.api.result-renderer', None)
        # look for potential override of logging behavior
        result_log_level = dlcfg.get('datalad.log.result-level', None)

        # query cfg for defaults
        # .is_installed and .config can be costly, so ensure we do
        # it only once. See https://github.com/datalad/datalad/issues/3575
        dataset_arg = allkwargs.get('dataset', None)
        from datalad.distribution.dataset import Dataset
        ds = dataset_arg if isinstance(dataset_arg, Dataset) \
            else Dataset(dataset_arg) if dataset_arg else None
        # do not reuse a dataset's existing config manager here
        # they are configured to read the committed dataset configuration
        # too. That means a datalad update can silently bring in new
        # procedure definitions from the outside, and in some sense enable
        # remote code execution by a 3rd-party
        # To avoid that, create a new config manager that only reads local
        # config (system and .git/config), plus any overrides given to this
        # datalad session
        proc_cfg = ConfigManager(
            ds, source='local', overrides=dlcfg.overrides
        ) if ds and ds.is_installed() else dlcfg

        # look for hooks
        hooks = get_jsonhooks_from_config(proc_cfg)

        # this internal helper function actually drives the command
        # generator-style, it may generate an exception if desired,
        # on incomplete results
        def generator_func(*_args, **_kwargs):
            # flag whether to raise an exception
            incomplete_results = []
            # track what actions were performed how many times
            action_summary = {}

            # if a custom summary is to be provided, collect the results
            # of the command execution
            results = []
            do_custom_result_summary = result_renderer in ('tailored', 'default') \
                and hasattr(wrapped_class, 'custom_result_summary_renderer')

            # process main results
            for r in _process_results(
                    # execution
                    wrapped(*_args, **_kwargs),
                    wrapped_class,
                    common_params['on_failure'],
                    # bookkeeping
                    action_summary,
                    incomplete_results,
                    # communication
                    result_renderer,
                    result_log_level,
                    # let renderers get to see how a command was called
                    allkwargs):
                for hook, spec in hooks.items():
                    # run the hooks before we yield the result
                    # this ensures that they are executed before
                    # a potentially wrapper command gets to act
                    # on them
                    if match_jsonhook2result(hook, r, spec['match']):
                        lgr.debug('Result %s matches hook %s', r, hook)
                        # a hook is also a command that yields results
                        # so yield them outside too
                        # users need to pay attention to void infinite
                        # loops, i.e. when a hook yields a result that
                        # triggers that same hook again
                        for hr in run_jsonhook(hook, spec, r, dataset_arg):
                            # apply same logic as for main results, otherwise
                            # any filters would only tackle the primary results
                            # and a mixture of return values could happen
                            if not keep_result(hr, result_filter, **allkwargs):
                                continue
                            hr = xfm_result(hr, result_xfm)
                            # rationale for conditional is a few lines down
                            if hr:
                                yield hr
                if not keep_result(r, result_filter, **allkwargs):
                    continue
                r = xfm_result(r, result_xfm)
                # in case the result_xfm decided to not give us anything
                # exclude it from the results. There is no particular reason
                # to do so other than that it was established behavior when
                # this comment was written. This will not affect any real
                # result record
                if r:
                    yield r

                # collect if summary is desired
                if do_custom_result_summary:
                    results.append(r)

            # result summary before a potential exception
            # custom first
            if do_custom_result_summary:
                wrapped_class.custom_result_summary_renderer(results)
            elif result_renderer == 'default' and action_summary and \
                    sum(sum(s.values()) for s in action_summary.values()) > 1:
                # give a summary in default mode, when there was more than one
                # action performed
                ui.message("action summary:\n  {}".format(
                    '\n  '.join('{} ({})'.format(
                        act,
                        ', '.join('{}: {}'.format(status, action_summary[act][status])
                                  for status in sorted(action_summary[act])))
                                for act in sorted(action_summary))))

            if incomplete_results:
                raise IncompleteResultsError(
                    failed=incomplete_results,
                    msg="Command did not complete successfully")

        if return_type == 'generator':
            # hand over the generator
            lgr.log(2, "Returning generator_func from eval_func for %s", wrapped_class)
            return generator_func(*args, **kwargs)
        else:
            @wrapt.decorator
            def return_func(wrapped_, instance_, args_, kwargs_):
                results = wrapped_(*args_, **kwargs_)
                if inspect.isgenerator(results):
                    # unwind generator if there is one, this actually runs
                    # any processing
                    results = list(results)
                # render summaries
                if not result_xfm and result_renderer in ('tailored', 'default'):
                    # cannot render transformed results
                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):
                        wrapped_class.custom_result_summary_renderer(results)
                if return_type == 'item-or-list' and \
                        len(results) < 2:
                    return results[0] if results else None
                else:
                    return results
            lgr.log(2, "Returning return_func from eval_func for %s", wrapped_class)
            return return_func(generator_func)(*args, **kwargs)

    return eval_func(func)


def default_result_renderer(res):
    if res.get('status', None) != 'notneeded':
        path = res['path']
        path = str(path)
        ui.message('{action}({status}): {path}{type}{msg}'.format(
                action=ac.color_word(res['action'], ac.BOLD),
                status=ac.color_status(res['status']),
                path=relpath(path, res['refds']) if res.get('refds') else path,
                type=' ({})'.format(
                        ac.color_word(res['type'], ac.MAGENTA)
                ) if 'type' in res else '',
                msg=' [{}]'.format(
                        res['message'][0] % res['message'][1:]
                        if isinstance(res['message'], tuple) else res[
                            'message'])
                if res.get('message', None) else ''))


def _display_suppressed_message(nsimilar, ndisplayed):
    # +1 because there was the original result + nsimilar displayed.
    n_suppressed = nsimilar - ndisplayed + 1
    if n_suppressed > 0:
        ui.message('  [{} similar {} been suppressed]'
                   .format(n_suppressed,
                           single_or_plural("message has",
                                            "messages have",
                                            n_suppressed, False)))


def _process_results(
        results,
        cmd_class,
        on_failure,
        action_summary,
        incomplete_results,
        result_renderer,
        result_log_level,
        allkwargs):
    # private helper pf @eval_results
    # loop over results generated from some source and handle each
    # of them according to the requested behavior (logging, rendering, ...)

    # used to track repeated messages in the default renderer
    last_result = None
    # which result dict keys to inspect for changes to discover repetions
    # of similar messages
    repetition_keys = set(('action', 'status', 'type', 'refds'))
    # counter for detected repetitions
    result_repetitions = 0
    # how many repetitions to show, before suppression kicks in
    render_n_repetitions = 10

    for res in results:
        if not res or 'action' not in res:
            # XXX Yarik has to no clue on how to track the origin of the
            # record to figure out WTF, so he just skips it
            continue

        actsum = action_summary.get(res['action'], {})
        if res['status']:
            actsum[res['status']] = actsum.get(res['status'], 0) + 1
            action_summary[res['action']] = actsum
        ## log message, if there is one and a logger was given
        msg = res.get('message', None)
        # remove logger instance from results, as it is no longer useful
        # after logging was done, it isn't serializable, and generally
        # pollutes the output
        res_lgr = res.pop('logger', None)
        if msg and res_lgr:
            if isinstance(res_lgr, logging.Logger):
                # didn't get a particular log function, go with default
                res_lgr = getattr(
                    res_lgr,
                    default_logchannels[res['status']]
                    if result_log_level is None
                    else result_log_level)
            msg = res['message']
            msgargs = None
            if isinstance(msg, tuple):
                msgargs = msg[1:]
                msg = msg[0]
            if 'path' in res:
                msg = '{} [{}({})]'.format(
                    msg, res['action'], res['path'])
            if msgargs:
                # support string expansion of logging to avoid runtime cost
                res_lgr(msg, *msgargs)
            else:
                res_lgr(msg)

        ## output rendering
        # TODO RF this in a simple callable that gets passed into this function
        if result_renderer is None or result_renderer == 'disabled':
            pass
        elif result_renderer == 'default':
            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}
            if res.get('status', None) != 'notneeded' \
                    and trimmed_result == last_result:
                # this is a similar report, suppress if too many, but count it
                result_repetitions += 1
                if result_repetitions < render_n_repetitions:
                    default_result_renderer(res)
            else:
                # this one is new, first report on any prev. suppressed results
                # by number, and then render this fresh one
                _display_suppressed_message(
                    result_repetitions, render_n_repetitions)
                default_result_renderer(res)
                result_repetitions = 0
            last_result = trimmed_result
        elif result_renderer in ('json', 'json_pp'):
            ui.message(json.dumps(
                {k: v for k, v in res.items()
                 if k not in ('message', 'logger')},
                sort_keys=True,
                indent=2 if result_renderer.endswith('_pp') else None,
                default=lambda x: str(x)))
        elif result_renderer in ('tailored', 'default'):
            if hasattr(cmd_class, 'custom_result_renderer'):
                cmd_class.custom_result_renderer(res, **allkwargs)
        elif hasattr(result_renderer, '__call__'):
            try:
                result_renderer(res, **allkwargs)
            except Exception as e:
                lgr.warning('Result rendering failed for: %s [%s]',
                            res, exc_str(e))
        else:
            raise ValueError('unknown result renderer "{}"'.format(result_renderer))

        ## error handling
        # looks for error status, and report at the end via
        # an exception
        if on_failure in ('continue', 'stop') \
                and res['status'] in ('impossible', 'error'):
            incomplete_results.append(res)
            if on_failure == 'stop':
                # first fail -> that's it
                # raise will happen after the loop
                break
        yield res
    # make sure to report on any issues that we had suppressed
    _display_suppressed_message(
        result_repetitions, render_n_repetitions)


def keep_result(res, rfilter, **kwargs):
    if not rfilter:
        return True
    try:
        if not rfilter(res, **kwargs):
            # give the slightest indication which filter was employed
            raise ValueError(
                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))
    except ValueError as e:
        # make sure to report the excluded result to massively improve
        # debugging experience
        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)
        return False
    return True


def xfm_result(res, xfm):
    if not xfm:
        return res

    return xfm(res)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Suppression Feedback Not Displayed When Handling Multiple Similar Results

**Description:**
When processing a large number of similar results, the interface is supposed to suppress redundant messages and provide a summary indicating how many messages were suppressed. Currently, the suppression feedback is either missing or not timely, resulting in all messages being displayed without any summary. This cluttered output makes it difficult to identify the actual number of unique messages and understand the extent of suppression.

**Example Code:**
```python
tu = TestUtils()

def generate_similar_results(count):
    for i in range(count):
        yield dict(action='process', status='ok', path=f'item{i}')

# Attempt to process 11 similar results
list(tu(11, result_fn=generate_similar_results, result_renderer='default'))
```

**Expected Behavior:**
After processing a set number of similar messages (e.g., 10), additional similar messages should be suppressed. A summary message such as "[1 similar message has been suppressed]" should be displayed to indicate that redundant messages are not being shown individually.

**Actual Behavior:**
All 11 messages are displayed individually without any suppression summary. This leads to a cluttered output where it's hard to distinguish between unique messages and suppressed duplicates.
[/ISSUE]
{
    "test_eval_results_plus_build_doc": "PASSED",
    "test_result_filter": "PASSED",
    "test_utils_suppress_similar": "PASSED",
    "test_discover_ds_trace": "ERROR",
    "test_dirty": "FAILED",
    "test_save_hierarchy": "FAILED"
}
diff --git a/datalad/interface/utils.py b/datalad/interface/utils.py
index db03e38dc..67d62f588 100644
--- a/datalad/interface/utils.py
+++ b/datalad/interface/utils.py
@@ -514,7 +514,7 @@ def default_result_renderer(res):
                 if res.get('message', None) else ''))
 
 
-def _display_suppressed_message(nsimilar, ndisplayed):
+def _display_suppressed_message(nsimilar, ndisplayed, final=False):
     # +1 because there was the original result + nsimilar displayed.
     n_suppressed = nsimilar - ndisplayed + 1
     if n_suppressed > 0:
@@ -522,7 +522,8 @@ def _display_suppressed_message(nsimilar, ndisplayed):
                    .format(n_suppressed,
                            single_or_plural("message has",
                                             "messages have",
-                                            n_suppressed, False)))
+                                            n_suppressed, False)),
+                   cr="\n" if final else "\r")
 
 
 def _process_results(
@@ -546,7 +547,7 @@ def _process_results(
     # counter for detected repetitions
     result_repetitions = 0
     # how many repetitions to show, before suppression kicks in
-    render_n_repetitions = 10
+    render_n_repetitions = 10 if sys.stdout.isatty() else float("inf")
 
     for res in results:
         if not res or 'action' not in res:
@@ -598,11 +599,15 @@ def _process_results(
                 result_repetitions += 1
                 if result_repetitions < render_n_repetitions:
                     default_result_renderer(res)
+                else:
+                    _display_suppressed_message(
+                        result_repetitions, render_n_repetitions)
             else:
                 # this one is new, first report on any prev. suppressed results
                 # by number, and then render this fresh one
                 _display_suppressed_message(
-                    result_repetitions, render_n_repetitions)
+                    result_repetitions, render_n_repetitions,
+                    final=True)
                 default_result_renderer(res)
                 result_repetitions = 0
             last_result = trimmed_result
@@ -638,7 +643,7 @@ def _process_results(
         yield res
     # make sure to report on any issues that we had suppressed
     _display_suppressed_message(
-        result_repetitions, render_n_repetitions)
+        result_repetitions, render_n_repetitions, final=True)
 
 
 def keep_result(res, rfilter, **kwargs):
 

{"datalad/interface/utils.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport inspect\nimport logging\nimport wrapt\nimport sys\nimport re\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nimport os.path as op\nfrom os.path import join as opj\nfrom os.path import isdir\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\nimport json\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import (\n    path_startswith,\n    path_is_subpath,\n    assure_unicode,\n    getargspec,\n    get_wrapped_class,\n)\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad import cfg as dlcfg\nfrom datalad.dochelpers import (\n    exc_str,\n    single_or_plural,\n)\n\nfrom datalad.ui import ui\nimport datalad.support.ansi_colors as ac\n\nfrom datalad.interface.base import default_logchannels\nfrom datalad.interface.base import get_allargs_as_kwargs\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom .results import known_result_xfms\nfrom datalad.config import ConfigManager\nfrom datalad.core.local.resulthooks import (\n    get_jsonhooks_from_config,\n    match_jsonhook2result,\n    run_jsonhook,\n)\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef cls2cmdlinename(cls):\n    \"Return the cmdline command name from an Interface class\"\n    r = re.compile(r'([a-z0-9])([A-Z])')\n    return r.sub('\\\\1-\\\\2', cls.__name__).lower()\n\n\n# TODO remove\n# only `drop` and `uninstall` are still using this\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.dirty:\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.core.local.save import Save\n        Save.__call__(dataset=ds, message=msg, updated=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\n# TODO remove\n# only `remove` and `uninstall` use this, the uses path `path_is_subpath`\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        path = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    path_drive, _ = op.splitdrive(path)\n    for p in values:\n        p_drive, _ = op.splitdrive(p)\n        # need to protect against unsupported use of relpath() with\n        # abspaths on windows from different drives (gh-3724)\n        if path_drive != p_drive:\n        # different drives, enough evidence for \"not under\"\n            continue\n        rpath = relpath(p, start=path)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,\n                                      spec, includeds=None):\n    \"\"\"Discover the edges and nodes in a dataset tree to given target paths\n\n    Parameters\n    ----------\n    basepath : path\n      Path to a start or top-level dataset. Really has to be a path to a\n      dataset!\n    targetpaths : list(path)\n      Any non-zero number of paths that are termination points for the\n      search algorithm. Can be paths to datasets, directories, or files\n      (and any combination thereof).\n    current_trace : list\n      For a top-level call this should probably always be `[]`\n    spec : dict\n      `content_by_ds`-style dictionary that will receive information about the\n      discovered datasets. Specifically, for each discovered dataset there\n      will be in item with its path under the key (path) of the respective\n      superdataset.\n    includeds : sequence, optional\n      Any paths given are treated as existing subdatasets, regardless of\n      whether they can be found in the filesystem. Such subdatasets will appear\n      under the key of the closest existing dataset in the `spec`.\n\n    Returns\n    -------\n    None\n      Function calls itself recursively and populates `spec` dict in-place.\n      Keys are dataset paths, values are sets of subdataset paths\n    \"\"\"\n    # convert to set for faster lookup\n    includeds = includeds if isinstance(includeds, set) else \\\n        set() if includeds is None else set(includeds)\n    # this beast walks the directory tree from a given `basepath` until\n    # it discovers any of the given `targetpaths`\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(basepath)\n    if valid_repo:\n        # we are passing into a new dataset, extend the dataset trace\n        current_trace = current_trace + [basepath]\n    # this edge is not done, we need to try to reach any downstream\n    # dataset\n    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)\n    # whether anything in this directory matched a targetpath\n    filematch = False\n    if isdir(basepath):\n        for p in listdir(basepath):\n            p = assure_unicode(opj(basepath, p))\n            if not isdir(p):\n                if p in targetpaths:\n                    filematch = True\n                # we cannot have anything below this one\n                continue\n            # OPT listdir might be large and we could have only few items\n            # in `targetpaths` -- so traverse only those in spec which have\n            # leading dir basepath\n            # filter targets matching this downward path\n            downward_targets = set(\n                t for t in targetpaths if path_startswith(t, p))\n            if not downward_targets:\n                continue\n            # remove the matching ones from the \"todo\" list\n            undiscovered_ds.difference_update(downward_targets)\n            # go one deeper\n            discover_dataset_trace_to_targets(\n                p, downward_targets, current_trace, spec,\n                includeds=includeds if not includeds else includeds.intersection(\n                    downward_targets))\n    undiscovered_ds = [t for t in undiscovered_ds\n                       if includeds and\n                          path_is_subpath(t, current_trace[-1]) and\n                          t in includeds]\n    if filematch or basepath in targetpaths or undiscovered_ds:\n        for i, p in enumerate(current_trace[:-1]):\n            # TODO RF prepare proper annotated path dicts\n            subds = spec.get(p, set())\n            subds.add(current_trace[i + 1])\n            spec[p] = subds\n        if undiscovered_ds:\n            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(\n                undiscovered_ds)\n\n\ndef get_result_filter(fx):\n    \"\"\"Wrap a filter into a helper to be able to accept additional\n    arguments, if the filter doesn't support it already\"\"\"\n    _fx = fx\n    if fx and not getargspec(fx).keywords:\n        def _fx(res, **kwargs):\n            return fx(res)\n    return _fx\n\n\ndef eval_results(func):\n    \"\"\"Decorator for return value evaluation of datalad commands.\n\n    Note, this decorator is only compatible with commands that return\n    status dict sequences!\n\n    Two basic modes of operation are supported: 1) \"generator mode\" that\n    `yields` individual results, and 2) \"list mode\" that returns a sequence of\n    results. The behavior can be selected via the kwarg `return_type`.\n    Default is \"list mode\".\n\n    This decorator implements common functionality for result rendering/output,\n    error detection/handling, and logging.\n\n    Result rendering/output can be triggered via the\n    `datalad.api.result-renderer` configuration variable, or the\n    `result_renderer` keyword argument of each decorated command. Supported\n    modes are: 'default' (one line per result with action, status, path,\n    and an optional message); 'json' (one object per result, like git-annex),\n    'json_pp' (like 'json', but pretty-printed spanning multiple lines),\n    'tailored' custom output formatting provided by each command\n    class (if any).\n\n    Error detection works by inspecting the `status` item of all result\n    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'\n    will cause an IncompleteResultsError exception to be raised that carries\n    the failed actions' status dictionaries in its `failed` attribute.\n\n    Status messages will be logged automatically, by default the following\n    association of result status and log channel will be used: 'ok' (debug),\n    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger\n    instances included in the results are used to capture the origin of a\n    status report.\n\n    Parameters\n    ----------\n    func: function\n      __call__ method of a subclass of Interface,\n      i.e. a datalad command definition\n    \"\"\"\n\n    @wrapt.decorator\n    def eval_func(wrapped, instance, args, kwargs):\n        lgr.log(2, \"Entered eval_func for %s\", func)\n        # for result filters\n        # we need to produce a dict with argname/argvalue pairs for all args\n        # incl. defaults and args given as positionals\n        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)\n\n        # determine the command class associated with `wrapped`\n        wrapped_class = get_wrapped_class(wrapped)\n\n        # retrieve common options from kwargs, and fall back on the command\n        # class attributes, or general defaults if needed\n        kwargs = kwargs.copy()  # we will pop, which might cause side-effect\n        common_params = {\n            p_name: kwargs.pop(\n                # go with any explicitly given default\n                p_name,\n                # otherwise determine the command class and pull any\n                # default set in that class\n                getattr(\n                    wrapped_class,\n                    p_name,\n                    # or the common default\n                    eval_defaults[p_name]))\n            for p_name in eval_params}\n\n        # short cuts and configured setup for common options\n        return_type = common_params['return_type']\n        result_filter = get_result_filter(common_params['result_filter'])\n        # resolve string labels for transformers too\n        result_xfm = known_result_xfms.get(\n            common_params['result_xfm'],\n            # use verbatim, if not a known label\n            common_params['result_xfm'])\n        result_renderer = common_params['result_renderer']\n        # TODO remove this conditional branch entirely, done outside\n        if not result_renderer:\n            result_renderer = dlcfg.get('datalad.api.result-renderer', None)\n        # look for potential override of logging behavior\n        result_log_level = dlcfg.get('datalad.log.result-level', None)\n\n        # query cfg for defaults\n        # .is_installed and .config can be costly, so ensure we do\n        # it only once. See https://github.com/datalad/datalad/issues/3575\n        dataset_arg = allkwargs.get('dataset', None)\n        from datalad.distribution.dataset import Dataset\n        ds = dataset_arg if isinstance(dataset_arg, Dataset) \\\n            else Dataset(dataset_arg) if dataset_arg else None\n        # do not reuse a dataset's existing config manager here\n        # they are configured to read the committed dataset configuration\n        # too. That means a datalad update can silently bring in new\n        # procedure definitions from the outside, and in some sense enable\n        # remote code execution by a 3rd-party\n        # To avoid that, create a new config manager that only reads local\n        # config (system and .git/config), plus any overrides given to this\n        # datalad session\n        proc_cfg = ConfigManager(\n            ds, source='local', overrides=dlcfg.overrides\n        ) if ds and ds.is_installed() else dlcfg\n\n        # look for hooks\n        hooks = get_jsonhooks_from_config(proc_cfg)\n\n        # this internal helper function actually drives the command\n        # generator-style, it may generate an exception if desired,\n        # on incomplete results\n        def generator_func(*_args, **_kwargs):\n            # flag whether to raise an exception\n            incomplete_results = []\n            # track what actions were performed how many times\n            action_summary = {}\n\n            # if a custom summary is to be provided, collect the results\n            # of the command execution\n            results = []\n            do_custom_result_summary = result_renderer in ('tailored', 'default') \\\n                and hasattr(wrapped_class, 'custom_result_summary_renderer')\n\n            # process main results\n            for r in _process_results(\n                    # execution\n                    wrapped(*_args, **_kwargs),\n                    wrapped_class,\n                    common_params['on_failure'],\n                    # bookkeeping\n                    action_summary,\n                    incomplete_results,\n                    # communication\n                    result_renderer,\n                    result_log_level,\n                    # let renderers get to see how a command was called\n                    allkwargs):\n                for hook, spec in hooks.items():\n                    # run the hooks before we yield the result\n                    # this ensures that they are executed before\n                    # a potentially wrapper command gets to act\n                    # on them\n                    if match_jsonhook2result(hook, r, spec['match']):\n                        lgr.debug('Result %s matches hook %s', r, hook)\n                        # a hook is also a command that yields results\n                        # so yield them outside too\n                        # users need to pay attention to void infinite\n                        # loops, i.e. when a hook yields a result that\n                        # triggers that same hook again\n                        for hr in run_jsonhook(hook, spec, r, dataset_arg):\n                            # apply same logic as for main results, otherwise\n                            # any filters would only tackle the primary results\n                            # and a mixture of return values could happen\n                            if not keep_result(hr, result_filter, **allkwargs):\n                                continue\n                            hr = xfm_result(hr, result_xfm)\n                            # rationale for conditional is a few lines down\n                            if hr:\n                                yield hr\n                if not keep_result(r, result_filter, **allkwargs):\n                    continue\n                r = xfm_result(r, result_xfm)\n                # in case the result_xfm decided to not give us anything\n                # exclude it from the results. There is no particular reason\n                # to do so other than that it was established behavior when\n                # this comment was written. This will not affect any real\n                # result record\n                if r:\n                    yield r\n\n                # collect if summary is desired\n                if do_custom_result_summary:\n                    results.append(r)\n\n            # result summary before a potential exception\n            # custom first\n            if do_custom_result_summary:\n                wrapped_class.custom_result_summary_renderer(results)\n            elif result_renderer == 'default' and action_summary and \\\n                    sum(sum(s.values()) for s in action_summary.values()) > 1:\n                # give a summary in default mode, when there was more than one\n                # action performed\n                ui.message(\"action summary:\\n  {}\".format(\n                    '\\n  '.join('{} ({})'.format(\n                        act,\n                        ', '.join('{}: {}'.format(status, action_summary[act][status])\n                                  for status in sorted(action_summary[act])))\n                                for act in sorted(action_summary))))\n\n            if incomplete_results:\n                raise IncompleteResultsError(\n                    failed=incomplete_results,\n                    msg=\"Command did not complete successfully\")\n\n        if return_type == 'generator':\n            # hand over the generator\n            lgr.log(2, \"Returning generator_func from eval_func for %s\", wrapped_class)\n            return generator_func(*args, **kwargs)\n        else:\n            @wrapt.decorator\n            def return_func(wrapped_, instance_, args_, kwargs_):\n                results = wrapped_(*args_, **kwargs_)\n                if inspect.isgenerator(results):\n                    # unwind generator if there is one, this actually runs\n                    # any processing\n                    results = list(results)\n                # render summaries\n                if not result_xfm and result_renderer in ('tailored', 'default'):\n                    # cannot render transformed results\n                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):\n                        wrapped_class.custom_result_summary_renderer(results)\n                if return_type == 'item-or-list' and \\\n                        len(results) < 2:\n                    return results[0] if results else None\n                else:\n                    return results\n            lgr.log(2, \"Returning return_func from eval_func for %s\", wrapped_class)\n            return return_func(generator_func)(*args, **kwargs)\n\n    return eval_func(func)\n\n\ndef default_result_renderer(res):\n    if res.get('status', None) != 'notneeded':\n        path = res['path']\n        path = str(path)\n        ui.message('{action}({status}): {path}{type}{msg}'.format(\n                action=ac.color_word(res['action'], ac.BOLD),\n                status=ac.color_status(res['status']),\n                path=relpath(path, res['refds']) if res.get('refds') else path,\n                type=' ({})'.format(\n                        ac.color_word(res['type'], ac.MAGENTA)\n                ) if 'type' in res else '',\n                msg=' [{}]'.format(\n                        res['message'][0] % res['message'][1:]\n                        if isinstance(res['message'], tuple) else res[\n                            'message'])\n                if res.get('message', None) else ''))\n\n\ndef _display_suppressed_message(nsimilar, ndisplayed):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)))\n\n\ndef _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions)\n\n\ndef keep_result(res, rfilter, **kwargs):\n    if not rfilter:\n        return True\n    try:\n        if not rfilter(res, **kwargs):\n            # give the slightest indication which filter was employed\n            raise ValueError(\n                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))\n    except ValueError as e:\n        # make sure to report the excluded result to massively improve\n        # debugging experience\n        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)\n        return False\n    return True\n\n\ndef xfm_result(res, xfm):\n    if not xfm:\n        return res\n\n    return xfm(res)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Suppression Feedback Not Displayed When Handling Multiple Similar Results

**Description:**
When processing a large number of similar results, the interface is supposed to suppress redundant messages and provide a summary indicating how many messages were suppressed. Currently, the suppression feedback is either missing or not timely, resulting in all messages being displayed without any summary. This cluttered output makes it difficult to identify the actual number of unique messages and understand the extent of suppression.

**Example Code:**
```python
tu = TestUtils()

def generate_similar_results(count):
    for i in range(count):
        yield dict(action='process', status='ok', path=f'item{i}')

# Attempt to process 11 similar results
list(tu(11, result_fn=generate_similar_results, result_renderer='default'))
```

**Expected Behavior:**
After processing a set number of similar messages (e.g., 10), additional similar messages should be suppressed. A summary message such as "[1 similar message has been suppressed]" should be displayed to indicate that redundant messages are not being shown individually.

**Actual Behavior:**
All 11 messages are displayed individually without any suppression summary. This leads to a cluttered output where it's hard to distinguish between unique messages and suppressed duplicates.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/utils.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface utility functions

"""

__docformat__ = 'restructuredtext'

import inspect
import logging
import wrapt
import sys
import re
from os import curdir
from os import pardir
from os import listdir
import os.path as op
from os.path import join as opj
from os.path import isdir
from os.path import relpath
from os.path import sep
from os.path import split as psplit
from itertools import chain

import json

# avoid import from API to not get into circular imports
from datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us
from datalad.utils import (
    path_startswith,
    path_is_subpath,
    assure_unicode,
    getargspec,
    get_wrapped_class,
)
from datalad.support.gitrepo import GitRepo
from datalad.support.exceptions import IncompleteResultsError
from datalad import cfg as dlcfg
from datalad.dochelpers import (
    exc_str,
    single_or_plural,
)

from datalad.ui import ui
import datalad.support.ansi_colors as ac

from datalad.interface.base import default_logchannels
from datalad.interface.base import get_allargs_as_kwargs
from datalad.interface.common_opts import eval_params
from datalad.interface.common_opts import eval_defaults
from .results import known_result_xfms
from datalad.config import ConfigManager
from datalad.core.local.resulthooks import (
    get_jsonhooks_from_config,
    match_jsonhook2result,
    run_jsonhook,
)


lgr = logging.getLogger('datalad.interface.utils')


def cls2cmdlinename(cls):
    "Return the cmdline command name from an Interface class"
    r = re.compile(r'([a-z0-9])([A-Z])')
    return r.sub('\\1-\\2', cls.__name__).lower()


# TODO remove
# only `drop` and `uninstall` are still using this
def handle_dirty_dataset(ds, mode, msg=None):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    ds : Dataset or None
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    msg : str or None
      Custom message to use for a potential commit.

    Returns
    -------
    None
    """
    if ds is None:
        # nothing to be handled
        return
    if msg is None:
        msg = '[DATALAD] auto-saved changes'

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    if ds.repo:
        ds.repo.precommit()

    if mode == 'ignore':
        return
    elif mode == 'fail':
        if not ds.repo or ds.repo.dirty:
            raise RuntimeError('dataset {} has unsaved changes'.format(ds))
    elif mode == 'save-before':
        if not ds.is_installed():
            raise RuntimeError('dataset {} is not yet installed'.format(ds))
        from datalad.core.local.save import Save
        Save.__call__(dataset=ds, message=msg, updated=True)
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def get_tree_roots(paths):
    """Return common root paths for a set of paths

    This function determines the smallest set of common root
    paths and sorts all given paths under the respective
    root.

    Returns
    -------
    dict
      paths by root
    """
    paths_ws = [_with_sep(p) for p in paths]
    # sort all paths under their potential roots
    roots = {}
    # start from the top to get all paths down the line
    # and collate them into as few roots as possible
    for s in sorted(paths_ws):
        if any([s.startswith(r) for r in roots]):
            # this path is already covered by a known root
            continue
        # find all sub paths
        subs = [p for p in paths if p.startswith(s)]
        roots[s.rstrip(sep)] = subs
    return roots


# TODO remove
# only `remove` and `uninstall` use this, the uses path `path_is_subpath`
def path_is_under(values, path=None):
    """Whether a given path is a subdirectory of any of the given test values

    Parameters
    ----------
    values : sequence or dict
      Paths to be tested against. This can be a dictionary in which case
      all values from all keys will be tested against.
    path : path or None
      Test path. If None is given, the process' working directory is
      used.

    Returns
    -------
    bool
    """
    if path is None:
        from datalad.utils import getpwd
        path = getpwd()
    if isinstance(values, dict):
        values = chain(*values.values())
    path_drive, _ = op.splitdrive(path)
    for p in values:
        p_drive, _ = op.splitdrive(p)
        # need to protect against unsupported use of relpath() with
        # abspaths on windows from different drives (gh-3724)
        if path_drive != p_drive:
        # different drives, enough evidence for "not under"
            continue
        rpath = relpath(p, start=path)
        if rpath == curdir \
                or rpath == pardir \
                or set(psplit(rpath)) == {pardir}:
            # first match is enough
            return True
    return False


def discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,
                                      spec, includeds=None):
    """Discover the edges and nodes in a dataset tree to given target paths

    Parameters
    ----------
    basepath : path
      Path to a start or top-level dataset. Really has to be a path to a
      dataset!
    targetpaths : list(path)
      Any non-zero number of paths that are termination points for the
      search algorithm. Can be paths to datasets, directories, or files
      (and any combination thereof).
    current_trace : list
      For a top-level call this should probably always be `[]`
    spec : dict
      `content_by_ds`-style dictionary that will receive information about the
      discovered datasets. Specifically, for each discovered dataset there
      will be in item with its path under the key (path) of the respective
      superdataset.
    includeds : sequence, optional
      Any paths given are treated as existing subdatasets, regardless of
      whether they can be found in the filesystem. Such subdatasets will appear
      under the key of the closest existing dataset in the `spec`.

    Returns
    -------
    None
      Function calls itself recursively and populates `spec` dict in-place.
      Keys are dataset paths, values are sets of subdataset paths
    """
    # convert to set for faster lookup
    includeds = includeds if isinstance(includeds, set) else \
        set() if includeds is None else set(includeds)
    # this beast walks the directory tree from a given `basepath` until
    # it discovers any of the given `targetpaths`
    # if it finds one, it commits any accummulated trace of visited
    # datasets on this edge to the spec
    valid_repo = GitRepo.is_valid_repo(basepath)
    if valid_repo:
        # we are passing into a new dataset, extend the dataset trace
        current_trace = current_trace + [basepath]
    # this edge is not done, we need to try to reach any downstream
    # dataset
    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)
    # whether anything in this directory matched a targetpath
    filematch = False
    if isdir(basepath):
        for p in listdir(basepath):
            p = assure_unicode(opj(basepath, p))
            if not isdir(p):
                if p in targetpaths:
                    filematch = True
                # we cannot have anything below this one
                continue
            # OPT listdir might be large and we could have only few items
            # in `targetpaths` -- so traverse only those in spec which have
            # leading dir basepath
            # filter targets matching this downward path
            downward_targets = set(
                t for t in targetpaths if path_startswith(t, p))
            if not downward_targets:
                continue
            # remove the matching ones from the "todo" list
            undiscovered_ds.difference_update(downward_targets)
            # go one deeper
            discover_dataset_trace_to_targets(
                p, downward_targets, current_trace, spec,
                includeds=includeds if not includeds else includeds.intersection(
                    downward_targets))
    undiscovered_ds = [t for t in undiscovered_ds
                       if includeds and
                          path_is_subpath(t, current_trace[-1]) and
                          t in includeds]
    if filematch or basepath in targetpaths or undiscovered_ds:
        for i, p in enumerate(current_trace[:-1]):
            # TODO RF prepare proper annotated path dicts
            subds = spec.get(p, set())
            subds.add(current_trace[i + 1])
            spec[p] = subds
        if undiscovered_ds:
            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(
                undiscovered_ds)


def get_result_filter(fx):
    """Wrap a filter into a helper to be able to accept additional
    arguments, if the filter doesn't support it already"""
    _fx = fx
    if fx and not getargspec(fx).keywords:
        def _fx(res, **kwargs):
            return fx(res)
    return _fx


def eval_results(func):
    """Decorator for return value evaluation of datalad commands.

    Note, this decorator is only compatible with commands that return
    status dict sequences!

    Two basic modes of operation are supported: 1) "generator mode" that
    `yields` individual results, and 2) "list mode" that returns a sequence of
    results. The behavior can be selected via the kwarg `return_type`.
    Default is "list mode".

    This decorator implements common functionality for result rendering/output,
    error detection/handling, and logging.

    Result rendering/output can be triggered via the
    `datalad.api.result-renderer` configuration variable, or the
    `result_renderer` keyword argument of each decorated command. Supported
    modes are: 'default' (one line per result with action, status, path,
    and an optional message); 'json' (one object per result, like git-annex),
    'json_pp' (like 'json', but pretty-printed spanning multiple lines),
    'tailored' custom output formatting provided by each command
    class (if any).

    Error detection works by inspecting the `status` item of all result
    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'
    will cause an IncompleteResultsError exception to be raised that carries
    the failed actions' status dictionaries in its `failed` attribute.

    Status messages will be logged automatically, by default the following
    association of result status and log channel will be used: 'ok' (debug),
    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger
    instances included in the results are used to capture the origin of a
    status report.

    Parameters
    ----------
    func: function
      __call__ method of a subclass of Interface,
      i.e. a datalad command definition
    """

    @wrapt.decorator
    def eval_func(wrapped, instance, args, kwargs):
        lgr.log(2, "Entered eval_func for %s", func)
        # for result filters
        # we need to produce a dict with argname/argvalue pairs for all args
        # incl. defaults and args given as positionals
        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)

        # determine the command class associated with `wrapped`
        wrapped_class = get_wrapped_class(wrapped)

        # retrieve common options from kwargs, and fall back on the command
        # class attributes, or general defaults if needed
        kwargs = kwargs.copy()  # we will pop, which might cause side-effect
        common_params = {
            p_name: kwargs.pop(
                # go with any explicitly given default
                p_name,
                # otherwise determine the command class and pull any
                # default set in that class
                getattr(
                    wrapped_class,
                    p_name,
                    # or the common default
                    eval_defaults[p_name]))
            for p_name in eval_params}

        # short cuts and configured setup for common options
        return_type = common_params['return_type']
        result_filter = get_result_filter(common_params['result_filter'])
        # resolve string labels for transformers too
        result_xfm = known_result_xfms.get(
            common_params['result_xfm'],
            # use verbatim, if not a known label
            common_params['result_xfm'])
        result_renderer = common_params['result_renderer']
        # TODO remove this conditional branch entirely, done outside
        if not result_renderer:
            result_renderer = dlcfg.get('datalad.api.result-renderer', None)
        # look for potential override of logging behavior
        result_log_level = dlcfg.get('datalad.log.result-level', None)

        # query cfg for defaults
        # .is_installed and .config can be costly, so ensure we do
        # it only once. See https://github.com/datalad/datalad/issues/3575
        dataset_arg = allkwargs.get('dataset', None)
        from datalad.distribution.dataset import Dataset
        ds = dataset_arg if isinstance(dataset_arg, Dataset) \
            else Dataset(dataset_arg) if dataset_arg else None
        # do not reuse a dataset's existing config manager here
        # they are configured to read the committed dataset configuration
        # too. That means a datalad update can silently bring in new
        # procedure definitions from the outside, and in some sense enable
        # remote code execution by a 3rd-party
        # To avoid that, create a new config manager that only reads local
        # config (system and .git/config), plus any overrides given to this
        # datalad session
        proc_cfg = ConfigManager(
            ds, source='local', overrides=dlcfg.overrides
        ) if ds and ds.is_installed() else dlcfg

        # look for hooks
        hooks = get_jsonhooks_from_config(proc_cfg)

        # this internal helper function actually drives the command
        # generator-style, it may generate an exception if desired,
        # on incomplete results
        def generator_func(*_args, **_kwargs):
            # flag whether to raise an exception
            incomplete_results = []
            # track what actions were performed how many times
            action_summary = {}

            # if a custom summary is to be provided, collect the results
            # of the command execution
            results = []
            do_custom_result_summary = result_renderer in ('tailored', 'default') \
                and hasattr(wrapped_class, 'custom_result_summary_renderer')

            # process main results
            for r in _process_results(
                    # execution
                    wrapped(*_args, **_kwargs),
                    wrapped_class,
                    common_params['on_failure'],
                    # bookkeeping
                    action_summary,
                    incomplete_results,
                    # communication
                    result_renderer,
                    result_log_level,
                    # let renderers get to see how a command was called
                    allkwargs):
                for hook, spec in hooks.items():
                    # run the hooks before we yield the result
                    # this ensures that they are executed before
                    # a potentially wrapper command gets to act
                    # on them
                    if match_jsonhook2result(hook, r, spec['match']):
                        lgr.debug('Result %s matches hook %s', r, hook)
                        # a hook is also a command that yields results
                        # so yield them outside too
                        # users need to pay attention to void infinite
                        # loops, i.e. when a hook yields a result that
                        # triggers that same hook again
                        for hr in run_jsonhook(hook, spec, r, dataset_arg):
                            # apply same logic as for main results, otherwise
                            # any filters would only tackle the primary results
                            # and a mixture of return values could happen
                            if not keep_result(hr, result_filter, **allkwargs):
                                continue
                            hr = xfm_result(hr, result_xfm)
                            # rationale for conditional is a few lines down
                            if hr:
                                yield hr
                if not keep_result(r, result_filter, **allkwargs):
                    continue
                r = xfm_result(r, result_xfm)
                # in case the result_xfm decided to not give us anything
                # exclude it from the results. There is no particular reason
                # to do so other than that it was established behavior when
                # this comment was written. This will not affect any real
                # result record
                if r:
                    yield r

                # collect if summary is desired
                if do_custom_result_summary:
                    results.append(r)

            # result summary before a potential exception
            # custom first
            if do_custom_result_summary:
                wrapped_class.custom_result_summary_renderer(results)
            elif result_renderer == 'default' and action_summary and \
                    sum(sum(s.values()) for s in action_summary.values()) > 1:
                # give a summary in default mode, when there was more than one
                # action performed
                ui.message("action summary:\n  {}".format(
                    '\n  '.join('{} ({})'.format(
                        act,
                        ', '.join('{}: {}'.format(status, action_summary[act][status])
                                  for status in sorted(action_summary[act])))
                                for act in sorted(action_summary))))

            if incomplete_results:
                raise IncompleteResultsError(
                    failed=incomplete_results,
                    msg="Command did not complete successfully")

        if return_type == 'generator':
            # hand over the generator
            lgr.log(2, "Returning generator_func from eval_func for %s", wrapped_class)
            return generator_func(*args, **kwargs)
        else:
            @wrapt.decorator
            def return_func(wrapped_, instance_, args_, kwargs_):
                results = wrapped_(*args_, **kwargs_)
                if inspect.isgenerator(results):
                    # unwind generator if there is one, this actually runs
                    # any processing
                    results = list(results)
                # render summaries
                if not result_xfm and result_renderer in ('tailored', 'default'):
                    # cannot render transformed results
                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):
                        wrapped_class.custom_result_summary_renderer(results)
                if return_type == 'item-or-list' and \
                        len(results) < 2:
                    return results[0] if results else None
                else:
                    return results
            lgr.log(2, "Returning return_func from eval_func for %s", wrapped_class)
            return return_func(generator_func)(*args, **kwargs)

    return eval_func(func)


def default_result_renderer(res):
    if res.get('status', None) != 'notneeded':
        path = res['path']
        path = str(path)
        ui.message('{action}({status}): {path}{type}{msg}'.format(
                action=ac.color_word(res['action'], ac.BOLD),
                status=ac.color_status(res['status']),
                path=relpath(path, res['refds']) if res.get('refds') else path,
                type=' ({})'.format(
                        ac.color_word(res['type'], ac.MAGENTA)
                ) if 'type' in res else '',
                msg=' [{}]'.format(
                        res['message'][0] % res['message'][1:]
                        if isinstance(res['message'], tuple) else res[
                            'message'])
                if res.get('message', None) else ''))


def _display_suppressed_message(nsimilar, ndisplayed):
    # +1 because there was the original result + nsimilar displayed.
    n_suppressed = nsimilar - ndisplayed + 1
    if n_suppressed > 0:
        ui.message('  [{} similar {} been suppressed]'
                   .format(n_suppressed,
                           single_or_plural("message has",
                                            "messages have",
                                            n_suppressed, False)))


def _process_results(
        results,
        cmd_class,
        on_failure,
        action_summary,
        incomplete_results,
        result_renderer,
        result_log_level,
        allkwargs):
    # private helper pf @eval_results
    # loop over results generated from some source and handle each
    # of them according to the requested behavior (logging, rendering, ...)

    # used to track repeated messages in the default renderer
    last_result = None
    # which result dict keys to inspect for changes to discover repetions
    # of similar messages
    repetition_keys = set(('action', 'status', 'type', 'refds'))
    # counter for detected repetitions
    result_repetitions = 0
    # how many repetitions to show, before suppression kicks in
    render_n_repetitions = 10

    for res in results:
        if not res or 'action' not in res:
            # XXX Yarik has to no clue on how to track the origin of the
            # record to figure out WTF, so he just skips it
            continue

        actsum = action_summary.get(res['action'], {})
        if res['status']:
            actsum[res['status']] = actsum.get(res['status'], 0) + 1
            action_summary[res['action']] = actsum
        ## log message, if there is one and a logger was given
        msg = res.get('message', None)
        # remove logger instance from results, as it is no longer useful
        # after logging was done, it isn't serializable, and generally
        # pollutes the output
        res_lgr = res.pop('logger', None)
        if msg and res_lgr:
            if isinstance(res_lgr, logging.Logger):
                # didn't get a particular log function, go with default
                res_lgr = getattr(
                    res_lgr,
                    default_logchannels[res['status']]
                    if result_log_level is None
                    else result_log_level)
            msg = res['message']
            msgargs = None
            if isinstance(msg, tuple):
                msgargs = msg[1:]
                msg = msg[0]
            if 'path' in res:
                msg = '{} [{}({})]'.format(
                    msg, res['action'], res['path'])
            if msgargs:
                # support string expansion of logging to avoid runtime cost
                res_lgr(msg, *msgargs)
            else:
                res_lgr(msg)

        ## output rendering
        # TODO RF this in a simple callable that gets passed into this function
        if result_renderer is None or result_renderer == 'disabled':
            pass
        elif result_renderer == 'default':
            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}
            if res.get('status', None) != 'notneeded' \
                    and trimmed_result == last_result:
                # this is a similar report, suppress if too many, but count it
                result_repetitions += 1
                if result_repetitions < render_n_repetitions:
                    default_result_renderer(res)
            else:
                # this one is new, first report on any prev. suppressed results
                # by number, and then render this fresh one
                _display_suppressed_message(
                    result_repetitions, render_n_repetitions)
                default_result_renderer(res)
                result_repetitions = 0
            last_result = trimmed_result
        elif result_renderer in ('json', 'json_pp'):
            ui.message(json.dumps(
                {k: v for k, v in res.items()
                 if k not in ('message', 'logger')},
                sort_keys=True,
                indent=2 if result_renderer.endswith('_pp') else None,
                default=lambda x: str(x)))
        elif result_renderer in ('tailored', 'default'):
            if hasattr(cmd_class, 'custom_result_renderer'):
                cmd_class.custom_result_renderer(res, **allkwargs)
        elif hasattr(result_renderer, '__call__'):
            try:
                result_renderer(res, **allkwargs)
            except Exception as e:
                lgr.warning('Result rendering failed for: %s [%s]',
                            res, exc_str(e))
        else:
            raise ValueError('unknown result renderer "{}"'.format(result_renderer))

        ## error handling
        # looks for error status, and report at the end via
        # an exception
        if on_failure in ('continue', 'stop') \
                and res['status'] in ('impossible', 'error'):
            incomplete_results.append(res)
            if on_failure == 'stop':
                # first fail -> that's it
                # raise will happen after the loop
                break
        yield res
    # make sure to report on any issues that we had suppressed
    _display_suppressed_message(
        result_repetitions, render_n_repetitions)


def keep_result(res, rfilter, **kwargs):
    if not rfilter:
        return True
    try:
        if not rfilter(res, **kwargs):
            # give the slightest indication which filter was employed
            raise ValueError(
                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))
    except ValueError as e:
        # make sure to report the excluded result to massively improve
        # debugging experience
        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)
        return False
    return True


def xfm_result(res, xfm):
    if not xfm:
        return res

    return xfm(res)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user