{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test dirty dataset handling\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os\nfrom os.path import join as opj\nfrom os.path import relpath\nfrom nose.tools import assert_raises, assert_equal\nfrom datalad.tests.utils import with_tempfile, assert_not_equal\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import create_tree\nfrom datalad.tests.utils import ok_clean_git\nfrom datalad.tests.utils import ok_\nfrom datalad.interface.utils import handle_dirty_dataset\nfrom datalad.interface.utils import get_paths_by_dataset\nfrom datalad.interface.utils import save_dataset_hierarchy\nfrom datalad.interface.utils import get_dataset_directories\nfrom datalad.interface.utils import filter_unmodified\nfrom datalad.interface.save import Save\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.utils import _install_subds_inplace\nfrom datalad.api import save\n\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: staged\n    with open(opj(ds.path, 'staged'), 'w') as f:\n        f.write('some')\n    ds.repo.add('staged', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    ok_clean_git(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.get_subdatasets(), ['subds'])\n\n\n@with_tempfile(mkdir=True)\ndef test_paths_by_dataset(path):\n    ds = Dataset(path).create()\n    subds = ds.create('one')\n    subsubds = subds.create('two')\n    d, ua, ne = get_paths_by_dataset([path])\n    for t in (ua, ne):\n        assert_equal(t, [])\n    assert_equal(d, {ds.path: [ds.path]})\n\n    d, ua, ne = get_paths_by_dataset(\n        [path], recursive=True)\n    for t in (ua, ne):\n        assert_equal(t, [])\n\n    os.makedirs(opj(ds.path, 'one', 'some'))\n    hidden = subds.create(opj('some', 'deep'))\n    testpath = opj(subds.path, 'some')\n    d, ua, ne = get_paths_by_dataset([testpath], recursive=True)\n    for t in (ua, ne):\n        assert_equal(t, [])\n    # must contain the containing dataset, and the testpath exactly\n    assert_equal(d[subds.path], [testpath])\n    # and also the subdataset underneath\n    assert_equal(d[hidden.path], [hidden.path])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree):\n    created_ds = []\n    for node, items in tree.items():\n        node_path = opj(path, node)\n        if isinstance(items, dict):\n            ds = make_demo_hierarchy_datasets(node_path, items)\n            created_ds.append(ds)\n    topds = Dataset(path)\n    if not topds.is_installed():\n        topds.create(force=True)\n        # TODO this farce would not be necessary if add() could add subdatasets\n        for ds in created_ds:\n            _install_subds_inplace(ds=topds, path=ds.path, relativepath=relpath(ds.path, topds.path))\n            ds.save()\n        return topds\n\n\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.add('.', recursive=True)\n    ok_clean_git(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    ds_bb.save(files=ds_bbaa.path, super_datasets=True)\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        ok_clean_git(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    save_dataset_hierarchy((d.path, da.path, db.path))\n    for d in (d, da, db):\n        ok_clean_git(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(files=(aa.path, ba.path, bb.path, c.path, ca.path, d.path),\n            super_datasets=True)\n    ok_clean_git(ds.path)\n\n\n@with_tempfile(mkdir=True)\ndef test_get_dataset_directories(path):\n    assert_raises(ValueError, get_dataset_directories, path)\n    ds = Dataset(path).create()\n    # ignores .git always and .datalad by default\n    assert_equal(get_dataset_directories(path), [])\n    assert_equal(get_dataset_directories(path, ignore_datalad=False),\n                 [opj(path, '.datalad')])\n    # find any directory, not just those known to git\n    testdir = opj(path, 'newdir')\n    os.makedirs(testdir)\n    assert_equal(get_dataset_directories(path), [testdir])\n    # do not find files\n    with open(opj(path, 'somefile'), 'w') as f:\n        f.write('some')\n    assert_equal(get_dataset_directories(path), [testdir])\n    # find more than one directory\n    testdir2 = opj(path, 'newdir2')\n    os.makedirs(testdir2)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2]))\n    # find subdataset mount points\n    subdsdir = opj(path, 'sub')\n    subds = ds.create(subdsdir)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    # do not find content within subdataset dirs\n    os.makedirs(opj(path, 'sub', 'deep'))\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    subsubdsdir = opj(subdsdir, 'subsub')\n    subds.create(subsubdsdir)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    # find nested directories\n    testdir3 = opj(testdir2, 'newdir21')\n    os.makedirs(testdir3)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, testdir3, subdsdir]))\n    # only return hits below the search path\n    assert_equal(sorted(get_dataset_directories(testdir2)), sorted([testdir3]))\n    # empty subdataset mount points are reported too\n    ds.uninstall(subds.path, check=False, recursive=True)\n    ok_(not subds.is_installed())\n    ok_(os.path.exists(subds.path))\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, testdir3, subdsdir]))\n\n\ndef test_interface_prep():\n    # verify sanity if nothing was given, as it would look like from the\n    # cmdline\n    assert_equal(Save._prep(path=[], dataset=None), ({}, []))\n\n\n@with_tree(demo_hierarchy['b'])\ndef test_filter_unmodified(path):\n    ds = Dataset(path).create(force=True)\n    suba = ds.create('ba', force=True)\n    subb = ds.create('bb', force=True)\n    subsub = ds.create(opj('bb', 'bba', 'bbaa'), force=True)\n    ds.add('.', recursive=True)\n    ok_clean_git(path)\n\n    spec, unavail = Save._prep('.', ds, recursive=True)\n    # just to be sure -- find all datasets, and just datasets\n    assert_equal(len(spec), 4)\n    for r, p in spec.items():\n        assert_equal([r], p)\n\n    orig_base_commit = ds.repo.repo.commit()\n    # nothing was modified compared to the status quo, output must be empty\n    assert_equal({}, filter_unmodified(spec, ds, orig_base_commit))\n\n    # modify one subdataset\n    added_path = opj(subb.path, 'added')\n    create_tree(subb.path, {'added': 'test'})\n    subb.add('added')\n\n    # still nothing was modified compared to orig commit, because the base\n    # dataset does not have the recent change saved\n    assert_equal({}, filter_unmodified(spec, ds, orig_base_commit))\n\n    ds.save(all_changes=True)\n\n    modspec, unavail = Save._prep('.', ds, recursive=True)\n    # arg sorting is not affected\n    assert_equal(spec, modspec)\n\n    # only the actually modified components per dataset are kept\n    assert_equal(\n        {\n            ds.path: [subb.path],\n            subb.path: [added_path]\n        },\n        filter_unmodified(spec, ds, orig_base_commit))\n\n    # deal with removal (force insufiicient copies error)\n    ds.remove(opj(subsub.path, 'file_bbaa'), check=False)\n    # saves all the way up\n    ok_clean_git(path)\n\n    modspec, unavail = Save._prep('.', ds, recursive=True)\n    # arg sorting is again not affected\n    assert_equal(spec, modspec)\n    # only the actually modified components per dataset are kept\n    assert_equal(\n        {\n            ds.path: [subb.path],\n            subb.path: [added_path, subsub.path],\n            subsub.path: []\n        },\n        {d: sorted(p) for d, p in filter_unmodified(spec, ds, orig_base_commit).items()})\n\n\ndef test_eval_results():\n\n    from ..base import Interface\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from inspect import getargspec\n\n    class FakeCommand(Interface):\n\n        @staticmethod\n        @datasetmethod(name='fake_command')\n        @eval_results\n        def __call__(number, dataset=None):\n\n            for i in range(number):\n                yield i\n\n    result = FakeCommand().__call__(2)\n    assert_equal(result, [0, 1])\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(result, [0, 1, 2])\n    assert_equal(getargspec(Dataset.fake_command)[0], ['number', 'dataset'])\n    assert_equal(getargspec(FakeCommand.__call__)[0], ['number', 'dataset'])\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test dirty dataset handling\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os\nfrom os.path import join as opj\nfrom os.path import relpath\nfrom nose.tools import assert_raises, assert_equal\nfrom datalad.tests.utils import with_tempfile, assert_not_equal\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import create_tree\nfrom datalad.tests.utils import ok_clean_git\nfrom datalad.tests.utils import ok_\nfrom datalad.interface.utils import handle_dirty_dataset\nfrom datalad.interface.utils import get_paths_by_dataset\nfrom datalad.interface.utils import save_dataset_hierarchy\nfrom datalad.interface.utils import get_dataset_directories\nfrom datalad.interface.utils import filter_unmodified\nfrom datalad.interface.save import Save\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.utils import _install_subds_inplace\nfrom datalad.api import save\n\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: staged\n    with open(opj(ds.path, 'staged'), 'w') as f:\n        f.write('some')\n    ds.repo.add('staged', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    ok_clean_git(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.get_subdatasets(), ['subds'])\n\n\n@with_tempfile(mkdir=True)\ndef test_paths_by_dataset(path):\n    ds = Dataset(path).create()\n    subds = ds.create('one')\n    subsubds = subds.create('two')\n    d, ua, ne = get_paths_by_dataset([path])\n    for t in (ua, ne):\n        assert_equal(t, [])\n    assert_equal(d, {ds.path: [ds.path]})\n\n    d, ua, ne = get_paths_by_dataset(\n        [path], recursive=True)\n    for t in (ua, ne):\n        assert_equal(t, [])\n\n    os.makedirs(opj(ds.path, 'one', 'some'))\n    hidden = subds.create(opj('some', 'deep'))\n    testpath = opj(subds.path, 'some')\n    d, ua, ne = get_paths_by_dataset([testpath], recursive=True)\n    for t in (ua, ne):\n        assert_equal(t, [])\n    # must contain the containing dataset, and the testpath exactly\n    assert_equal(d[subds.path], [testpath])\n    # and also the subdataset underneath\n    assert_equal(d[hidden.path], [hidden.path])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree):\n    created_ds = []\n    for node, items in tree.items():\n        node_path = opj(path, node)\n        if isinstance(items, dict):\n            ds = make_demo_hierarchy_datasets(node_path, items)\n            created_ds.append(ds)\n    topds = Dataset(path)\n    if not topds.is_installed():\n        topds.create(force=True)\n        # TODO this farce would not be necessary if add() could add subdatasets\n        for ds in created_ds:\n            _install_subds_inplace(ds=topds, path=ds.path, relativepath=relpath(ds.path, topds.path))\n            ds.save()\n        return topds\n\n\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.add('.', recursive=True)\n    ok_clean_git(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    ds_bb.save(files=ds_bbaa.path, super_datasets=True)\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        ok_clean_git(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    save_dataset_hierarchy((d.path, da.path, db.path))\n    for d in (d, da, db):\n        ok_clean_git(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(files=(aa.path, ba.path, bb.path, c.path, ca.path, d.path),\n            super_datasets=True)\n    ok_clean_git(ds.path)\n\n\n@with_tempfile(mkdir=True)\ndef test_get_dataset_directories(path):\n    assert_raises(ValueError, get_dataset_directories, path)\n    ds = Dataset(path).create()\n    # ignores .git always and .datalad by default\n    assert_equal(get_dataset_directories(path), [])\n    assert_equal(get_dataset_directories(path, ignore_datalad=False),\n                 [opj(path, '.datalad')])\n    # find any directory, not just those known to git\n    testdir = opj(path, 'newdir')\n    os.makedirs(testdir)\n    assert_equal(get_dataset_directories(path), [testdir])\n    # do not find files\n    with open(opj(path, 'somefile'), 'w') as f:\n        f.write('some')\n    assert_equal(get_dataset_directories(path), [testdir])\n    # find more than one directory\n    testdir2 = opj(path, 'newdir2')\n    os.makedirs(testdir2)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2]))\n    # find subdataset mount points\n    subdsdir = opj(path, 'sub')\n    subds = ds.create(subdsdir)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    # do not find content within subdataset dirs\n    os.makedirs(opj(path, 'sub', 'deep'))\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    subsubdsdir = opj(subdsdir, 'subsub')\n    subds.create(subsubdsdir)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    # find nested directories\n    testdir3 = opj(testdir2, 'newdir21')\n    os.makedirs(testdir3)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, testdir3, subdsdir]))\n    # only return hits below the search path\n    assert_equal(sorted(get_dataset_directories(testdir2)), sorted([testdir3]))\n    # empty subdataset mount points are reported too\n    ds.uninstall(subds.path, check=False, recursive=True)\n    ok_(not subds.is_installed())\n    ok_(os.path.exists(subds.path))\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, testdir3, subdsdir]))\n\n\ndef test_interface_prep():\n    # verify sanity if nothing was given, as it would look like from the\n    # cmdline\n    assert_equal(Save._prep(path=[], dataset=None), ({}, []))\n\n\n@with_tree(demo_hierarchy['b'])\ndef test_filter_unmodified(path):\n    ds = Dataset(path).create(force=True)\n    suba = ds.create('ba', force=True)\n    subb = ds.create('bb', force=True)\n    subsub = ds.create(opj('bb', 'bba', 'bbaa'), force=True)\n    ds.add('.', recursive=True)\n    ok_clean_git(path)\n\n    spec, unavail = Save._prep('.', ds, recursive=True)\n    # just to be sure -- find all datasets, and just datasets\n    assert_equal(len(spec), 4)\n    for r, p in spec.items():\n        assert_equal([r], p)\n\n    orig_base_commit = ds.repo.repo.commit()\n    # nothing was modified compared to the status quo, output must be empty\n    assert_equal({}, filter_unmodified(spec, ds, orig_base_commit))\n\n    # modify one subdataset\n    added_path = opj(subb.path, 'added')\n    create_tree(subb.path, {'added': 'test'})\n    subb.add('added')\n\n    # still nothing was modified compared to orig commit, because the base\n    # dataset does not have the recent change saved\n    assert_equal({}, filter_unmodified(spec, ds, orig_base_commit))\n\n    ds.save(all_changes=True)\n\n    modspec, unavail = Save._prep('.', ds, recursive=True)\n    # arg sorting is not affected\n    assert_equal(spec, modspec)\n\n    # only the actually modified components per dataset are kept\n    assert_equal(\n        {\n            ds.path: [subb.path],\n            subb.path: [added_path]\n        },\n        filter_unmodified(spec, ds, orig_base_commit))\n\n    # deal with removal (force insufiicient copies error)\n    ds.remove(opj(subsub.path, 'file_bbaa'), check=False)\n    # saves all the way up\n    ok_clean_git(path)\n\n    modspec, unavail = Save._prep('.', ds, recursive=True)\n    # arg sorting is again not affected\n    assert_equal(spec, modspec)\n    # only the actually modified components per dataset are kept\n    assert_equal(\n        {\n            ds.path: [subb.path],\n            subb.path: [added_path, subsub.path],\n            subsub.path: []\n        },\n        {d: sorted(p) for d, p in filter_unmodified(spec, ds, orig_base_commit).items()})\n\n\ndef test_eval_results():\n\n    from ..base import Interface\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from inspect import getargspec\n\n    class Switcher(object):\n\n        def __init__(self, value):\n            self._value = value\n\n        def __call__(self, value=None):\n            if value is not None:\n                self._value = value\n            else:\n                return self._value\n\n    class FakeCommand(Interface):\n\n        switch = Switcher(False)\n        \n        @staticmethod\n        @datasetmethod(name='fake_command')\n        @eval_results(switch)\n        def __call__(number, dataset=None):\n\n            for i in range(number):\n                yield i\n\n    # test results:\n    result = FakeCommand().__call__(2)\n    assert_equal(result, [0, 1])\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(result, [0, 1, 2])\n    # test signature:\n    assert_equal(getargspec(Dataset.fake_command)[0], ['number', 'dataset'])\n    assert_equal(getargspec(FakeCommand.__call__)[0], ['number', 'dataset'])\n    # test switch:\n    from datalad.utils import swallow_logs\n    import logging\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        FakeCommand().__call__(2)\n        cml.assert_logged(\"switch OFF\", level='DEBUG')\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        FakeCommand.switch(True)\n        FakeCommand().__call__(2)\n        cml.assert_logged(\"switch ON\", level='DEBUG')\n\n",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "7f8999414",
                "new_commit_hash": "a6ab0e7a3",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 320,
                            "length": 19
                        },
                        "new_range": {
                            "start": 320,
                            "length": 45
                        },
                        "section": "def test_eval_results():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    from datalad.interface.utils import eval_results",
                                "type": "context"
                            },
                            {
                                "content": "    from inspect import getargspec",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    class Switcher(object):",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        def __init__(self, value):",
                                "type": "added"
                            },
                            {
                                "content": "            self._value = value",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        def __call__(self, value=None):",
                                "type": "added"
                            },
                            {
                                "content": "            if value is not None:",
                                "type": "added"
                            },
                            {
                                "content": "                self._value = value",
                                "type": "added"
                            },
                            {
                                "content": "            else:",
                                "type": "added"
                            },
                            {
                                "content": "                return self._value",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    class FakeCommand(Interface):",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        switch = Switcher(False)",
                                "type": "added"
                            },
                            {
                                "content": "        ",
                                "type": "added"
                            },
                            {
                                "content": "        @staticmethod",
                                "type": "context"
                            },
                            {
                                "content": "        @datasetmethod(name='fake_command')",
                                "type": "context"
                            },
                            {
                                "content": "        @eval_results",
                                "type": "deleted"
                            },
                            {
                                "content": "        @eval_results(switch)",
                                "type": "added"
                            },
                            {
                                "content": "        def __call__(number, dataset=None):",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "            for i in range(number):",
                                "type": "context"
                            },
                            {
                                "content": "                yield i",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # test results:",
                                "type": "added"
                            },
                            {
                                "content": "    result = FakeCommand().__call__(2)",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(result, [0, 1])",
                                "type": "context"
                            },
                            {
                                "content": "    result = Dataset('/does/not/matter').fake_command(3)",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(result, [0, 1, 2])",
                                "type": "context"
                            },
                            {
                                "content": "    # test signature:",
                                "type": "added"
                            },
                            {
                                "content": "    assert_equal(getargspec(Dataset.fake_command)[0], ['number', 'dataset'])",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(getargspec(FakeCommand.__call__)[0], ['number', 'dataset'])",
                                "type": "context"
                            },
                            {
                                "content": "    # test switch:",
                                "type": "added"
                            },
                            {
                                "content": "    from datalad.utils import swallow_logs",
                                "type": "added"
                            },
                            {
                                "content": "    import logging",
                                "type": "added"
                            },
                            {
                                "content": "    with swallow_logs(new_level=logging.DEBUG) as cml:",
                                "type": "added"
                            },
                            {
                                "content": "        FakeCommand().__call__(2)",
                                "type": "added"
                            },
                            {
                                "content": "        cml.assert_logged(\"switch OFF\", level='DEBUG')",
                                "type": "added"
                            },
                            {
                                "content": "    with swallow_logs(new_level=logging.DEBUG) as cml:",
                                "type": "added"
                            },
                            {
                                "content": "        FakeCommand.switch(True)",
                                "type": "added"
                            },
                            {
                                "content": "        FakeCommand().__call__(2)",
                                "type": "added"
                            },
                            {
                                "content": "        cml.assert_logged(\"switch ON\", level='DEBUG')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "function",
                            "name": "test_eval_results",
                            "content": "def test_eval_results():\n\n    from ..base import Interface\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from inspect import getargspec\n\n    class FakeCommand(Interface):\n\n        @staticmethod\n        @datasetmethod(name='fake_command')\n        @eval_results\n        def __call__(number, dataset=None):\n\n            for i in range(number):\n                yield i\n\n    result = FakeCommand().__call__(2)\n    assert_equal(result, [0, 1])\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(result, [0, 1, 2])\n    assert_equal(getargspec(Dataset.fake_command)[0], ['number', 'dataset'])\n    assert_equal(getargspec(FakeCommand.__call__)[0], ['number', 'dataset'])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 316,
                            "end_lineno": 338,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nfrom os.path import join as opj\nfrom os.path import lexists\nfrom os.path import isabs\nfrom os.path import isdir\nfrom os.path import dirname\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import assure_list\nfrom datalad.utils import get_trace\nfrom datalad.utils import walk\nfrom datalad.utils import get_dataset_root\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import better_wraps\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.annexrepo import AnnexRepo\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.distribution.utils import _install_subds_inplace\nfrom datalad.distribution.utils import get_git_dir\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.repo.is_dirty(index=True,\n                                                working_tree=True,\n                                                untracked_files=True,\n                                                submodules=True):\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.interface.save import Save\n        Save.__call__(dataset=ds, message=msg, all_changes=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef handle_dirty_datasets(dpaths,\n                          mode,\n                          base=None,\n                          msg='[DATALAD] auto-saved changes'):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    dpaths : sequence(path)\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    base : path or None, optional\n      Path of a common super dataset that should also be handled.\n    msg : str\n      Custom message to use for a potential saved state.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if mode == 'save-before':\n        save_dataset_hierarchy(\n            {d: [d] for d in dpaths},\n            base=base,\n            message=msg)\n    elif mode == 'ignore':\n        return\n    elif mode == 'fail':\n        for dpath in dpaths:\n            ds = Dataset(dpath)\n            if not ds.repo:\n                continue\n            ds.repo.precommit()\n            if ds.repo.repo.is_dirty(index=True,\n                                     working_tree=True,\n                                     untracked_files=True,\n                                     submodules=True):\n                raise RuntimeError(\n                    'dataset {} has unsaved changes'.format(ds))\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\ndef sort_paths_into_subdatasets(superds_path, target_subs, spec):\n    # XXX forge a chain: whenever some path needs to be pushed down\n    # put the receiving dataset as a components to process into the\n    # respective superdataset -- this will enable further processing\n    # of all datasets in a completely independent fashion\n    # (except for order of processing)\n\n    # get all existing subdataset as candidate nodes of the graph\n    # that needs to be built and checked\n    subds_graph = Dataset(superds_path).get_subdatasets(\n        absolute=True, recursive=True, edges=True, fulfilled=True)\n    if not subds_graph:\n        # no subdatasets, nothing to sort\n        return\n    for t in target_subs:\n        trace = get_trace(\n            subds_graph,\n            superds_path,\n            t)\n        if not trace:\n            # not connected, or identical\n            continue\n        tosort = [superds_path] + trace + [t]\n        # loop over all but the last one, simplifies logic below\n        for i, d in enumerate(tosort[:-1]):\n            paths = spec.get(d, [])\n            keep_paths = []\n            next_ds = tosort[i + 1]\n            next_dspaths = spec.get(next_ds, [])\n            comp = _with_sep(next_ds)\n            for p in assure_list(paths):\n                if p.startswith(comp):\n                    next_dspaths.append(p)\n                    # remember that we pushed the path into this dataset\n                    keep_paths.append(next_ds)\n                else:\n                    keep_paths.append(p)\n            spec[next_ds] = next_dspaths\n            spec[d] = keep_paths\n    # tidy up -- deduplicate\n    for c in spec:\n        spec[c] = list(set(spec[c]))\n\n\ndef save_dataset_hierarchy(\n        info,\n        base=None,\n        message='[DATALAD] saved changes',\n        version_tag=None):\n    \"\"\"Save (disjoint) hierarchies of datasets.\n\n    Saving is done in an order that guarantees that all to be saved\n    datasets reflect any possible change of any other to be saved\n    subdataset, before they are saved themselves.\n\n    Parameters\n    ----------\n    info : dict\n      Absolute paths of datasets to be saved are the keys, and paths in each\n      dataset to be saved are the values\n    base : path or None, optional\n      Common super dataset that should also be saved.\n    message : str\n      Message to be used for saving individual datasets\n\n    Returns\n    -------\n    list\n      Instances of saved datasets, in the order in which they where saved.\n    \"\"\"\n    if not isinstance(info, dict):\n        info = assure_list(info)\n        info = dict(zip(info, [[i] for i in info]))\n    dpaths = info.keys()\n    if base:\n        # just a convenience...\n        dpaths = assure_list(dpaths)\n        dpaths.append(base.path if isinstance(base, Dataset) else base)\n    # sort all datasets under their potential superdatasets\n    # start from the top to get all subdatasets down the line\n    # and collate them into as few superdatasets as possible\n    superdss = get_tree_roots(dpaths)\n    # for each \"superdataset\" check the tree of subdatasets and make sure\n    # we gather all datasets between the super and any subdataset\n    # so we can save them all bottom-up in order to be able to properly\n    # save the superdataset\n    for superds_path in superdss:\n        target_subs = superdss[superds_path]\n        sort_paths_into_subdatasets(superds_path, target_subs, info)\n    # iterate over all datasets, starting at the bottom\n    saved = []\n    for dpath in sorted(info.keys(), reverse=True):\n        ds = Dataset(dpath)\n        if ds.is_installed():\n            saved_state = save_dataset(\n                ds,\n                info[dpath],\n                message=message,\n                version_tag=version_tag)\n            if saved_state:\n                saved.append(ds)\n    return saved\n\n\ndef save_dataset(\n        ds,\n        paths=None,\n        message=None,\n        version_tag=None):\n    \"\"\"Save changes in a single dataset.\n\n    Parameters\n    ----------\n    ds : Dataset\n      The dataset to be saved.\n    paths : list, optional\n      Paths to dataset components to be saved.\n    message: str, optional\n      (Commit) message to be attached to the saved state.\n    version_tag : str, optional\n      Tag to be assigned to the saved state.\n\n    Returns\n    -------\n    bool\n      Whether a new state was saved. If all to be saved content was unmodified\n      no new state will be saved.\n    \"\"\"\n    # XXX paths must be in the given ds, no further sanity checks!\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    ds.repo.precommit()\n\n    # track what is to be committed, so it becomes\n    # possible to decide when/what to save further down\n    # and one level up\n    orig_hexsha = ds.repo.get_hexsha()\n\n    # always yields list; empty if None\n    files = list(\n        set(\n            [opj(ds.path, f) if not isabs(f) else f for f in assure_list(paths)]))\n\n    # try to consider existing and changed files, and prevent untracked\n    # files from being added\n    # XXX not acting upon untracked files would be very expensive, because\n    # I see no way to avoid using `add` below and git annex has no equivalent\n    # to git add's --update -- so for now don't bother\n    # XXX alternatively we could consider --no-ignore-removal to also\n    # have it delete any already vanished files\n    # asking yourself why we need to `add` at all? For example, freshly\n    # unlocked files in a v5 repo are listed as \"typechange\" and commit\n    # refuses to touch them without an explicit `add`\n    tostage = [f for f in files if lexists(f)]\n    if tostage:\n        lgr.debug('staging files for commit: %s', tostage)\n        if isinstance(ds.repo, AnnexRepo):\n            # to make this work without calling `git add` in addition,\n            # this needs git-annex v6.20161210 (see #1027)\n            ds.repo.add(tostage, commit=False)\n        else:\n            # --update will ignore any untracked files, sadly git-annex add\n            # above does not\n            # will complain about vanished files though, filter them here, but\n            # keep them for a later commit call\n            ds.repo.add(tostage, git_options=['--update'], commit=False)\n\n    _datalad_msg = False\n    if not message:\n        message = 'Recorded existing changes'\n        _datalad_msg = True\n\n    if files or ds.repo.repo.is_dirty(\n            index=True,\n            working_tree=False,\n            untracked_files=False,\n            submodules=True):\n        # either we have an explicit list of files, or we have something\n        # stages otherwise do not attempt to commit, as the underlying\n        # repo will happily commit any non-change\n        # not checking the working tree or untracked files should make this\n        # relavtively cheap\n\n        # TODO: commit() should rather report a dedicated ValueError\n        # waiting for #1170\n        from datalad.support.exceptions import CommandError\n        try:\n            # we will blindly call commit not knowing if there is anything to\n            # commit -- this is cheaper than to anticipate all possible ways\n            # a repo in whatever mode is dirty\n            # however, if nothing is dirty the whining wil start\n            # --> sucking it up right here\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                ds.repo.commit(message, options=files, _datalad_msg=_datalad_msg)\n        except CommandError as e:\n            # TODO until #1171 is resolved, test here for \"normal\" failure\n            # to commit\n            if 'nothing to commit' in str(e):\n                lgr.debug(\n                    \"Was instructed to commit %s files but repository is not dirty\",\n                    files)\n            elif 'no changes added to commit' in str(e) or 'nothing added to commit' in str(e):\n                lgr.info(\n                    'Nothing to save')\n            else:\n                # relay any prior whining in the exception\n                raise ValueError('{} [error log follows] {}; {}'.format(\n                    e, e.stdout, e.stderr))\n\n    # MIH: let's tag even if there was nothing commit. I'd forget this\n    # option too often...\n    if version_tag:\n        ds.repo.tag(version_tag)\n\n    _was_modified = ds.repo.get_hexsha() != orig_hexsha\n\n    return ds.repo.repo.head.commit if _was_modified else None\n\n\ndef amend_pathspec_with_superdatasets(spec, topmost=True, limit_single=False):\n    \"\"\"Amend a path spec dictionary with entries for superdatasets\n\n    The result will be a superdataset entry (if a superdataset exists)\n    for each input dataset. This entry will (at least) contain the path\n    to the subdataset.\n\n    Parameters\n    ----------\n    spec : dict\n      Path spec\n    topmost : Dataset or bool\n      Flag whether to grab the immediate, or the top-most superdataset\n      for each entry, alternatively this can be a dataset instance\n      that is used as the topmost dataset.\n    limit_single : bool\n      If a `topmost` dataset is provided, and this flag is True, only\n      the given topmost dataset will be considered as superdataset. Any\n      datasets in the spec that are not underneath this dataset will\n      not have associated superdataset entries added to the spec.\n\n    Returns\n    -------\n    dict\n      Amended path spec dictionary\n    \"\"\"\n    superdss = {}\n    for dpath in spec.keys():\n        superds = None\n        if isinstance(topmost, Dataset):\n            if limit_single and dpath == topmost.path:\n                # this is already the topmost, no further superdataset to\n                # consider\n                continue\n            if dpath.startswith(_with_sep(topmost.path)):\n                # the given topmost dataset is \"above\" the current\n                # datasets path\n                superds = topmost\n            elif limit_single:\n                continue\n        if not superds:\n            # grab the (topmost) superdataset\n            superds = Dataset(dpath).get_superdataset(\n                datalad_only=True, topmost=topmost)\n        if not superds:\n            continue\n        # register the subdatasets path in the spec of the superds\n        spaths = superdss.get(superds.path, [])\n        if not spaths:\n            spaths = spec.get(superds.path, [])\n        spaths.append(dpath)\n        superdss[superds.path] = spaths\n    spec.update(superdss)\n    return spec\n\n\ndef get_paths_by_dataset(paths, recursive=False, recursion_limit=None,\n                         out=None, dir_lookup=None):\n    \"\"\"Sort a list of paths per dataset they are contained in.\n\n    Any paths that are not part of a dataset, or presently unavailable are\n    reported.\n\n    Parameter\n    ---------\n    paths : sequence\n      A sequence of path specifications to sort.\n    recursive : bool\n      Flag whether to report subdatasets under any of the given paths\n    recursion_limit :\n      Depth constraint for recursion. See `Dataset.get_subdatasets()` for more\n      information.\n    out : dict or None\n      By default a new output dictionary is created, however an existing one\n      can be provided via this argument to enable incremental processing.\n    dir_lookup : dict or None\n      Optional lookup cache that maps paths to previously determined datasets.\n      This can speed up repeated processing.\n\n    Returns\n    -------\n    Tuple(dict, list, list)\n      Dict of `existing dataset path`: `path` mappings, the list of currently\n      non-existing paths (possibly matching currently uninstalled datasets),\n      and any paths that are not part of any dataset.\n    \"\"\"\n    # sort paths into the respective datasets\n    if dir_lookup is None:\n        dir_lookup = {}\n    if out is None:\n        out = {}\n    # paths that don't exist (yet)\n    unavailable_paths = []\n    nondataset_paths = []\n    for path in paths:\n        if not lexists(path):\n            # not there yet, impossible to say which ds it will actually\n            # be in, if any\n            unavailable_paths.append(path)\n            continue\n        # the path exists in some shape or form\n        if isdir(path):\n            # this could contain all types of additional content\n            d = path\n        else:\n            # for everything else we are interested in the container\n            d = dirname(path)\n            if not d:\n                d = curdir\n        # this could be `None` if there is no git repo\n        dspath = dir_lookup.get(d, get_dataset_root(d))\n        dir_lookup[d] = dspath\n        if not dspath:\n            nondataset_paths.append(path)\n            continue\n        if isdir(path):\n            ds = Dataset(dspath)\n            # we need to doublecheck that this is not a subdataset mount\n            # point, in which case get_toppath() would point to the parent\n            smpath = ds.get_containing_subdataset(\n                path, recursion_limit=1).path\n            if smpath != dspath:\n                # fix entry\n                dir_lookup[d] = smpath\n                # submodule still needs to be obtained\n                unavailable_paths.append(path)\n                continue\n            if recursive:\n                # make sure we get everything relevant in all _checked out_\n                # subdatasets, obtaining of previously unavailable subdataset\n                # else done elsewhere\n                subs = ds.get_subdatasets(fulfilled=True,\n                                          recursive=recursive,\n                                          recursion_limit=recursion_limit)\n                for sub in subs:\n                    subdspath = opj(dspath, sub)\n                    if subdspath.startswith(_with_sep(path)):\n                        # this subdatasets is underneath the search path\n                        # we want it all\n                        # be careful to not overwrite anything, in case\n                        # this subdataset has been processed before\n                        out[subdspath] = out.get(\n                            subdspath,\n                            [subdspath])\n        out[dspath] = out.get(dspath, []) + [path]\n    return out, unavailable_paths, nondataset_paths\n\n\ndef get_normalized_path_arguments(paths, dataset=None, default=None):\n    \"\"\"Apply standard resolution to path arguments\n\n    This is nothing more than a helper to standardize path argument\n    preprocessing.\n\n    Parameter\n    ---------\n    paths : sequence or single path\n      Path(s) to normalize\n    dataset : path or Dataset or None\n      Optional dataset identifying something against which to resolve input\n      path arguments\n    default: sequence of paths or single path or None\n      If `paths` is empty, use this instead\n\n    Returns\n    -------\n    tuple(list(paths), path)\n      Normalized paths and path to a potential dataset against which paths were\n      resolved.\n    \"\"\"\n    dataset_path = dataset.path if isinstance(dataset, Dataset) else dataset\n    if not paths and default:\n        paths = default\n    paths = assure_list(paths)\n    # resolve path(s):\n    resolved_paths = [resolve_path(p, dataset) for p in paths]\n    if dataset:\n        # guarantee absolute paths\n        resolved_paths = [opj(dataset_path, p) for p in resolved_paths]\n    lgr.debug('Resolved input path arguments: %s', resolved_paths)\n    return resolved_paths, dataset_path\n\n\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        pwd = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    for p in values:\n        rpath = relpath(p, start=pwd)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef get_dataset_directories(top, ignore_datalad=True):\n    \"\"\"Return a list of directories in the same dataset under a given path\n\n    Parameters\n    ----------\n    top : path\n      Top-level path\n    ignore_datalad : bool\n      Whether to exlcude the '.datalad' directory of a dataset and its content\n      from the results.\n\n    Returns\n    -------\n    list\n      List of directories matching the top-level path, regardless of whether\n      these directories are known to Git (i.e. contain tracked files). The\n      list does not include the top-level path itself, but it does include\n      any subdataset mount points (regardless of whether the particular\n      subdatasets are installed or not).\n    \"\"\"\n    def func(arg, top, names):\n        refpath, ignore, dirs = arg\n        legit_names = []\n        for n in names:\n            path = opj(top, n)\n            if not isdir(path) or path in ignore:\n                pass\n            elif path != refpath and GitRepo.is_valid_repo(path):\n                # mount point, keep but don't dive into\n                dirs.append(path)\n            else:\n                legit_names.append(n)\n                dirs.append(path)\n        names[:] = legit_names\n\n    # collects the directories\n    refpath = get_dataset_root(top)\n    if not refpath:\n        raise ValueError(\"`top` path {} is not in a dataset\".format(top))\n    ignore = [opj(refpath, get_git_dir(refpath))]\n    if ignore_datalad:\n        ignore.append(opj(refpath, '.datalad'))\n    d = []\n    walk(top, func, (refpath, ignore, d))\n    return d\n\n\n# XXX the following present a different approach to\n# amend_pathspec_with_superdatasets() for discovering datasets between\n# processed ones and a base\n# let it simmer for a while and RF to use one or the other\n# this one here seems more leightweight and less convoluted\ndef _discover_trace_to_known(path, trace, spec):\n    # this beast walks the directory tree from a given `path` until\n    # it discoveres a known dataset (i.e. recorded in the spec)\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(path)\n    if valid_repo:\n        trace = trace + [path]\n        if path in spec:\n            # found a known repo, commit the trace\n            for i, p in enumerate(trace[:-1]):\n                spec[p] = list(set(spec.get(p, []) + [trace[i + 1]]))\n            # this edge is not done, we need to try to reach any downstream\n            # dataset\n    for p in listdir(path):\n        if valid_repo and p == '.git':\n            # ignore gitdir to steed things up\n            continue\n        p = opj(path, p)\n        if not isdir(p):\n            continue\n        _discover_trace_to_known(p, trace, spec)\n\n\ndef filter_unmodified(content_by_ds, refds, since):\n    \"\"\"Filter per-dataset path specifications based on modification history.\n\n    This function takes a path specification dictionary, as produced by\n    `Interface._prep()` and filters it such that only that subset of paths\n    remains in the dictionary that corresponding to the set of changes in\n    the given reference dataset since a given state.\n\n    The change set is traced across all related subdatasets, i.e. if a submodule\n    in the reference dataset is reported as modified then all paths for any given\n    subdataset in the modified one are tested for changes too (based on the\n    state at which the parent dataset reports a change in the subdataset), and so\n    on.\n\n    In doing so, not only unmodified given paths are removed, but also modified\n    given paths are replaced by the set of actually modified paths within them.\n\n    Only committed changes are considered!\n\n    Parameters\n    ----------\n    content_by_ds : dict\n      Per-dataset path specifications, as produced ,for example, by\n      `Interface._prep()`\n    refds : Dataset or *Repo or path\n      Reference dataset for which to determine the initial change set\n    since : state\n      Any commit-ish/tree-ish supported by Git (tag, commit, branch, ...).\n      Changes between this given state and the most recent commit are\n      evaluated.\n\n    Returns\n    -------\n    dict\n      Filtered path spec dictionary. If `since` is not None, the output is\n      guaranteed to only contain paths to modified, and presently existing\n      components of subdatasets of the given reference dataset (and itself).\n    \"\"\"\n    if since is None:\n        # we want all, subds not matching the ref are assumed to have been\n        # sorted out before (e.g. one level up)\n        return content_by_ds\n    # turn refds argument into a usable repo instance\n    if not hasattr(refds, 'path'):\n        # not a Repo or Dataset\n        refds_path = refds\n        refds = GitRepo(refds, create=False)\n    else:\n        refds_path = refds.path\n    repo = refds.repo\n    if hasattr(repo, 'repo'):\n        # TODO use GitRepo.diff() when available (gh-1217)\n        repo = repo.repo\n\n    # life is simple: we diff the base dataset, and kill anything that\n    # does not start with something that is in the diff\n    # we cannot really limit the diff paths easily because we might get\n    # or miss content (e.g. subdatasets) if we don't figure out which ones\n    # are known -- and we don't want that\n    diff = repo.commit().diff(since)\n    # get all modified paths (with original? commit) that are still\n    # present\n    modified = dict((opj(refds_path, d.b_path),\n                    d.b_blob.hexsha if d.b_blob else None)\n                    for d in diff)\n    if not modified:\n        # nothing modified nothing to report\n        return {}\n    # determine the subset that is a directory and hence is relevant for possible\n    # subdatasets\n    modified_dirs = {_with_sep(d) for d in modified if isdir(d)}\n    # find the subdatasets matching modified paths, this will also kick out\n    # any paths that are not in the dataset sub-hierarchy\n    mod_subs = {candds: paths\n                for candds, paths in content_by_ds.items()\n                if candds != refds_path and\n                any(_with_sep(candds).startswith(md) for md in modified_dirs)}\n    # now query the next level down\n    keep_subs = \\\n        [filter_unmodified(mod_subs, subds_path, modified[subds_path])\n         for subds_path in mod_subs\n         if subds_path in modified]\n    # merge result list into a single dict\n    keep = {k: v for d in keep_subs for k, v in d.items()}\n\n    paths_refds = content_by_ds[refds_path]\n    keep[refds_path] = [m for m in modified\n                        if lexists(m)  # still around\n                        and (m in paths_refds  # listed file, or subds\n                        # or a modified path under a given directory\n                        or any(m.startswith(_with_sep(p)) for p in paths_refds))]\n    return keep\n\nimport wrapt\n@wrapt.decorator\ndef eval_results(wrapped, instance, args, kwargs):\n    \"\"\"Decorator providing functionality to evaluate return values of datalad\n    commands\n    \"\"\"\n    from inspect import isgenerator\n\n    # rudimentary wrapper to harvest generators\n    results = wrapped(*args, **kwargs)\n    if isgenerator(results):\n        return list(results)\n    else:\n        return results\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nfrom os.path import join as opj\nfrom os.path import lexists\nfrom os.path import isabs\nfrom os.path import isdir\nfrom os.path import dirname\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import assure_list\nfrom datalad.utils import get_trace\nfrom datalad.utils import walk\nfrom datalad.utils import get_dataset_root\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import better_wraps\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.annexrepo import AnnexRepo\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.distribution.utils import _install_subds_inplace\nfrom datalad.distribution.utils import get_git_dir\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.repo.is_dirty(index=True,\n                                                working_tree=True,\n                                                untracked_files=True,\n                                                submodules=True):\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.interface.save import Save\n        Save.__call__(dataset=ds, message=msg, all_changes=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef handle_dirty_datasets(dpaths,\n                          mode,\n                          base=None,\n                          msg='[DATALAD] auto-saved changes'):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    dpaths : sequence(path)\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    base : path or None, optional\n      Path of a common super dataset that should also be handled.\n    msg : str\n      Custom message to use for a potential saved state.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if mode == 'save-before':\n        save_dataset_hierarchy(\n            {d: [d] for d in dpaths},\n            base=base,\n            message=msg)\n    elif mode == 'ignore':\n        return\n    elif mode == 'fail':\n        for dpath in dpaths:\n            ds = Dataset(dpath)\n            if not ds.repo:\n                continue\n            ds.repo.precommit()\n            if ds.repo.repo.is_dirty(index=True,\n                                     working_tree=True,\n                                     untracked_files=True,\n                                     submodules=True):\n                raise RuntimeError(\n                    'dataset {} has unsaved changes'.format(ds))\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\ndef sort_paths_into_subdatasets(superds_path, target_subs, spec):\n    # XXX forge a chain: whenever some path needs to be pushed down\n    # put the receiving dataset as a components to process into the\n    # respective superdataset -- this will enable further processing\n    # of all datasets in a completely independent fashion\n    # (except for order of processing)\n\n    # get all existing subdataset as candidate nodes of the graph\n    # that needs to be built and checked\n    subds_graph = Dataset(superds_path).get_subdatasets(\n        absolute=True, recursive=True, edges=True, fulfilled=True)\n    if not subds_graph:\n        # no subdatasets, nothing to sort\n        return\n    for t in target_subs:\n        trace = get_trace(\n            subds_graph,\n            superds_path,\n            t)\n        if not trace:\n            # not connected, or identical\n            continue\n        tosort = [superds_path] + trace + [t]\n        # loop over all but the last one, simplifies logic below\n        for i, d in enumerate(tosort[:-1]):\n            paths = spec.get(d, [])\n            keep_paths = []\n            next_ds = tosort[i + 1]\n            next_dspaths = spec.get(next_ds, [])\n            comp = _with_sep(next_ds)\n            for p in assure_list(paths):\n                if p.startswith(comp):\n                    next_dspaths.append(p)\n                    # remember that we pushed the path into this dataset\n                    keep_paths.append(next_ds)\n                else:\n                    keep_paths.append(p)\n            spec[next_ds] = next_dspaths\n            spec[d] = keep_paths\n    # tidy up -- deduplicate\n    for c in spec:\n        spec[c] = list(set(spec[c]))\n\n\ndef save_dataset_hierarchy(\n        info,\n        base=None,\n        message='[DATALAD] saved changes',\n        version_tag=None):\n    \"\"\"Save (disjoint) hierarchies of datasets.\n\n    Saving is done in an order that guarantees that all to be saved\n    datasets reflect any possible change of any other to be saved\n    subdataset, before they are saved themselves.\n\n    Parameters\n    ----------\n    info : dict\n      Absolute paths of datasets to be saved are the keys, and paths in each\n      dataset to be saved are the values\n    base : path or None, optional\n      Common super dataset that should also be saved.\n    message : str\n      Message to be used for saving individual datasets\n\n    Returns\n    -------\n    list\n      Instances of saved datasets, in the order in which they where saved.\n    \"\"\"\n    if not isinstance(info, dict):\n        info = assure_list(info)\n        info = dict(zip(info, [[i] for i in info]))\n    dpaths = info.keys()\n    if base:\n        # just a convenience...\n        dpaths = assure_list(dpaths)\n        dpaths.append(base.path if isinstance(base, Dataset) else base)\n    # sort all datasets under their potential superdatasets\n    # start from the top to get all subdatasets down the line\n    # and collate them into as few superdatasets as possible\n    superdss = get_tree_roots(dpaths)\n    # for each \"superdataset\" check the tree of subdatasets and make sure\n    # we gather all datasets between the super and any subdataset\n    # so we can save them all bottom-up in order to be able to properly\n    # save the superdataset\n    for superds_path in superdss:\n        target_subs = superdss[superds_path]\n        sort_paths_into_subdatasets(superds_path, target_subs, info)\n    # iterate over all datasets, starting at the bottom\n    saved = []\n    for dpath in sorted(info.keys(), reverse=True):\n        ds = Dataset(dpath)\n        if ds.is_installed():\n            saved_state = save_dataset(\n                ds,\n                info[dpath],\n                message=message,\n                version_tag=version_tag)\n            if saved_state:\n                saved.append(ds)\n    return saved\n\n\ndef save_dataset(\n        ds,\n        paths=None,\n        message=None,\n        version_tag=None):\n    \"\"\"Save changes in a single dataset.\n\n    Parameters\n    ----------\n    ds : Dataset\n      The dataset to be saved.\n    paths : list, optional\n      Paths to dataset components to be saved.\n    message: str, optional\n      (Commit) message to be attached to the saved state.\n    version_tag : str, optional\n      Tag to be assigned to the saved state.\n\n    Returns\n    -------\n    bool\n      Whether a new state was saved. If all to be saved content was unmodified\n      no new state will be saved.\n    \"\"\"\n    # XXX paths must be in the given ds, no further sanity checks!\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    ds.repo.precommit()\n\n    # track what is to be committed, so it becomes\n    # possible to decide when/what to save further down\n    # and one level up\n    orig_hexsha = ds.repo.get_hexsha()\n\n    # always yields list; empty if None\n    files = list(\n        set(\n            [opj(ds.path, f) if not isabs(f) else f for f in assure_list(paths)]))\n\n    # try to consider existing and changed files, and prevent untracked\n    # files from being added\n    # XXX not acting upon untracked files would be very expensive, because\n    # I see no way to avoid using `add` below and git annex has no equivalent\n    # to git add's --update -- so for now don't bother\n    # XXX alternatively we could consider --no-ignore-removal to also\n    # have it delete any already vanished files\n    # asking yourself why we need to `add` at all? For example, freshly\n    # unlocked files in a v5 repo are listed as \"typechange\" and commit\n    # refuses to touch them without an explicit `add`\n    tostage = [f for f in files if lexists(f)]\n    if tostage:\n        lgr.debug('staging files for commit: %s', tostage)\n        if isinstance(ds.repo, AnnexRepo):\n            # to make this work without calling `git add` in addition,\n            # this needs git-annex v6.20161210 (see #1027)\n            ds.repo.add(tostage, commit=False)\n        else:\n            # --update will ignore any untracked files, sadly git-annex add\n            # above does not\n            # will complain about vanished files though, filter them here, but\n            # keep them for a later commit call\n            ds.repo.add(tostage, git_options=['--update'], commit=False)\n\n    _datalad_msg = False\n    if not message:\n        message = 'Recorded existing changes'\n        _datalad_msg = True\n\n    if files or ds.repo.repo.is_dirty(\n            index=True,\n            working_tree=False,\n            untracked_files=False,\n            submodules=True):\n        # either we have an explicit list of files, or we have something\n        # stages otherwise do not attempt to commit, as the underlying\n        # repo will happily commit any non-change\n        # not checking the working tree or untracked files should make this\n        # relavtively cheap\n\n        # TODO: commit() should rather report a dedicated ValueError\n        # waiting for #1170\n        from datalad.support.exceptions import CommandError\n        try:\n            # we will blindly call commit not knowing if there is anything to\n            # commit -- this is cheaper than to anticipate all possible ways\n            # a repo in whatever mode is dirty\n            # however, if nothing is dirty the whining wil start\n            # --> sucking it up right here\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                ds.repo.commit(message, options=files, _datalad_msg=_datalad_msg)\n        except CommandError as e:\n            # TODO until #1171 is resolved, test here for \"normal\" failure\n            # to commit\n            if 'nothing to commit' in str(e):\n                lgr.debug(\n                    \"Was instructed to commit %s files but repository is not dirty\",\n                    files)\n            elif 'no changes added to commit' in str(e) or 'nothing added to commit' in str(e):\n                lgr.info(\n                    'Nothing to save')\n            else:\n                # relay any prior whining in the exception\n                raise ValueError('{} [error log follows] {}; {}'.format(\n                    e, e.stdout, e.stderr))\n\n    # MIH: let's tag even if there was nothing commit. I'd forget this\n    # option too often...\n    if version_tag:\n        ds.repo.tag(version_tag)\n\n    _was_modified = ds.repo.get_hexsha() != orig_hexsha\n\n    return ds.repo.repo.head.commit if _was_modified else None\n\n\ndef amend_pathspec_with_superdatasets(spec, topmost=True, limit_single=False):\n    \"\"\"Amend a path spec dictionary with entries for superdatasets\n\n    The result will be a superdataset entry (if a superdataset exists)\n    for each input dataset. This entry will (at least) contain the path\n    to the subdataset.\n\n    Parameters\n    ----------\n    spec : dict\n      Path spec\n    topmost : Dataset or bool\n      Flag whether to grab the immediate, or the top-most superdataset\n      for each entry, alternatively this can be a dataset instance\n      that is used as the topmost dataset.\n    limit_single : bool\n      If a `topmost` dataset is provided, and this flag is True, only\n      the given topmost dataset will be considered as superdataset. Any\n      datasets in the spec that are not underneath this dataset will\n      not have associated superdataset entries added to the spec.\n\n    Returns\n    -------\n    dict\n      Amended path spec dictionary\n    \"\"\"\n    superdss = {}\n    for dpath in spec.keys():\n        superds = None\n        if isinstance(topmost, Dataset):\n            if limit_single and dpath == topmost.path:\n                # this is already the topmost, no further superdataset to\n                # consider\n                continue\n            if dpath.startswith(_with_sep(topmost.path)):\n                # the given topmost dataset is \"above\" the current\n                # datasets path\n                superds = topmost\n            elif limit_single:\n                continue\n        if not superds:\n            # grab the (topmost) superdataset\n            superds = Dataset(dpath).get_superdataset(\n                datalad_only=True, topmost=topmost)\n        if not superds:\n            continue\n        # register the subdatasets path in the spec of the superds\n        spaths = superdss.get(superds.path, [])\n        if not spaths:\n            spaths = spec.get(superds.path, [])\n        spaths.append(dpath)\n        superdss[superds.path] = spaths\n    spec.update(superdss)\n    return spec\n\n\ndef get_paths_by_dataset(paths, recursive=False, recursion_limit=None,\n                         out=None, dir_lookup=None):\n    \"\"\"Sort a list of paths per dataset they are contained in.\n\n    Any paths that are not part of a dataset, or presently unavailable are\n    reported.\n\n    Parameter\n    ---------\n    paths : sequence\n      A sequence of path specifications to sort.\n    recursive : bool\n      Flag whether to report subdatasets under any of the given paths\n    recursion_limit :\n      Depth constraint for recursion. See `Dataset.get_subdatasets()` for more\n      information.\n    out : dict or None\n      By default a new output dictionary is created, however an existing one\n      can be provided via this argument to enable incremental processing.\n    dir_lookup : dict or None\n      Optional lookup cache that maps paths to previously determined datasets.\n      This can speed up repeated processing.\n\n    Returns\n    -------\n    Tuple(dict, list, list)\n      Dict of `existing dataset path`: `path` mappings, the list of currently\n      non-existing paths (possibly matching currently uninstalled datasets),\n      and any paths that are not part of any dataset.\n    \"\"\"\n    # sort paths into the respective datasets\n    if dir_lookup is None:\n        dir_lookup = {}\n    if out is None:\n        out = {}\n    # paths that don't exist (yet)\n    unavailable_paths = []\n    nondataset_paths = []\n    for path in paths:\n        if not lexists(path):\n            # not there yet, impossible to say which ds it will actually\n            # be in, if any\n            unavailable_paths.append(path)\n            continue\n        # the path exists in some shape or form\n        if isdir(path):\n            # this could contain all types of additional content\n            d = path\n        else:\n            # for everything else we are interested in the container\n            d = dirname(path)\n            if not d:\n                d = curdir\n        # this could be `None` if there is no git repo\n        dspath = dir_lookup.get(d, get_dataset_root(d))\n        dir_lookup[d] = dspath\n        if not dspath:\n            nondataset_paths.append(path)\n            continue\n        if isdir(path):\n            ds = Dataset(dspath)\n            # we need to doublecheck that this is not a subdataset mount\n            # point, in which case get_toppath() would point to the parent\n            smpath = ds.get_containing_subdataset(\n                path, recursion_limit=1).path\n            if smpath != dspath:\n                # fix entry\n                dir_lookup[d] = smpath\n                # submodule still needs to be obtained\n                unavailable_paths.append(path)\n                continue\n            if recursive:\n                # make sure we get everything relevant in all _checked out_\n                # subdatasets, obtaining of previously unavailable subdataset\n                # else done elsewhere\n                subs = ds.get_subdatasets(fulfilled=True,\n                                          recursive=recursive,\n                                          recursion_limit=recursion_limit)\n                for sub in subs:\n                    subdspath = opj(dspath, sub)\n                    if subdspath.startswith(_with_sep(path)):\n                        # this subdatasets is underneath the search path\n                        # we want it all\n                        # be careful to not overwrite anything, in case\n                        # this subdataset has been processed before\n                        out[subdspath] = out.get(\n                            subdspath,\n                            [subdspath])\n        out[dspath] = out.get(dspath, []) + [path]\n    return out, unavailable_paths, nondataset_paths\n\n\ndef get_normalized_path_arguments(paths, dataset=None, default=None):\n    \"\"\"Apply standard resolution to path arguments\n\n    This is nothing more than a helper to standardize path argument\n    preprocessing.\n\n    Parameter\n    ---------\n    paths : sequence or single path\n      Path(s) to normalize\n    dataset : path or Dataset or None\n      Optional dataset identifying something against which to resolve input\n      path arguments\n    default: sequence of paths or single path or None\n      If `paths` is empty, use this instead\n\n    Returns\n    -------\n    tuple(list(paths), path)\n      Normalized paths and path to a potential dataset against which paths were\n      resolved.\n    \"\"\"\n    dataset_path = dataset.path if isinstance(dataset, Dataset) else dataset\n    if not paths and default:\n        paths = default\n    paths = assure_list(paths)\n    # resolve path(s):\n    resolved_paths = [resolve_path(p, dataset) for p in paths]\n    if dataset:\n        # guarantee absolute paths\n        resolved_paths = [opj(dataset_path, p) for p in resolved_paths]\n    lgr.debug('Resolved input path arguments: %s', resolved_paths)\n    return resolved_paths, dataset_path\n\n\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        pwd = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    for p in values:\n        rpath = relpath(p, start=pwd)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef get_dataset_directories(top, ignore_datalad=True):\n    \"\"\"Return a list of directories in the same dataset under a given path\n\n    Parameters\n    ----------\n    top : path\n      Top-level path\n    ignore_datalad : bool\n      Whether to exlcude the '.datalad' directory of a dataset and its content\n      from the results.\n\n    Returns\n    -------\n    list\n      List of directories matching the top-level path, regardless of whether\n      these directories are known to Git (i.e. contain tracked files). The\n      list does not include the top-level path itself, but it does include\n      any subdataset mount points (regardless of whether the particular\n      subdatasets are installed or not).\n    \"\"\"\n    def func(arg, top, names):\n        refpath, ignore, dirs = arg\n        legit_names = []\n        for n in names:\n            path = opj(top, n)\n            if not isdir(path) or path in ignore:\n                pass\n            elif path != refpath and GitRepo.is_valid_repo(path):\n                # mount point, keep but don't dive into\n                dirs.append(path)\n            else:\n                legit_names.append(n)\n                dirs.append(path)\n        names[:] = legit_names\n\n    # collects the directories\n    refpath = get_dataset_root(top)\n    if not refpath:\n        raise ValueError(\"`top` path {} is not in a dataset\".format(top))\n    ignore = [opj(refpath, get_git_dir(refpath))]\n    if ignore_datalad:\n        ignore.append(opj(refpath, '.datalad'))\n    d = []\n    walk(top, func, (refpath, ignore, d))\n    return d\n\n\n# XXX the following present a different approach to\n# amend_pathspec_with_superdatasets() for discovering datasets between\n# processed ones and a base\n# let it simmer for a while and RF to use one or the other\n# this one here seems more leightweight and less convoluted\ndef _discover_trace_to_known(path, trace, spec):\n    # this beast walks the directory tree from a given `path` until\n    # it discoveres a known dataset (i.e. recorded in the spec)\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(path)\n    if valid_repo:\n        trace = trace + [path]\n        if path in spec:\n            # found a known repo, commit the trace\n            for i, p in enumerate(trace[:-1]):\n                spec[p] = list(set(spec.get(p, []) + [trace[i + 1]]))\n            # this edge is not done, we need to try to reach any downstream\n            # dataset\n    for p in listdir(path):\n        if valid_repo and p == '.git':\n            # ignore gitdir to steed things up\n            continue\n        p = opj(path, p)\n        if not isdir(p):\n            continue\n        _discover_trace_to_known(p, trace, spec)\n\n\ndef filter_unmodified(content_by_ds, refds, since):\n    \"\"\"Filter per-dataset path specifications based on modification history.\n\n    This function takes a path specification dictionary, as produced by\n    `Interface._prep()` and filters it such that only that subset of paths\n    remains in the dictionary that corresponding to the set of changes in\n    the given reference dataset since a given state.\n\n    The change set is traced across all related subdatasets, i.e. if a submodule\n    in the reference dataset is reported as modified then all paths for any given\n    subdataset in the modified one are tested for changes too (based on the\n    state at which the parent dataset reports a change in the subdataset), and so\n    on.\n\n    In doing so, not only unmodified given paths are removed, but also modified\n    given paths are replaced by the set of actually modified paths within them.\n\n    Only committed changes are considered!\n\n    Parameters\n    ----------\n    content_by_ds : dict\n      Per-dataset path specifications, as produced ,for example, by\n      `Interface._prep()`\n    refds : Dataset or *Repo or path\n      Reference dataset for which to determine the initial change set\n    since : state\n      Any commit-ish/tree-ish supported by Git (tag, commit, branch, ...).\n      Changes between this given state and the most recent commit are\n      evaluated.\n\n    Returns\n    -------\n    dict\n      Filtered path spec dictionary. If `since` is not None, the output is\n      guaranteed to only contain paths to modified, and presently existing\n      components of subdatasets of the given reference dataset (and itself).\n    \"\"\"\n    if since is None:\n        # we want all, subds not matching the ref are assumed to have been\n        # sorted out before (e.g. one level up)\n        return content_by_ds\n    # turn refds argument into a usable repo instance\n    if not hasattr(refds, 'path'):\n        # not a Repo or Dataset\n        refds_path = refds\n        refds = GitRepo(refds, create=False)\n    else:\n        refds_path = refds.path\n    repo = refds.repo\n    if hasattr(repo, 'repo'):\n        # TODO use GitRepo.diff() when available (gh-1217)\n        repo = repo.repo\n\n    # life is simple: we diff the base dataset, and kill anything that\n    # does not start with something that is in the diff\n    # we cannot really limit the diff paths easily because we might get\n    # or miss content (e.g. subdatasets) if we don't figure out which ones\n    # are known -- and we don't want that\n    diff = repo.commit().diff(since)\n    # get all modified paths (with original? commit) that are still\n    # present\n    modified = dict((opj(refds_path, d.b_path),\n                    d.b_blob.hexsha if d.b_blob else None)\n                    for d in diff)\n    if not modified:\n        # nothing modified nothing to report\n        return {}\n    # determine the subset that is a directory and hence is relevant for possible\n    # subdatasets\n    modified_dirs = {_with_sep(d) for d in modified if isdir(d)}\n    # find the subdatasets matching modified paths, this will also kick out\n    # any paths that are not in the dataset sub-hierarchy\n    mod_subs = {candds: paths\n                for candds, paths in content_by_ds.items()\n                if candds != refds_path and\n                any(_with_sep(candds).startswith(md) for md in modified_dirs)}\n    # now query the next level down\n    keep_subs = \\\n        [filter_unmodified(mod_subs, subds_path, modified[subds_path])\n         for subds_path in mod_subs\n         if subds_path in modified]\n    # merge result list into a single dict\n    keep = {k: v for d in keep_subs for k, v in d.items()}\n\n    paths_refds = content_by_ds[refds_path]\n    keep[refds_path] = [m for m in modified\n                        if lexists(m)  # still around\n                        and (m in paths_refds  # listed file, or subds\n                        # or a modified path under a given directory\n                        or any(m.startswith(_with_sep(p)) for p in paths_refds))]\n    return keep\n\n\nimport wrapt\n\ndef eval_results(switch):\n    \"\"\"Decorator providing functionality to evaluate return values of datalad\n    commands\n    \"\"\"\n    from inspect import isgenerator\n\n    @wrapt.decorator\n    def new_func(wrapped, instance, args, kwargs):\n        # rudimentary wrapper to harvest generators\n        if switch():\n            lgr.debug(\"switch ON\")\n        else:\n            lgr.debug(\"switch OFF\")\n        results = wrapped(*args, **kwargs)\n        if isgenerator(results):\n            return list(results)\n        else:\n            return results\n    return new_func\n\n\n\n",
            "header": {
                "file": {
                    "path": "datalad/interface/utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "6c7ebe930",
                "new_commit_hash": "80ec3063b",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 763,
                            "length": 17
                        },
                        "new_range": {
                            "start": 763,
                            "length": 28
                        },
                        "section": "def filter_unmodified(content_by_ds, refds, since):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                        or any(m.startswith(_with_sep(p)) for p in paths_refds))]",
                                "type": "context"
                            },
                            {
                                "content": "    return keep",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "import wrapt",
                                "type": "context"
                            },
                            {
                                "content": "@wrapt.decorator",
                                "type": "deleted"
                            },
                            {
                                "content": "def eval_results(wrapped, instance, args, kwargs):",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def eval_results(switch):",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Decorator providing functionality to evaluate return values of datalad",
                                "type": "context"
                            },
                            {
                                "content": "    commands",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    from inspect import isgenerator",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # rudimentary wrapper to harvest generators",
                                "type": "deleted"
                            },
                            {
                                "content": "    results = wrapped(*args, **kwargs)",
                                "type": "deleted"
                            },
                            {
                                "content": "    if isgenerator(results):",
                                "type": "deleted"
                            },
                            {
                                "content": "        return list(results)",
                                "type": "deleted"
                            },
                            {
                                "content": "    else:",
                                "type": "deleted"
                            },
                            {
                                "content": "        return results",
                                "type": "deleted"
                            },
                            {
                                "content": "    @wrapt.decorator",
                                "type": "added"
                            },
                            {
                                "content": "    def new_func(wrapped, instance, args, kwargs):",
                                "type": "added"
                            },
                            {
                                "content": "        # rudimentary wrapper to harvest generators",
                                "type": "added"
                            },
                            {
                                "content": "        if switch():",
                                "type": "added"
                            },
                            {
                                "content": "            lgr.debug(\"switch ON\")",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            lgr.debug(\"switch OFF\")",
                                "type": "added"
                            },
                            {
                                "content": "        results = wrapped(*args, **kwargs)",
                                "type": "added"
                            },
                            {
                                "content": "        if isgenerator(results):",
                                "type": "added"
                            },
                            {
                                "content": "            return list(results)",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            return results",
                                "type": "added"
                            },
                            {
                                "content": "    return new_func",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "eval_results",
                            "content": "def eval_results(wrapped, instance, args, kwargs):\n    \"\"\"Decorator providing functionality to evaluate return values of datalad\n    commands\n    \"\"\"\n    from inspect import isgenerator\n\n    # rudimentary wrapper to harvest generators\n    results = wrapped(*args, **kwargs)\n    if isgenerator(results):\n        return list(results)\n    else:\n        return results",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 768,
                            "end_lineno": 779,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "e89b37688dbaf36ab4376871071004df90b060cd^",
    "new_commit_hash": "e89b37688dbaf36ab4376871071004df90b060cd",
    "commit_message": "ENH: Proposing a mechanism for a switch for eval_results, that can be accessed from within eval_results without polluting the signature of the decorated command function",
    "commit_date": "2017-03-02T12:12:05+01:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "e89b37688dbaf36ab4376871071004df90b060cd",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test dirty dataset handling\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os\nfrom os.path import join as opj\nfrom os.path import relpath\nfrom nose.tools import assert_raises, assert_equal\nfrom datalad.tests.utils import with_tempfile, assert_not_equal\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import create_tree\nfrom datalad.tests.utils import ok_clean_git\nfrom datalad.tests.utils import ok_\nfrom datalad.interface.utils import handle_dirty_dataset\nfrom datalad.interface.utils import get_paths_by_dataset\nfrom datalad.interface.utils import save_dataset_hierarchy\nfrom datalad.interface.utils import get_dataset_directories\nfrom datalad.interface.utils import filter_unmodified\nfrom datalad.interface.save import Save\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.utils import _install_subds_inplace\nfrom datalad.api import save\n\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: staged\n    with open(opj(ds.path, 'staged'), 'w') as f:\n        f.write('some')\n    ds.repo.add('staged', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    ok_clean_git(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.get_subdatasets(), ['subds'])\n\n\n@with_tempfile(mkdir=True)\ndef test_paths_by_dataset(path):\n    ds = Dataset(path).create()\n    subds = ds.create('one')\n    subsubds = subds.create('two')\n    d, ua, ne = get_paths_by_dataset([path])\n    for t in (ua, ne):\n        assert_equal(t, [])\n    assert_equal(d, {ds.path: [ds.path]})\n\n    d, ua, ne = get_paths_by_dataset(\n        [path], recursive=True)\n    for t in (ua, ne):\n        assert_equal(t, [])\n\n    os.makedirs(opj(ds.path, 'one', 'some'))\n    hidden = subds.create(opj('some', 'deep'))\n    testpath = opj(subds.path, 'some')\n    d, ua, ne = get_paths_by_dataset([testpath], recursive=True)\n    for t in (ua, ne):\n        assert_equal(t, [])\n    # must contain the containing dataset, and the testpath exactly\n    assert_equal(d[subds.path], [testpath])\n    # and also the subdataset underneath\n    assert_equal(d[hidden.path], [hidden.path])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree):\n    created_ds = []\n    for node, items in tree.items():\n        node_path = opj(path, node)\n        if isinstance(items, dict):\n            ds = make_demo_hierarchy_datasets(node_path, items)\n            created_ds.append(ds)\n    topds = Dataset(path)\n    if not topds.is_installed():\n        topds.create(force=True)\n        # TODO this farce would not be necessary if add() could add subdatasets\n        for ds in created_ds:\n            _install_subds_inplace(ds=topds, path=ds.path, relativepath=relpath(ds.path, topds.path))\n            ds.save()\n        return topds\n\n\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.add('.', recursive=True)\n    ok_clean_git(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    ds_bb.save(files=ds_bbaa.path, super_datasets=True)\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        ok_clean_git(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    save_dataset_hierarchy((d.path, da.path, db.path))\n    for d in (d, da, db):\n        ok_clean_git(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(files=(aa.path, ba.path, bb.path, c.path, ca.path, d.path),\n            super_datasets=True)\n    ok_clean_git(ds.path)\n\n\n@with_tempfile(mkdir=True)\ndef test_get_dataset_directories(path):\n    assert_raises(ValueError, get_dataset_directories, path)\n    ds = Dataset(path).create()\n    # ignores .git always and .datalad by default\n    assert_equal(get_dataset_directories(path), [])\n    assert_equal(get_dataset_directories(path, ignore_datalad=False),\n                 [opj(path, '.datalad')])\n    # find any directory, not just those known to git\n    testdir = opj(path, 'newdir')\n    os.makedirs(testdir)\n    assert_equal(get_dataset_directories(path), [testdir])\n    # do not find files\n    with open(opj(path, 'somefile'), 'w') as f:\n        f.write('some')\n    assert_equal(get_dataset_directories(path), [testdir])\n    # find more than one directory\n    testdir2 = opj(path, 'newdir2')\n    os.makedirs(testdir2)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2]))\n    # find subdataset mount points\n    subdsdir = opj(path, 'sub')\n    subds = ds.create(subdsdir)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    # do not find content within subdataset dirs\n    os.makedirs(opj(path, 'sub', 'deep'))\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    subsubdsdir = opj(subdsdir, 'subsub')\n    subds.create(subsubdsdir)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, subdsdir]))\n    # find nested directories\n    testdir3 = opj(testdir2, 'newdir21')\n    os.makedirs(testdir3)\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, testdir3, subdsdir]))\n    # only return hits below the search path\n    assert_equal(sorted(get_dataset_directories(testdir2)), sorted([testdir3]))\n    # empty subdataset mount points are reported too\n    ds.uninstall(subds.path, check=False, recursive=True)\n    ok_(not subds.is_installed())\n    ok_(os.path.exists(subds.path))\n    assert_equal(sorted(get_dataset_directories(path)), sorted([testdir, testdir2, testdir3, subdsdir]))\n\n\ndef test_interface_prep():\n    # verify sanity if nothing was given, as it would look like from the\n    # cmdline\n    assert_equal(Save._prep(path=[], dataset=None), ({}, []))\n\n\n@with_tree(demo_hierarchy['b'])\ndef test_filter_unmodified(path):\n    ds = Dataset(path).create(force=True)\n    suba = ds.create('ba', force=True)\n    subb = ds.create('bb', force=True)\n    subsub = ds.create(opj('bb', 'bba', 'bbaa'), force=True)\n    ds.add('.', recursive=True)\n    ok_clean_git(path)\n\n    spec, unavail = Save._prep('.', ds, recursive=True)\n    # just to be sure -- find all datasets, and just datasets\n    assert_equal(len(spec), 4)\n    for r, p in spec.items():\n        assert_equal([r], p)\n\n    orig_base_commit = ds.repo.repo.commit()\n    # nothing was modified compared to the status quo, output must be empty\n    assert_equal({}, filter_unmodified(spec, ds, orig_base_commit))\n\n    # modify one subdataset\n    added_path = opj(subb.path, 'added')\n    create_tree(subb.path, {'added': 'test'})\n    subb.add('added')\n\n    # still nothing was modified compared to orig commit, because the base\n    # dataset does not have the recent change saved\n    assert_equal({}, filter_unmodified(spec, ds, orig_base_commit))\n\n    ds.save(all_changes=True)\n\n    modspec, unavail = Save._prep('.', ds, recursive=True)\n    # arg sorting is not affected\n    assert_equal(spec, modspec)\n\n    # only the actually modified components per dataset are kept\n    assert_equal(\n        {\n            ds.path: [subb.path],\n            subb.path: [added_path]\n        },\n        filter_unmodified(spec, ds, orig_base_commit))\n\n    # deal with removal (force insufiicient copies error)\n    ds.remove(opj(subsub.path, 'file_bbaa'), check=False)\n    # saves all the way up\n    ok_clean_git(path)\n\n    modspec, unavail = Save._prep('.', ds, recursive=True)\n    # arg sorting is again not affected\n    assert_equal(spec, modspec)\n    # only the actually modified components per dataset are kept\n    assert_equal(\n        {\n            ds.path: [subb.path],\n            subb.path: [added_path, subsub.path],\n            subsub.path: []\n        },\n        {d: sorted(p) for d, p in filter_unmodified(spec, ds, orig_base_commit).items()})\n\n\ndef test_eval_results():\n\n    from datalad.interface.base import Interface\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from inspect import getargspec\n\n    class Switcher(object):\n\n        def __init__(self, value):\n            self._value = value\n\n        def __call__(self, value=None):\n            if value is not None:\n                self._value = value\n            else:\n                return self._value\n\n    class FakeCommand(Interface):\n\n        switch = Switcher(False)\n        \n        @staticmethod\n        @datasetmethod(name='fake_command')\n        @eval_results(switch)\n        def __call__(number, dataset=None):\n\n            for i in range(number):\n                yield i\n\n    # test results:\n    result = FakeCommand().__call__(2)\n    assert_equal(result, [0, 1])\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(result, [0, 1, 2])\n    # test signature:\n    assert_equal(getargspec(Dataset.fake_command)[0], ['number', 'dataset'])\n    assert_equal(getargspec(FakeCommand.__call__)[0], ['number', 'dataset'])\n    # test switch:\n    from datalad.utils import swallow_logs\n    import logging\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        FakeCommand().__call__(2)\n        cml.assert_logged(\"switch OFF\", level='DEBUG')\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        FakeCommand.switch(True)\n        FakeCommand().__call__(2)\n        cml.assert_logged(\"switch ON\", level='DEBUG')\n\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.4.1.dev570\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 124ms\nInstalled 25 packages in 4.90s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpNT7MFH/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpNT7MFH/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpNT7MFH/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpNT7MFH/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpNT7MFH/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptoa8pu/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptoa8pu/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptoa8pu/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptoa8pu/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptoa8pu/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_e89b37688dbaf36ab4376871071004df90b060cd/datalad/__init__.py\", line 17, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_e89b37688dbaf36ab4376871071004df90b060cd/datalad/cmd.py\", line 32, in <module>\n    from .utils import on_windows\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_e89b37688dbaf36ab4376871071004df90b060cd/datalad/utils.py\", line 24, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 300ms\nInstalled 26 packages in 3.01s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 77 packages in 1.28s\nPrepared 1 package in 845ms\nInstalled 71 packages in 44ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + cachetools==5.5.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + charset-normalizer==3.4.1\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.4.1.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_e89b37688dbaf36ab4376871071004df90b060cd)\n + deprecated==1.2.15\n + filelock==3.12.2\n + frozendict==2.4.6\n + gitdb==4.0.12\n + gitpython==3.1.44\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + iso8601==2.1.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack-python==0.5.6\n + multidict==6.0.5\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydispatcher==2.0.7\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pyld==2.0.4\n + pynacl==1.5.0\n + pyopenssl==24.3.0\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + requests-ftp==0.3.1\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + simplejson==3.19.3\n + smmap==5.0.2\n + soupsieve==2.4.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nResolved 100 packages in 150ms\nInstalled 20 packages in 67ms\n + alabaster==0.7.13\n + babel==2.14.0\n + docutils==0.19\n + imagesize==1.4.1\n + jinja2==3.1.5\n + line-profiler==4.1.3\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + pygments==2.17.2\n + pypandoc==1.14\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_e89b37688dbaf36ab4376871071004df90b060cd\nplugins: cov-4.1.0\ncollected 7 items\n\nr2e_tests/test_1.py FFFF.F.                                              [100%]\n\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexocluw5e0'}\nfilename = '/tmp/datalad_temp_test_dirty1464c5ao'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:564: TypeError\n____________________________ test_paths_by_dataset _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex6pnsgwqf'}\nfilename = '/tmp/datalad_temp_test_paths_by_dataseto71gjw2v'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_paths_by_dataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:564: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexyvnrz4ry'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchyumo3jpfh'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:429: TypeError\n_________________________ test_get_dataset_directories _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexk678xpu6'}\nfilename = '/tmp/datalad_temp_test_get_dataset_directories5rsp9sym'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_dataset_directories() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:564: TypeError\n____________________________ test_filter_unmodified ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexerbg24h4'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_filter_unmodified'}\nd = '/tmp/datalad_temp_tree_test_filter_unmodifiedf_bu7hlp'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_filter_unmodified() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:429: TypeError\n==================================== PASSES ====================================\n______________________________ test_eval_results _______________________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.interface.utils:utils.py:781 switch OFF\nDEBUG    datalad.interface.utils:utils.py:779 switch ON\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_interface_prep\nPASSED r2e_tests/test_1.py::test_eval_results\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_paths_by_dataset - TypeError: test_paths_by_...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\nFAILED r2e_tests/test_1.py::test_get_dataset_directories - TypeError: test_ge...\nFAILED r2e_tests/test_1.py::test_filter_unmodified - TypeError: test_filter_u...\n========================= 5 failed, 2 passed in 0.96s ==========================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_e89b37688dbaf36ab4376871071004df90b060cd\nplugins: cov-4.1.0\ncollected 7 items\n\nr2e_tests/test_1.py FFFF.FF                                              [100%]\n\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex2___jzab'}\nfilename = '/tmp/datalad_temp_test_dirtyfmzx8shp'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:564: TypeError\n____________________________ test_paths_by_dataset _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexcmh9vmd2'}\nfilename = '/tmp/datalad_temp_test_paths_by_datasetb2z1dunj'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_paths_by_dataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:564: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexybypnfp_'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchyhlxr80c9'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:429: TypeError\n_________________________ test_get_dataset_directories _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexkajhuwvb'}\nfilename = '/tmp/datalad_temp_test_get_dataset_directoriesp5h10f8r'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_dataset_directories() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:564: TypeError\n____________________________ test_filter_unmodified ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexg1jn9d10'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_filter_unmodified'}\nd = '/tmp/datalad_temp_tree_test_filter_unmodifiedww5_sdy6'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_filter_unmodified() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:429: TypeError\n______________________________ test_eval_results _______________________________\n\n    def test_eval_results():\n    \n        from datalad.interface.base import Interface\n        from datalad.distribution.dataset import datasetmethod\n        from datalad.interface.utils import eval_results\n        from inspect import getargspec\n    \n        class Switcher(object):\n    \n            def __init__(self, value):\n                self._value = value\n    \n            def __call__(self, value=None):\n                if value is not None:\n                    self._value = value\n                else:\n                    return self._value\n    \n        class FakeCommand(Interface):\n    \n            switch = Switcher(False)\n    \n            @staticmethod\n            @datasetmethod(name='fake_command')\n            @eval_results(switch)\n            def __call__(number, dataset=None):\n    \n                for i in range(number):\n                    yield i\n    \n        # test results:\n>       result = FakeCommand().__call__(2)\nE       TypeError: 'NoneType' object is not callable\n\nr2e_tests/test_1.py:347: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_interface_prep\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_paths_by_dataset - TypeError: test_paths_by_...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\nFAILED r2e_tests/test_1.py::test_get_dataset_directories - TypeError: test_ge...\nFAILED r2e_tests/test_1.py::test_filter_unmodified - TypeError: test_filter_u...\nFAILED r2e_tests/test_1.py::test_eval_results - TypeError: 'NoneType' object ...\n========================= 6 failed, 1 passed in 0.61s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/interface/tests/test_utils.py datalad/interface/utils.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Applying `eval_results` Decorator Without Switch Causes TypeError

**Description:**
When using the `@eval_results` decorator without providing a switch mechanism, the decorated function becomes `None`. As a result, calling the decorated function raises a `TypeError`.

**Example Code:**
```python
from datalad.interface.utils import eval_results

@eval_results
def fake_command():
    yield 1

fake_command()
```

**Expected Behavior:**
The `fake_command` function should execute normally and return the expected results without any errors.

**Actual Behavior:**
Calling `fake_command()` results in the following error:
```
TypeError: 'NoneType' object is not callable
```
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/utils.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface utility functions

"""

__docformat__ = 'restructuredtext'

import logging
from os import curdir
from os import pardir
from os import listdir
from os.path import join as opj
from os.path import lexists
from os.path import isabs
from os.path import isdir
from os.path import dirname
from os.path import relpath
from os.path import sep
from os.path import split as psplit
from itertools import chain

# avoid import from API to not get into circular imports
from datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us
from datalad.utils import assure_list
from datalad.utils import get_trace
from datalad.utils import walk
from datalad.utils import get_dataset_root
from datalad.utils import swallow_logs
from datalad.utils import better_wraps
from datalad.support.gitrepo import GitRepo
from datalad.support.annexrepo import AnnexRepo
from datalad.distribution.dataset import Dataset
from datalad.distribution.dataset import resolve_path
from datalad.distribution.utils import _install_subds_inplace
from datalad.distribution.utils import get_git_dir


lgr = logging.getLogger('datalad.interface.utils')


def handle_dirty_dataset(ds, mode, msg=None):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    ds : Dataset or None
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    msg : str or None
      Custom message to use for a potential commit.

    Returns
    -------
    None
    """
    if ds is None:
        # nothing to be handled
        return
    if msg is None:
        msg = '[DATALAD] auto-saved changes'

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    if ds.repo:
        ds.repo.precommit()

    if mode == 'ignore':
        return
    elif mode == 'fail':
        if not ds.repo or ds.repo.repo.is_dirty(index=True,
                                                working_tree=True,
                                                untracked_files=True,
                                                submodules=True):
            raise RuntimeError('dataset {} has unsaved changes'.format(ds))
    elif mode == 'save-before':
        if not ds.is_installed():
            raise RuntimeError('dataset {} is not yet installed'.format(ds))
        from datalad.interface.save import Save
        Save.__call__(dataset=ds, message=msg, all_changes=True)
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def handle_dirty_datasets(dpaths,
                          mode,
                          base=None,
                          msg='[DATALAD] auto-saved changes'):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    dpaths : sequence(path)
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    base : path or None, optional
      Path of a common super dataset that should also be handled.
    msg : str
      Custom message to use for a potential saved state.

    Returns
    -------
    None
    """
    if mode == 'save-before':
        save_dataset_hierarchy(
            {d: [d] for d in dpaths},
            base=base,
            message=msg)
    elif mode == 'ignore':
        return
    elif mode == 'fail':
        for dpath in dpaths:
            ds = Dataset(dpath)
            if not ds.repo:
                continue
            ds.repo.precommit()
            if ds.repo.repo.is_dirty(index=True,
                                     working_tree=True,
                                     untracked_files=True,
                                     submodules=True):
                raise RuntimeError(
                    'dataset {} has unsaved changes'.format(ds))
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def get_tree_roots(paths):
    """Return common root paths for a set of paths

    This function determines the smallest set of common root
    paths and sorts all given paths under the respective
    root.

    Returns
    -------
    dict
      paths by root
    """
    paths_ws = [_with_sep(p) for p in paths]
    # sort all paths under their potential roots
    roots = {}
    # start from the top to get all paths down the line
    # and collate them into as few roots as possible
    for s in sorted(paths_ws):
        if any([s.startswith(r) for r in roots]):
            # this path is already covered by a known root
            continue
        # find all sub paths
        subs = [p for p in paths if p.startswith(s)]
        roots[s.rstrip(sep)] = subs
    return roots


def sort_paths_into_subdatasets(superds_path, target_subs, spec):
    # XXX forge a chain: whenever some path needs to be pushed down
    # put the receiving dataset as a components to process into the
    # respective superdataset -- this will enable further processing
    # of all datasets in a completely independent fashion
    # (except for order of processing)

    # get all existing subdataset as candidate nodes of the graph
    # that needs to be built and checked
    subds_graph = Dataset(superds_path).get_subdatasets(
        absolute=True, recursive=True, edges=True, fulfilled=True)
    if not subds_graph:
        # no subdatasets, nothing to sort
        return
    for t in target_subs:
        trace = get_trace(
            subds_graph,
            superds_path,
            t)
        if not trace:
            # not connected, or identical
            continue
        tosort = [superds_path] + trace + [t]
        # loop over all but the last one, simplifies logic below
        for i, d in enumerate(tosort[:-1]):
            paths = spec.get(d, [])
            keep_paths = []
            next_ds = tosort[i + 1]
            next_dspaths = spec.get(next_ds, [])
            comp = _with_sep(next_ds)
            for p in assure_list(paths):
                if p.startswith(comp):
                    next_dspaths.append(p)
                    # remember that we pushed the path into this dataset
                    keep_paths.append(next_ds)
                else:
                    keep_paths.append(p)
            spec[next_ds] = next_dspaths
            spec[d] = keep_paths
    # tidy up -- deduplicate
    for c in spec:
        spec[c] = list(set(spec[c]))


def save_dataset_hierarchy(
        info,
        base=None,
        message='[DATALAD] saved changes',
        version_tag=None):
    """Save (disjoint) hierarchies of datasets.

    Saving is done in an order that guarantees that all to be saved
    datasets reflect any possible change of any other to be saved
    subdataset, before they are saved themselves.

    Parameters
    ----------
    info : dict
      Absolute paths of datasets to be saved are the keys, and paths in each
      dataset to be saved are the values
    base : path or None, optional
      Common super dataset that should also be saved.
    message : str
      Message to be used for saving individual datasets

    Returns
    -------
    list
      Instances of saved datasets, in the order in which they where saved.
    """
    if not isinstance(info, dict):
        info = assure_list(info)
        info = dict(zip(info, [[i] for i in info]))
    dpaths = info.keys()
    if base:
        # just a convenience...
        dpaths = assure_list(dpaths)
        dpaths.append(base.path if isinstance(base, Dataset) else base)
    # sort all datasets under their potential superdatasets
    # start from the top to get all subdatasets down the line
    # and collate them into as few superdatasets as possible
    superdss = get_tree_roots(dpaths)
    # for each "superdataset" check the tree of subdatasets and make sure
    # we gather all datasets between the super and any subdataset
    # so we can save them all bottom-up in order to be able to properly
    # save the superdataset
    for superds_path in superdss:
        target_subs = superdss[superds_path]
        sort_paths_into_subdatasets(superds_path, target_subs, info)
    # iterate over all datasets, starting at the bottom
    saved = []
    for dpath in sorted(info.keys(), reverse=True):
        ds = Dataset(dpath)
        if ds.is_installed():
            saved_state = save_dataset(
                ds,
                info[dpath],
                message=message,
                version_tag=version_tag)
            if saved_state:
                saved.append(ds)
    return saved


def save_dataset(
        ds,
        paths=None,
        message=None,
        version_tag=None):
    """Save changes in a single dataset.

    Parameters
    ----------
    ds : Dataset
      The dataset to be saved.
    paths : list, optional
      Paths to dataset components to be saved.
    message: str, optional
      (Commit) message to be attached to the saved state.
    version_tag : str, optional
      Tag to be assigned to the saved state.

    Returns
    -------
    bool
      Whether a new state was saved. If all to be saved content was unmodified
      no new state will be saved.
    """
    # XXX paths must be in the given ds, no further sanity checks!

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    ds.repo.precommit()

    # track what is to be committed, so it becomes
    # possible to decide when/what to save further down
    # and one level up
    orig_hexsha = ds.repo.get_hexsha()

    # always yields list; empty if None
    files = list(
        set(
            [opj(ds.path, f) if not isabs(f) else f for f in assure_list(paths)]))

    # try to consider existing and changed files, and prevent untracked
    # files from being added
    # XXX not acting upon untracked files would be very expensive, because
    # I see no way to avoid using `add` below and git annex has no equivalent
    # to git add's --update -- so for now don't bother
    # XXX alternatively we could consider --no-ignore-removal to also
    # have it delete any already vanished files
    # asking yourself why we need to `add` at all? For example, freshly
    # unlocked files in a v5 repo are listed as "typechange" and commit
    # refuses to touch them without an explicit `add`
    tostage = [f for f in files if lexists(f)]
    if tostage:
        lgr.debug('staging files for commit: %s', tostage)
        if isinstance(ds.repo, AnnexRepo):
            # to make this work without calling `git add` in addition,
            # this needs git-annex v6.20161210 (see #1027)
            ds.repo.add(tostage, commit=False)
        else:
            # --update will ignore any untracked files, sadly git-annex add
            # above does not
            # will complain about vanished files though, filter them here, but
            # keep them for a later commit call
            ds.repo.add(tostage, git_options=['--update'], commit=False)

    _datalad_msg = False
    if not message:
        message = 'Recorded existing changes'
        _datalad_msg = True

    if files or ds.repo.repo.is_dirty(
            index=True,
            working_tree=False,
            untracked_files=False,
            submodules=True):
        # either we have an explicit list of files, or we have something
        # stages otherwise do not attempt to commit, as the underlying
        # repo will happily commit any non-change
        # not checking the working tree or untracked files should make this
        # relavtively cheap

        # TODO: commit() should rather report a dedicated ValueError
        # waiting for #1170
        from datalad.support.exceptions import CommandError
        try:
            # we will blindly call commit not knowing if there is anything to
            # commit -- this is cheaper than to anticipate all possible ways
            # a repo in whatever mode is dirty
            # however, if nothing is dirty the whining wil start
            # --> sucking it up right here
            with swallow_logs(new_level=logging.ERROR) as cml:
                ds.repo.commit(message, options=files, _datalad_msg=_datalad_msg)
        except CommandError as e:
            # TODO until #1171 is resolved, test here for "normal" failure
            # to commit
            if 'nothing to commit' in str(e):
                lgr.debug(
                    "Was instructed to commit %s files but repository is not dirty",
                    files)
            elif 'no changes added to commit' in str(e) or 'nothing added to commit' in str(e):
                lgr.info(
                    'Nothing to save')
            else:
                # relay any prior whining in the exception
                raise ValueError('{} [error log follows] {}; {}'.format(
                    e, e.stdout, e.stderr))

    # MIH: let's tag even if there was nothing commit. I'd forget this
    # option too often...
    if version_tag:
        ds.repo.tag(version_tag)

    _was_modified = ds.repo.get_hexsha() != orig_hexsha

    return ds.repo.repo.head.commit if _was_modified else None


def amend_pathspec_with_superdatasets(spec, topmost=True, limit_single=False):
    """Amend a path spec dictionary with entries for superdatasets

    The result will be a superdataset entry (if a superdataset exists)
    for each input dataset. This entry will (at least) contain the path
    to the subdataset.

    Parameters
    ----------
    spec : dict
      Path spec
    topmost : Dataset or bool
      Flag whether to grab the immediate, or the top-most superdataset
      for each entry, alternatively this can be a dataset instance
      that is used as the topmost dataset.
    limit_single : bool
      If a `topmost` dataset is provided, and this flag is True, only
      the given topmost dataset will be considered as superdataset. Any
      datasets in the spec that are not underneath this dataset will
      not have associated superdataset entries added to the spec.

    Returns
    -------
    dict
      Amended path spec dictionary
    """
    superdss = {}
    for dpath in spec.keys():
        superds = None
        if isinstance(topmost, Dataset):
            if limit_single and dpath == topmost.path:
                # this is already the topmost, no further superdataset to
                # consider
                continue
            if dpath.startswith(_with_sep(topmost.path)):
                # the given topmost dataset is "above" the current
                # datasets path
                superds = topmost
            elif limit_single:
                continue
        if not superds:
            # grab the (topmost) superdataset
            superds = Dataset(dpath).get_superdataset(
                datalad_only=True, topmost=topmost)
        if not superds:
            continue
        # register the subdatasets path in the spec of the superds
        spaths = superdss.get(superds.path, [])
        if not spaths:
            spaths = spec.get(superds.path, [])
        spaths.append(dpath)
        superdss[superds.path] = spaths
    spec.update(superdss)
    return spec


def get_paths_by_dataset(paths, recursive=False, recursion_limit=None,
                         out=None, dir_lookup=None):
    """Sort a list of paths per dataset they are contained in.

    Any paths that are not part of a dataset, or presently unavailable are
    reported.

    Parameter
    ---------
    paths : sequence
      A sequence of path specifications to sort.
    recursive : bool
      Flag whether to report subdatasets under any of the given paths
    recursion_limit :
      Depth constraint for recursion. See `Dataset.get_subdatasets()` for more
      information.
    out : dict or None
      By default a new output dictionary is created, however an existing one
      can be provided via this argument to enable incremental processing.
    dir_lookup : dict or None
      Optional lookup cache that maps paths to previously determined datasets.
      This can speed up repeated processing.

    Returns
    -------
    Tuple(dict, list, list)
      Dict of `existing dataset path`: `path` mappings, the list of currently
      non-existing paths (possibly matching currently uninstalled datasets),
      and any paths that are not part of any dataset.
    """
    # sort paths into the respective datasets
    if dir_lookup is None:
        dir_lookup = {}
    if out is None:
        out = {}
    # paths that don't exist (yet)
    unavailable_paths = []
    nondataset_paths = []
    for path in paths:
        if not lexists(path):
            # not there yet, impossible to say which ds it will actually
            # be in, if any
            unavailable_paths.append(path)
            continue
        # the path exists in some shape or form
        if isdir(path):
            # this could contain all types of additional content
            d = path
        else:
            # for everything else we are interested in the container
            d = dirname(path)
            if not d:
                d = curdir
        # this could be `None` if there is no git repo
        dspath = dir_lookup.get(d, get_dataset_root(d))
        dir_lookup[d] = dspath
        if not dspath:
            nondataset_paths.append(path)
            continue
        if isdir(path):
            ds = Dataset(dspath)
            # we need to doublecheck that this is not a subdataset mount
            # point, in which case get_toppath() would point to the parent
            smpath = ds.get_containing_subdataset(
                path, recursion_limit=1).path
            if smpath != dspath:
                # fix entry
                dir_lookup[d] = smpath
                # submodule still needs to be obtained
                unavailable_paths.append(path)
                continue
            if recursive:
                # make sure we get everything relevant in all _checked out_
                # subdatasets, obtaining of previously unavailable subdataset
                # else done elsewhere
                subs = ds.get_subdatasets(fulfilled=True,
                                          recursive=recursive,
                                          recursion_limit=recursion_limit)
                for sub in subs:
                    subdspath = opj(dspath, sub)
                    if subdspath.startswith(_with_sep(path)):
                        # this subdatasets is underneath the search path
                        # we want it all
                        # be careful to not overwrite anything, in case
                        # this subdataset has been processed before
                        out[subdspath] = out.get(
                            subdspath,
                            [subdspath])
        out[dspath] = out.get(dspath, []) + [path]
    return out, unavailable_paths, nondataset_paths


def get_normalized_path_arguments(paths, dataset=None, default=None):
    """Apply standard resolution to path arguments

    This is nothing more than a helper to standardize path argument
    preprocessing.

    Parameter
    ---------
    paths : sequence or single path
      Path(s) to normalize
    dataset : path or Dataset or None
      Optional dataset identifying something against which to resolve input
      path arguments
    default: sequence of paths or single path or None
      If `paths` is empty, use this instead

    Returns
    -------
    tuple(list(paths), path)
      Normalized paths and path to a potential dataset against which paths were
      resolved.
    """
    dataset_path = dataset.path if isinstance(dataset, Dataset) else dataset
    if not paths and default:
        paths = default
    paths = assure_list(paths)
    # resolve path(s):
    resolved_paths = [resolve_path(p, dataset) for p in paths]
    if dataset:
        # guarantee absolute paths
        resolved_paths = [opj(dataset_path, p) for p in resolved_paths]
    lgr.debug('Resolved input path arguments: %s', resolved_paths)
    return resolved_paths, dataset_path


def path_is_under(values, path=None):
    """Whether a given path is a subdirectory of any of the given test values

    Parameters
    ----------
    values : sequence or dict
      Paths to be tested against. This can be a dictionary in which case
      all values from all keys will be tested against.
    path : path or None
      Test path. If None is given, the process' working directory is
      used.

    Returns
    -------
    bool
    """
    if path is None:
        from datalad.utils import getpwd
        pwd = getpwd()
    if isinstance(values, dict):
        values = chain(*values.values())
    for p in values:
        rpath = relpath(p, start=pwd)
        if rpath == curdir \
                or rpath == pardir \
                or set(psplit(rpath)) == {pardir}:
            # first match is enough
            return True
    return False


def get_dataset_directories(top, ignore_datalad=True):
    """Return a list of directories in the same dataset under a given path

    Parameters
    ----------
    top : path
      Top-level path
    ignore_datalad : bool
      Whether to exlcude the '.datalad' directory of a dataset and its content
      from the results.

    Returns
    -------
    list
      List of directories matching the top-level path, regardless of whether
      these directories are known to Git (i.e. contain tracked files). The
      list does not include the top-level path itself, but it does include
      any subdataset mount points (regardless of whether the particular
      subdatasets are installed or not).
    """
    def func(arg, top, names):
        refpath, ignore, dirs = arg
        legit_names = []
        for n in names:
            path = opj(top, n)
            if not isdir(path) or path in ignore:
                pass
            elif path != refpath and GitRepo.is_valid_repo(path):
                # mount point, keep but don't dive into
                dirs.append(path)
            else:
                legit_names.append(n)
                dirs.append(path)
        names[:] = legit_names

    # collects the directories
    refpath = get_dataset_root(top)
    if not refpath:
        raise ValueError("`top` path {} is not in a dataset".format(top))
    ignore = [opj(refpath, get_git_dir(refpath))]
    if ignore_datalad:
        ignore.append(opj(refpath, '.datalad'))
    d = []
    walk(top, func, (refpath, ignore, d))
    return d


# XXX the following present a different approach to
# amend_pathspec_with_superdatasets() for discovering datasets between
# processed ones and a base
# let it simmer for a while and RF to use one or the other
# this one here seems more leightweight and less convoluted
def _discover_trace_to_known(path, trace, spec):
    # this beast walks the directory tree from a given `path` until
    # it discoveres a known dataset (i.e. recorded in the spec)
    # if it finds one, it commits any accummulated trace of visited
    # datasets on this edge to the spec
    valid_repo = GitRepo.is_valid_repo(path)
    if valid_repo:
        trace = trace + [path]
        if path in spec:
            # found a known repo, commit the trace
            for i, p in enumerate(trace[:-1]):
                spec[p] = list(set(spec.get(p, []) + [trace[i + 1]]))
            # this edge is not done, we need to try to reach any downstream
            # dataset
    for p in listdir(path):
        if valid_repo and p == '.git':
            # ignore gitdir to steed things up
            continue
        p = opj(path, p)
        if not isdir(p):
            continue
        _discover_trace_to_known(p, trace, spec)


def filter_unmodified(content_by_ds, refds, since):
    """Filter per-dataset path specifications based on modification history.

    This function takes a path specification dictionary, as produced by
    `Interface._prep()` and filters it such that only that subset of paths
    remains in the dictionary that corresponding to the set of changes in
    the given reference dataset since a given state.

    The change set is traced across all related subdatasets, i.e. if a submodule
    in the reference dataset is reported as modified then all paths for any given
    subdataset in the modified one are tested for changes too (based on the
    state at which the parent dataset reports a change in the subdataset), and so
    on.

    In doing so, not only unmodified given paths are removed, but also modified
    given paths are replaced by the set of actually modified paths within them.

    Only committed changes are considered!

    Parameters
    ----------
    content_by_ds : dict
      Per-dataset path specifications, as produced ,for example, by
      `Interface._prep()`
    refds : Dataset or *Repo or path
      Reference dataset for which to determine the initial change set
    since : state
      Any commit-ish/tree-ish supported by Git (tag, commit, branch, ...).
      Changes between this given state and the most recent commit are
      evaluated.

    Returns
    -------
    dict
      Filtered path spec dictionary. If `since` is not None, the output is
      guaranteed to only contain paths to modified, and presently existing
      components of subdatasets of the given reference dataset (and itself).
    """
    if since is None:
        # we want all, subds not matching the ref are assumed to have been
        # sorted out before (e.g. one level up)
        return content_by_ds
    # turn refds argument into a usable repo instance
    if not hasattr(refds, 'path'):
        # not a Repo or Dataset
        refds_path = refds
        refds = GitRepo(refds, create=False)
    else:
        refds_path = refds.path
    repo = refds.repo
    if hasattr(repo, 'repo'):
        # TODO use GitRepo.diff() when available (gh-1217)
        repo = repo.repo

    # life is simple: we diff the base dataset, and kill anything that
    # does not start with something that is in the diff
    # we cannot really limit the diff paths easily because we might get
    # or miss content (e.g. subdatasets) if we don't figure out which ones
    # are known -- and we don't want that
    diff = repo.commit().diff(since)
    # get all modified paths (with original? commit) that are still
    # present
    modified = dict((opj(refds_path, d.b_path),
                    d.b_blob.hexsha if d.b_blob else None)
                    for d in diff)
    if not modified:
        # nothing modified nothing to report
        return {}
    # determine the subset that is a directory and hence is relevant for possible
    # subdatasets
    modified_dirs = {_with_sep(d) for d in modified if isdir(d)}
    # find the subdatasets matching modified paths, this will also kick out
    # any paths that are not in the dataset sub-hierarchy
    mod_subs = {candds: paths
                for candds, paths in content_by_ds.items()
                if candds != refds_path and
                any(_with_sep(candds).startswith(md) for md in modified_dirs)}
    # now query the next level down
    keep_subs = \
        [filter_unmodified(mod_subs, subds_path, modified[subds_path])
         for subds_path in mod_subs
         if subds_path in modified]
    # merge result list into a single dict
    keep = {k: v for d in keep_subs for k, v in d.items()}

    paths_refds = content_by_ds[refds_path]
    keep[refds_path] = [m for m in modified
                        if lexists(m)  # still around
                        and (m in paths_refds  # listed file, or subds
                        # or a modified path under a given directory
                        or any(m.startswith(_with_sep(p)) for p in paths_refds))]
    return keep

import wrapt
@wrapt.decorator
def eval_results(wrapped, instance, args, kwargs):
    """Decorator providing functionality to evaluate return values of datalad
    commands
    """
    from inspect import isgenerator

    # rudimentary wrapper to harvest generators
    results = wrapped(*args, **kwargs)
    if isgenerator(results):
        return list(results)
    else:
        return results

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Applying `eval_results` Decorator Without Switch Causes TypeError

**Description:**
When using the `@eval_results` decorator without providing a switch mechanism, the decorated function becomes `None`. As a result, calling the decorated function raises a `TypeError`.

**Example Code:**
```python
from datalad.interface.utils import eval_results

@eval_results
def fake_command():
    yield 1

fake_command()
```

**Expected Behavior:**
The `fake_command` function should execute normally and return the expected results without any errors.

**Actual Behavior:**
Calling `fake_command()` results in the following error:
```
TypeError: 'NoneType' object is not callable
```
[/ISSUE]
{
    "test_interface_prep": "PASSED",
    "test_eval_results": "PASSED",
    "test_dirty": "FAILED",
    "test_paths_by_dataset": "FAILED",
    "test_save_hierarchy": "FAILED",
    "test_get_dataset_directories": "FAILED",
    "test_filter_unmodified": "FAILED"
}
diff --git a/datalad/interface/utils.py b/datalad/interface/utils.py
index 6c7ebe930..80ec3063b 100644
--- a/datalad/interface/utils.py
+++ b/datalad/interface/utils.py
@@ -763,17 +763,28 @@ def filter_unmodified(content_by_ds, refds, since):
                         or any(m.startswith(_with_sep(p)) for p in paths_refds))]
     return keep
 
+
 import wrapt
-@wrapt.decorator
-def eval_results(wrapped, instance, args, kwargs):
+
+def eval_results(switch):
     """Decorator providing functionality to evaluate return values of datalad
     commands
     """
     from inspect import isgenerator
 
-    # rudimentary wrapper to harvest generators
-    results = wrapped(*args, **kwargs)
-    if isgenerator(results):
-        return list(results)
-    else:
-        return results
+    @wrapt.decorator
+    def new_func(wrapped, instance, args, kwargs):
+        # rudimentary wrapper to harvest generators
+        if switch():
+            lgr.debug("switch ON")
+        else:
+            lgr.debug("switch OFF")
+        results = wrapped(*args, **kwargs)
+        if isgenerator(results):
+            return list(results)
+        else:
+            return results
+    return new_func
+
+
+
 

{"datalad/interface/utils.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nfrom os.path import join as opj\nfrom os.path import lexists\nfrom os.path import isabs\nfrom os.path import isdir\nfrom os.path import dirname\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import assure_list\nfrom datalad.utils import get_trace\nfrom datalad.utils import walk\nfrom datalad.utils import get_dataset_root\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import better_wraps\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.annexrepo import AnnexRepo\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.distribution.utils import _install_subds_inplace\nfrom datalad.distribution.utils import get_git_dir\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.repo.is_dirty(index=True,\n                                                working_tree=True,\n                                                untracked_files=True,\n                                                submodules=True):\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.interface.save import Save\n        Save.__call__(dataset=ds, message=msg, all_changes=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef handle_dirty_datasets(dpaths,\n                          mode,\n                          base=None,\n                          msg='[DATALAD] auto-saved changes'):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    dpaths : sequence(path)\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    base : path or None, optional\n      Path of a common super dataset that should also be handled.\n    msg : str\n      Custom message to use for a potential saved state.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if mode == 'save-before':\n        save_dataset_hierarchy(\n            {d: [d] for d in dpaths},\n            base=base,\n            message=msg)\n    elif mode == 'ignore':\n        return\n    elif mode == 'fail':\n        for dpath in dpaths:\n            ds = Dataset(dpath)\n            if not ds.repo:\n                continue\n            ds.repo.precommit()\n            if ds.repo.repo.is_dirty(index=True,\n                                     working_tree=True,\n                                     untracked_files=True,\n                                     submodules=True):\n                raise RuntimeError(\n                    'dataset {} has unsaved changes'.format(ds))\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\ndef sort_paths_into_subdatasets(superds_path, target_subs, spec):\n    # XXX forge a chain: whenever some path needs to be pushed down\n    # put the receiving dataset as a components to process into the\n    # respective superdataset -- this will enable further processing\n    # of all datasets in a completely independent fashion\n    # (except for order of processing)\n\n    # get all existing subdataset as candidate nodes of the graph\n    # that needs to be built and checked\n    subds_graph = Dataset(superds_path).get_subdatasets(\n        absolute=True, recursive=True, edges=True, fulfilled=True)\n    if not subds_graph:\n        # no subdatasets, nothing to sort\n        return\n    for t in target_subs:\n        trace = get_trace(\n            subds_graph,\n            superds_path,\n            t)\n        if not trace:\n            # not connected, or identical\n            continue\n        tosort = [superds_path] + trace + [t]\n        # loop over all but the last one, simplifies logic below\n        for i, d in enumerate(tosort[:-1]):\n            paths = spec.get(d, [])\n            keep_paths = []\n            next_ds = tosort[i + 1]\n            next_dspaths = spec.get(next_ds, [])\n            comp = _with_sep(next_ds)\n            for p in assure_list(paths):\n                if p.startswith(comp):\n                    next_dspaths.append(p)\n                    # remember that we pushed the path into this dataset\n                    keep_paths.append(next_ds)\n                else:\n                    keep_paths.append(p)\n            spec[next_ds] = next_dspaths\n            spec[d] = keep_paths\n    # tidy up -- deduplicate\n    for c in spec:\n        spec[c] = list(set(spec[c]))\n\n\ndef save_dataset_hierarchy(\n        info,\n        base=None,\n        message='[DATALAD] saved changes',\n        version_tag=None):\n    \"\"\"Save (disjoint) hierarchies of datasets.\n\n    Saving is done in an order that guarantees that all to be saved\n    datasets reflect any possible change of any other to be saved\n    subdataset, before they are saved themselves.\n\n    Parameters\n    ----------\n    info : dict\n      Absolute paths of datasets to be saved are the keys, and paths in each\n      dataset to be saved are the values\n    base : path or None, optional\n      Common super dataset that should also be saved.\n    message : str\n      Message to be used for saving individual datasets\n\n    Returns\n    -------\n    list\n      Instances of saved datasets, in the order in which they where saved.\n    \"\"\"\n    if not isinstance(info, dict):\n        info = assure_list(info)\n        info = dict(zip(info, [[i] for i in info]))\n    dpaths = info.keys()\n    if base:\n        # just a convenience...\n        dpaths = assure_list(dpaths)\n        dpaths.append(base.path if isinstance(base, Dataset) else base)\n    # sort all datasets under their potential superdatasets\n    # start from the top to get all subdatasets down the line\n    # and collate them into as few superdatasets as possible\n    superdss = get_tree_roots(dpaths)\n    # for each \"superdataset\" check the tree of subdatasets and make sure\n    # we gather all datasets between the super and any subdataset\n    # so we can save them all bottom-up in order to be able to properly\n    # save the superdataset\n    for superds_path in superdss:\n        target_subs = superdss[superds_path]\n        sort_paths_into_subdatasets(superds_path, target_subs, info)\n    # iterate over all datasets, starting at the bottom\n    saved = []\n    for dpath in sorted(info.keys(), reverse=True):\n        ds = Dataset(dpath)\n        if ds.is_installed():\n            saved_state = save_dataset(\n                ds,\n                info[dpath],\n                message=message,\n                version_tag=version_tag)\n            if saved_state:\n                saved.append(ds)\n    return saved\n\n\ndef save_dataset(\n        ds,\n        paths=None,\n        message=None,\n        version_tag=None):\n    \"\"\"Save changes in a single dataset.\n\n    Parameters\n    ----------\n    ds : Dataset\n      The dataset to be saved.\n    paths : list, optional\n      Paths to dataset components to be saved.\n    message: str, optional\n      (Commit) message to be attached to the saved state.\n    version_tag : str, optional\n      Tag to be assigned to the saved state.\n\n    Returns\n    -------\n    bool\n      Whether a new state was saved. If all to be saved content was unmodified\n      no new state will be saved.\n    \"\"\"\n    # XXX paths must be in the given ds, no further sanity checks!\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    ds.repo.precommit()\n\n    # track what is to be committed, so it becomes\n    # possible to decide when/what to save further down\n    # and one level up\n    orig_hexsha = ds.repo.get_hexsha()\n\n    # always yields list; empty if None\n    files = list(\n        set(\n            [opj(ds.path, f) if not isabs(f) else f for f in assure_list(paths)]))\n\n    # try to consider existing and changed files, and prevent untracked\n    # files from being added\n    # XXX not acting upon untracked files would be very expensive, because\n    # I see no way to avoid using `add` below and git annex has no equivalent\n    # to git add's --update -- so for now don't bother\n    # XXX alternatively we could consider --no-ignore-removal to also\n    # have it delete any already vanished files\n    # asking yourself why we need to `add` at all? For example, freshly\n    # unlocked files in a v5 repo are listed as \"typechange\" and commit\n    # refuses to touch them without an explicit `add`\n    tostage = [f for f in files if lexists(f)]\n    if tostage:\n        lgr.debug('staging files for commit: %s', tostage)\n        if isinstance(ds.repo, AnnexRepo):\n            # to make this work without calling `git add` in addition,\n            # this needs git-annex v6.20161210 (see #1027)\n            ds.repo.add(tostage, commit=False)\n        else:\n            # --update will ignore any untracked files, sadly git-annex add\n            # above does not\n            # will complain about vanished files though, filter them here, but\n            # keep them for a later commit call\n            ds.repo.add(tostage, git_options=['--update'], commit=False)\n\n    _datalad_msg = False\n    if not message:\n        message = 'Recorded existing changes'\n        _datalad_msg = True\n\n    if files or ds.repo.repo.is_dirty(\n            index=True,\n            working_tree=False,\n            untracked_files=False,\n            submodules=True):\n        # either we have an explicit list of files, or we have something\n        # stages otherwise do not attempt to commit, as the underlying\n        # repo will happily commit any non-change\n        # not checking the working tree or untracked files should make this\n        # relavtively cheap\n\n        # TODO: commit() should rather report a dedicated ValueError\n        # waiting for #1170\n        from datalad.support.exceptions import CommandError\n        try:\n            # we will blindly call commit not knowing if there is anything to\n            # commit -- this is cheaper than to anticipate all possible ways\n            # a repo in whatever mode is dirty\n            # however, if nothing is dirty the whining wil start\n            # --> sucking it up right here\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                ds.repo.commit(message, options=files, _datalad_msg=_datalad_msg)\n        except CommandError as e:\n            # TODO until #1171 is resolved, test here for \"normal\" failure\n            # to commit\n            if 'nothing to commit' in str(e):\n                lgr.debug(\n                    \"Was instructed to commit %s files but repository is not dirty\",\n                    files)\n            elif 'no changes added to commit' in str(e) or 'nothing added to commit' in str(e):\n                lgr.info(\n                    'Nothing to save')\n            else:\n                # relay any prior whining in the exception\n                raise ValueError('{} [error log follows] {}; {}'.format(\n                    e, e.stdout, e.stderr))\n\n    # MIH: let's tag even if there was nothing commit. I'd forget this\n    # option too often...\n    if version_tag:\n        ds.repo.tag(version_tag)\n\n    _was_modified = ds.repo.get_hexsha() != orig_hexsha\n\n    return ds.repo.repo.head.commit if _was_modified else None\n\n\ndef amend_pathspec_with_superdatasets(spec, topmost=True, limit_single=False):\n    \"\"\"Amend a path spec dictionary with entries for superdatasets\n\n    The result will be a superdataset entry (if a superdataset exists)\n    for each input dataset. This entry will (at least) contain the path\n    to the subdataset.\n\n    Parameters\n    ----------\n    spec : dict\n      Path spec\n    topmost : Dataset or bool\n      Flag whether to grab the immediate, or the top-most superdataset\n      for each entry, alternatively this can be a dataset instance\n      that is used as the topmost dataset.\n    limit_single : bool\n      If a `topmost` dataset is provided, and this flag is True, only\n      the given topmost dataset will be considered as superdataset. Any\n      datasets in the spec that are not underneath this dataset will\n      not have associated superdataset entries added to the spec.\n\n    Returns\n    -------\n    dict\n      Amended path spec dictionary\n    \"\"\"\n    superdss = {}\n    for dpath in spec.keys():\n        superds = None\n        if isinstance(topmost, Dataset):\n            if limit_single and dpath == topmost.path:\n                # this is already the topmost, no further superdataset to\n                # consider\n                continue\n            if dpath.startswith(_with_sep(topmost.path)):\n                # the given topmost dataset is \"above\" the current\n                # datasets path\n                superds = topmost\n            elif limit_single:\n                continue\n        if not superds:\n            # grab the (topmost) superdataset\n            superds = Dataset(dpath).get_superdataset(\n                datalad_only=True, topmost=topmost)\n        if not superds:\n            continue\n        # register the subdatasets path in the spec of the superds\n        spaths = superdss.get(superds.path, [])\n        if not spaths:\n            spaths = spec.get(superds.path, [])\n        spaths.append(dpath)\n        superdss[superds.path] = spaths\n    spec.update(superdss)\n    return spec\n\n\ndef get_paths_by_dataset(paths, recursive=False, recursion_limit=None,\n                         out=None, dir_lookup=None):\n    \"\"\"Sort a list of paths per dataset they are contained in.\n\n    Any paths that are not part of a dataset, or presently unavailable are\n    reported.\n\n    Parameter\n    ---------\n    paths : sequence\n      A sequence of path specifications to sort.\n    recursive : bool\n      Flag whether to report subdatasets under any of the given paths\n    recursion_limit :\n      Depth constraint for recursion. See `Dataset.get_subdatasets()` for more\n      information.\n    out : dict or None\n      By default a new output dictionary is created, however an existing one\n      can be provided via this argument to enable incremental processing.\n    dir_lookup : dict or None\n      Optional lookup cache that maps paths to previously determined datasets.\n      This can speed up repeated processing.\n\n    Returns\n    -------\n    Tuple(dict, list, list)\n      Dict of `existing dataset path`: `path` mappings, the list of currently\n      non-existing paths (possibly matching currently uninstalled datasets),\n      and any paths that are not part of any dataset.\n    \"\"\"\n    # sort paths into the respective datasets\n    if dir_lookup is None:\n        dir_lookup = {}\n    if out is None:\n        out = {}\n    # paths that don't exist (yet)\n    unavailable_paths = []\n    nondataset_paths = []\n    for path in paths:\n        if not lexists(path):\n            # not there yet, impossible to say which ds it will actually\n            # be in, if any\n            unavailable_paths.append(path)\n            continue\n        # the path exists in some shape or form\n        if isdir(path):\n            # this could contain all types of additional content\n            d = path\n        else:\n            # for everything else we are interested in the container\n            d = dirname(path)\n            if not d:\n                d = curdir\n        # this could be `None` if there is no git repo\n        dspath = dir_lookup.get(d, get_dataset_root(d))\n        dir_lookup[d] = dspath\n        if not dspath:\n            nondataset_paths.append(path)\n            continue\n        if isdir(path):\n            ds = Dataset(dspath)\n            # we need to doublecheck that this is not a subdataset mount\n            # point, in which case get_toppath() would point to the parent\n            smpath = ds.get_containing_subdataset(\n                path, recursion_limit=1).path\n            if smpath != dspath:\n                # fix entry\n                dir_lookup[d] = smpath\n                # submodule still needs to be obtained\n                unavailable_paths.append(path)\n                continue\n            if recursive:\n                # make sure we get everything relevant in all _checked out_\n                # subdatasets, obtaining of previously unavailable subdataset\n                # else done elsewhere\n                subs = ds.get_subdatasets(fulfilled=True,\n                                          recursive=recursive,\n                                          recursion_limit=recursion_limit)\n                for sub in subs:\n                    subdspath = opj(dspath, sub)\n                    if subdspath.startswith(_with_sep(path)):\n                        # this subdatasets is underneath the search path\n                        # we want it all\n                        # be careful to not overwrite anything, in case\n                        # this subdataset has been processed before\n                        out[subdspath] = out.get(\n                            subdspath,\n                            [subdspath])\n        out[dspath] = out.get(dspath, []) + [path]\n    return out, unavailable_paths, nondataset_paths\n\n\ndef get_normalized_path_arguments(paths, dataset=None, default=None):\n    \"\"\"Apply standard resolution to path arguments\n\n    This is nothing more than a helper to standardize path argument\n    preprocessing.\n\n    Parameter\n    ---------\n    paths : sequence or single path\n      Path(s) to normalize\n    dataset : path or Dataset or None\n      Optional dataset identifying something against which to resolve input\n      path arguments\n    default: sequence of paths or single path or None\n      If `paths` is empty, use this instead\n\n    Returns\n    -------\n    tuple(list(paths), path)\n      Normalized paths and path to a potential dataset against which paths were\n      resolved.\n    \"\"\"\n    dataset_path = dataset.path if isinstance(dataset, Dataset) else dataset\n    if not paths and default:\n        paths = default\n    paths = assure_list(paths)\n    # resolve path(s):\n    resolved_paths = [resolve_path(p, dataset) for p in paths]\n    if dataset:\n        # guarantee absolute paths\n        resolved_paths = [opj(dataset_path, p) for p in resolved_paths]\n    lgr.debug('Resolved input path arguments: %s', resolved_paths)\n    return resolved_paths, dataset_path\n\n\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        pwd = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    for p in values:\n        rpath = relpath(p, start=pwd)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef get_dataset_directories(top, ignore_datalad=True):\n    \"\"\"Return a list of directories in the same dataset under a given path\n\n    Parameters\n    ----------\n    top : path\n      Top-level path\n    ignore_datalad : bool\n      Whether to exlcude the '.datalad' directory of a dataset and its content\n      from the results.\n\n    Returns\n    -------\n    list\n      List of directories matching the top-level path, regardless of whether\n      these directories are known to Git (i.e. contain tracked files). The\n      list does not include the top-level path itself, but it does include\n      any subdataset mount points (regardless of whether the particular\n      subdatasets are installed or not).\n    \"\"\"\n    def func(arg, top, names):\n        refpath, ignore, dirs = arg\n        legit_names = []\n        for n in names:\n            path = opj(top, n)\n            if not isdir(path) or path in ignore:\n                pass\n            elif path != refpath and GitRepo.is_valid_repo(path):\n                # mount point, keep but don't dive into\n                dirs.append(path)\n            else:\n                legit_names.append(n)\n                dirs.append(path)\n        names[:] = legit_names\n\n    # collects the directories\n    refpath = get_dataset_root(top)\n    if not refpath:\n        raise ValueError(\"`top` path {} is not in a dataset\".format(top))\n    ignore = [opj(refpath, get_git_dir(refpath))]\n    if ignore_datalad:\n        ignore.append(opj(refpath, '.datalad'))\n    d = []\n    walk(top, func, (refpath, ignore, d))\n    return d\n\n\n# XXX the following present a different approach to\n# amend_pathspec_with_superdatasets() for discovering datasets between\n# processed ones and a base\n# let it simmer for a while and RF to use one or the other\n# this one here seems more leightweight and less convoluted\ndef _discover_trace_to_known(path, trace, spec):\n    # this beast walks the directory tree from a given `path` until\n    # it discoveres a known dataset (i.e. recorded in the spec)\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(path)\n    if valid_repo:\n        trace = trace + [path]\n        if path in spec:\n            # found a known repo, commit the trace\n            for i, p in enumerate(trace[:-1]):\n                spec[p] = list(set(spec.get(p, []) + [trace[i + 1]]))\n            # this edge is not done, we need to try to reach any downstream\n            # dataset\n    for p in listdir(path):\n        if valid_repo and p == '.git':\n            # ignore gitdir to steed things up\n            continue\n        p = opj(path, p)\n        if not isdir(p):\n            continue\n        _discover_trace_to_known(p, trace, spec)\n\n\ndef filter_unmodified(content_by_ds, refds, since):\n    \"\"\"Filter per-dataset path specifications based on modification history.\n\n    This function takes a path specification dictionary, as produced by\n    `Interface._prep()` and filters it such that only that subset of paths\n    remains in the dictionary that corresponding to the set of changes in\n    the given reference dataset since a given state.\n\n    The change set is traced across all related subdatasets, i.e. if a submodule\n    in the reference dataset is reported as modified then all paths for any given\n    subdataset in the modified one are tested for changes too (based on the\n    state at which the parent dataset reports a change in the subdataset), and so\n    on.\n\n    In doing so, not only unmodified given paths are removed, but also modified\n    given paths are replaced by the set of actually modified paths within them.\n\n    Only committed changes are considered!\n\n    Parameters\n    ----------\n    content_by_ds : dict\n      Per-dataset path specifications, as produced ,for example, by\n      `Interface._prep()`\n    refds : Dataset or *Repo or path\n      Reference dataset for which to determine the initial change set\n    since : state\n      Any commit-ish/tree-ish supported by Git (tag, commit, branch, ...).\n      Changes between this given state and the most recent commit are\n      evaluated.\n\n    Returns\n    -------\n    dict\n      Filtered path spec dictionary. If `since` is not None, the output is\n      guaranteed to only contain paths to modified, and presently existing\n      components of subdatasets of the given reference dataset (and itself).\n    \"\"\"\n    if since is None:\n        # we want all, subds not matching the ref are assumed to have been\n        # sorted out before (e.g. one level up)\n        return content_by_ds\n    # turn refds argument into a usable repo instance\n    if not hasattr(refds, 'path'):\n        # not a Repo or Dataset\n        refds_path = refds\n        refds = GitRepo(refds, create=False)\n    else:\n        refds_path = refds.path\n    repo = refds.repo\n    if hasattr(repo, 'repo'):\n        # TODO use GitRepo.diff() when available (gh-1217)\n        repo = repo.repo\n\n    # life is simple: we diff the base dataset, and kill anything that\n    # does not start with something that is in the diff\n    # we cannot really limit the diff paths easily because we might get\n    # or miss content (e.g. subdatasets) if we don't figure out which ones\n    # are known -- and we don't want that\n    diff = repo.commit().diff(since)\n    # get all modified paths (with original? commit) that are still\n    # present\n    modified = dict((opj(refds_path, d.b_path),\n                    d.b_blob.hexsha if d.b_blob else None)\n                    for d in diff)\n    if not modified:\n        # nothing modified nothing to report\n        return {}\n    # determine the subset that is a directory and hence is relevant for possible\n    # subdatasets\n    modified_dirs = {_with_sep(d) for d in modified if isdir(d)}\n    # find the subdatasets matching modified paths, this will also kick out\n    # any paths that are not in the dataset sub-hierarchy\n    mod_subs = {candds: paths\n                for candds, paths in content_by_ds.items()\n                if candds != refds_path and\n                any(_with_sep(candds).startswith(md) for md in modified_dirs)}\n    # now query the next level down\n    keep_subs = \\\n        [filter_unmodified(mod_subs, subds_path, modified[subds_path])\n         for subds_path in mod_subs\n         if subds_path in modified]\n    # merge result list into a single dict\n    keep = {k: v for d in keep_subs for k, v in d.items()}\n\n    paths_refds = content_by_ds[refds_path]\n    keep[refds_path] = [m for m in modified\n                        if lexists(m)  # still around\n                        and (m in paths_refds  # listed file, or subds\n                        # or a modified path under a given directory\n                        or any(m.startswith(_with_sep(p)) for p in paths_refds))]\n    return keep\n\nimport wrapt\n@wrapt.decorator\ndef eval_results(wrapped, instance, args, kwargs):\n    \"\"\"Decorator providing functionality to evaluate return values of datalad\n    commands\n    \"\"\"\n    from inspect import isgenerator\n\n    # rudimentary wrapper to harvest generators\n    results = wrapped(*args, **kwargs)\n    if isgenerator(results):\n        return list(results)\n    else:\n        return results\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Applying `eval_results` Decorator Without Switch Causes TypeError

**Description:**
When using the `@eval_results` decorator without providing a switch mechanism, the decorated function becomes `None`. As a result, calling the decorated function raises a `TypeError`.

**Example Code:**
```python
from datalad.interface.utils import eval_results

@eval_results
def fake_command():
    yield 1

fake_command()
```

**Expected Behavior:**
The `fake_command` function should execute normally and return the expected results without any errors.

**Actual Behavior:**
Calling `fake_command()` results in the following error:
```
TypeError: 'NoneType' object is not callable
```
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/utils.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface utility functions

"""

__docformat__ = 'restructuredtext'

import logging
from os import curdir
from os import pardir
from os import listdir
from os.path import join as opj
from os.path import lexists
from os.path import isabs
from os.path import isdir
from os.path import dirname
from os.path import relpath
from os.path import sep
from os.path import split as psplit
from itertools import chain

# avoid import from API to not get into circular imports
from datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us
from datalad.utils import assure_list
from datalad.utils import get_trace
from datalad.utils import walk
from datalad.utils import get_dataset_root
from datalad.utils import swallow_logs
from datalad.utils import better_wraps
from datalad.support.gitrepo import GitRepo
from datalad.support.annexrepo import AnnexRepo
from datalad.distribution.dataset import Dataset
from datalad.distribution.dataset import resolve_path
from datalad.distribution.utils import _install_subds_inplace
from datalad.distribution.utils import get_git_dir


lgr = logging.getLogger('datalad.interface.utils')


def handle_dirty_dataset(ds, mode, msg=None):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    ds : Dataset or None
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    msg : str or None
      Custom message to use for a potential commit.

    Returns
    -------
    None
    """
    if ds is None:
        # nothing to be handled
        return
    if msg is None:
        msg = '[DATALAD] auto-saved changes'

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    if ds.repo:
        ds.repo.precommit()

    if mode == 'ignore':
        return
    elif mode == 'fail':
        if not ds.repo or ds.repo.repo.is_dirty(index=True,
                                                working_tree=True,
                                                untracked_files=True,
                                                submodules=True):
            raise RuntimeError('dataset {} has unsaved changes'.format(ds))
    elif mode == 'save-before':
        if not ds.is_installed():
            raise RuntimeError('dataset {} is not yet installed'.format(ds))
        from datalad.interface.save import Save
        Save.__call__(dataset=ds, message=msg, all_changes=True)
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def handle_dirty_datasets(dpaths,
                          mode,
                          base=None,
                          msg='[DATALAD] auto-saved changes'):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    dpaths : sequence(path)
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    base : path or None, optional
      Path of a common super dataset that should also be handled.
    msg : str
      Custom message to use for a potential saved state.

    Returns
    -------
    None
    """
    if mode == 'save-before':
        save_dataset_hierarchy(
            {d: [d] for d in dpaths},
            base=base,
            message=msg)
    elif mode == 'ignore':
        return
    elif mode == 'fail':
        for dpath in dpaths:
            ds = Dataset(dpath)
            if not ds.repo:
                continue
            ds.repo.precommit()
            if ds.repo.repo.is_dirty(index=True,
                                     working_tree=True,
                                     untracked_files=True,
                                     submodules=True):
                raise RuntimeError(
                    'dataset {} has unsaved changes'.format(ds))
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def get_tree_roots(paths):
    """Return common root paths for a set of paths

    This function determines the smallest set of common root
    paths and sorts all given paths under the respective
    root.

    Returns
    -------
    dict
      paths by root
    """
    paths_ws = [_with_sep(p) for p in paths]
    # sort all paths under their potential roots
    roots = {}
    # start from the top to get all paths down the line
    # and collate them into as few roots as possible
    for s in sorted(paths_ws):
        if any([s.startswith(r) for r in roots]):
            # this path is already covered by a known root
            continue
        # find all sub paths
        subs = [p for p in paths if p.startswith(s)]
        roots[s.rstrip(sep)] = subs
    return roots


def sort_paths_into_subdatasets(superds_path, target_subs, spec):
    # XXX forge a chain: whenever some path needs to be pushed down
    # put the receiving dataset as a components to process into the
    # respective superdataset -- this will enable further processing
    # of all datasets in a completely independent fashion
    # (except for order of processing)

    # get all existing subdataset as candidate nodes of the graph
    # that needs to be built and checked
    subds_graph = Dataset(superds_path).get_subdatasets(
        absolute=True, recursive=True, edges=True, fulfilled=True)
    if not subds_graph:
        # no subdatasets, nothing to sort
        return
    for t in target_subs:
        trace = get_trace(
            subds_graph,
            superds_path,
            t)
        if not trace:
            # not connected, or identical
            continue
        tosort = [superds_path] + trace + [t]
        # loop over all but the last one, simplifies logic below
        for i, d in enumerate(tosort[:-1]):
            paths = spec.get(d, [])
            keep_paths = []
            next_ds = tosort[i + 1]
            next_dspaths = spec.get(next_ds, [])
            comp = _with_sep(next_ds)
            for p in assure_list(paths):
                if p.startswith(comp):
                    next_dspaths.append(p)
                    # remember that we pushed the path into this dataset
                    keep_paths.append(next_ds)
                else:
                    keep_paths.append(p)
            spec[next_ds] = next_dspaths
            spec[d] = keep_paths
    # tidy up -- deduplicate
    for c in spec:
        spec[c] = list(set(spec[c]))


def save_dataset_hierarchy(
        info,
        base=None,
        message='[DATALAD] saved changes',
        version_tag=None):
    """Save (disjoint) hierarchies of datasets.

    Saving is done in an order that guarantees that all to be saved
    datasets reflect any possible change of any other to be saved
    subdataset, before they are saved themselves.

    Parameters
    ----------
    info : dict
      Absolute paths of datasets to be saved are the keys, and paths in each
      dataset to be saved are the values
    base : path or None, optional
      Common super dataset that should also be saved.
    message : str
      Message to be used for saving individual datasets

    Returns
    -------
    list
      Instances of saved datasets, in the order in which they where saved.
    """
    if not isinstance(info, dict):
        info = assure_list(info)
        info = dict(zip(info, [[i] for i in info]))
    dpaths = info.keys()
    if base:
        # just a convenience...
        dpaths = assure_list(dpaths)
        dpaths.append(base.path if isinstance(base, Dataset) else base)
    # sort all datasets under their potential superdatasets
    # start from the top to get all subdatasets down the line
    # and collate them into as few superdatasets as possible
    superdss = get_tree_roots(dpaths)
    # for each "superdataset" check the tree of subdatasets and make sure
    # we gather all datasets between the super and any subdataset
    # so we can save them all bottom-up in order to be able to properly
    # save the superdataset
    for superds_path in superdss:
        target_subs = superdss[superds_path]
        sort_paths_into_subdatasets(superds_path, target_subs, info)
    # iterate over all datasets, starting at the bottom
    saved = []
    for dpath in sorted(info.keys(), reverse=True):
        ds = Dataset(dpath)
        if ds.is_installed():
            saved_state = save_dataset(
                ds,
                info[dpath],
                message=message,
                version_tag=version_tag)
            if saved_state:
                saved.append(ds)
    return saved


def save_dataset(
        ds,
        paths=None,
        message=None,
        version_tag=None):
    """Save changes in a single dataset.

    Parameters
    ----------
    ds : Dataset
      The dataset to be saved.
    paths : list, optional
      Paths to dataset components to be saved.
    message: str, optional
      (Commit) message to be attached to the saved state.
    version_tag : str, optional
      Tag to be assigned to the saved state.

    Returns
    -------
    bool
      Whether a new state was saved. If all to be saved content was unmodified
      no new state will be saved.
    """
    # XXX paths must be in the given ds, no further sanity checks!

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    ds.repo.precommit()

    # track what is to be committed, so it becomes
    # possible to decide when/what to save further down
    # and one level up
    orig_hexsha = ds.repo.get_hexsha()

    # always yields list; empty if None
    files = list(
        set(
            [opj(ds.path, f) if not isabs(f) else f for f in assure_list(paths)]))

    # try to consider existing and changed files, and prevent untracked
    # files from being added
    # XXX not acting upon untracked files would be very expensive, because
    # I see no way to avoid using `add` below and git annex has no equivalent
    # to git add's --update -- so for now don't bother
    # XXX alternatively we could consider --no-ignore-removal to also
    # have it delete any already vanished files
    # asking yourself why we need to `add` at all? For example, freshly
    # unlocked files in a v5 repo are listed as "typechange" and commit
    # refuses to touch them without an explicit `add`
    tostage = [f for f in files if lexists(f)]
    if tostage:
        lgr.debug('staging files for commit: %s', tostage)
        if isinstance(ds.repo, AnnexRepo):
            # to make this work without calling `git add` in addition,
            # this needs git-annex v6.20161210 (see #1027)
            ds.repo.add(tostage, commit=False)
        else:
            # --update will ignore any untracked files, sadly git-annex add
            # above does not
            # will complain about vanished files though, filter them here, but
            # keep them for a later commit call
            ds.repo.add(tostage, git_options=['--update'], commit=False)

    _datalad_msg = False
    if not message:
        message = 'Recorded existing changes'
        _datalad_msg = True

    if files or ds.repo.repo.is_dirty(
            index=True,
            working_tree=False,
            untracked_files=False,
            submodules=True):
        # either we have an explicit list of files, or we have something
        # stages otherwise do not attempt to commit, as the underlying
        # repo will happily commit any non-change
        # not checking the working tree or untracked files should make this
        # relavtively cheap

        # TODO: commit() should rather report a dedicated ValueError
        # waiting for #1170
        from datalad.support.exceptions import CommandError
        try:
            # we will blindly call commit not knowing if there is anything to
            # commit -- this is cheaper than to anticipate all possible ways
            # a repo in whatever mode is dirty
            # however, if nothing is dirty the whining wil start
            # --> sucking it up right here
            with swallow_logs(new_level=logging.ERROR) as cml:
                ds.repo.commit(message, options=files, _datalad_msg=_datalad_msg)
        except CommandError as e:
            # TODO until #1171 is resolved, test here for "normal" failure
            # to commit
            if 'nothing to commit' in str(e):
                lgr.debug(
                    "Was instructed to commit %s files but repository is not dirty",
                    files)
            elif 'no changes added to commit' in str(e) or 'nothing added to commit' in str(e):
                lgr.info(
                    'Nothing to save')
            else:
                # relay any prior whining in the exception
                raise ValueError('{} [error log follows] {}; {}'.format(
                    e, e.stdout, e.stderr))

    # MIH: let's tag even if there was nothing commit. I'd forget this
    # option too often...
    if version_tag:
        ds.repo.tag(version_tag)

    _was_modified = ds.repo.get_hexsha() != orig_hexsha

    return ds.repo.repo.head.commit if _was_modified else None


def amend_pathspec_with_superdatasets(spec, topmost=True, limit_single=False):
    """Amend a path spec dictionary with entries for superdatasets

    The result will be a superdataset entry (if a superdataset exists)
    for each input dataset. This entry will (at least) contain the path
    to the subdataset.

    Parameters
    ----------
    spec : dict
      Path spec
    topmost : Dataset or bool
      Flag whether to grab the immediate, or the top-most superdataset
      for each entry, alternatively this can be a dataset instance
      that is used as the topmost dataset.
    limit_single : bool
      If a `topmost` dataset is provided, and this flag is True, only
      the given topmost dataset will be considered as superdataset. Any
      datasets in the spec that are not underneath this dataset will
      not have associated superdataset entries added to the spec.

    Returns
    -------
    dict
      Amended path spec dictionary
    """
    superdss = {}
    for dpath in spec.keys():
        superds = None
        if isinstance(topmost, Dataset):
            if limit_single and dpath == topmost.path:
                # this is already the topmost, no further superdataset to
                # consider
                continue
            if dpath.startswith(_with_sep(topmost.path)):
                # the given topmost dataset is "above" the current
                # datasets path
                superds = topmost
            elif limit_single:
                continue
        if not superds:
            # grab the (topmost) superdataset
            superds = Dataset(dpath).get_superdataset(
                datalad_only=True, topmost=topmost)
        if not superds:
            continue
        # register the subdatasets path in the spec of the superds
        spaths = superdss.get(superds.path, [])
        if not spaths:
            spaths = spec.get(superds.path, [])
        spaths.append(dpath)
        superdss[superds.path] = spaths
    spec.update(superdss)
    return spec


def get_paths_by_dataset(paths, recursive=False, recursion_limit=None,
                         out=None, dir_lookup=None):
    """Sort a list of paths per dataset they are contained in.

    Any paths that are not part of a dataset, or presently unavailable are
    reported.

    Parameter
    ---------
    paths : sequence
      A sequence of path specifications to sort.
    recursive : bool
      Flag whether to report subdatasets under any of the given paths
    recursion_limit :
      Depth constraint for recursion. See `Dataset.get_subdatasets()` for more
      information.
    out : dict or None
      By default a new output dictionary is created, however an existing one
      can be provided via this argument to enable incremental processing.
    dir_lookup : dict or None
      Optional lookup cache that maps paths to previously determined datasets.
      This can speed up repeated processing.

    Returns
    -------
    Tuple(dict, list, list)
      Dict of `existing dataset path`: `path` mappings, the list of currently
      non-existing paths (possibly matching currently uninstalled datasets),
      and any paths that are not part of any dataset.
    """
    # sort paths into the respective datasets
    if dir_lookup is None:
        dir_lookup = {}
    if out is None:
        out = {}
    # paths that don't exist (yet)
    unavailable_paths = []
    nondataset_paths = []
    for path in paths:
        if not lexists(path):
            # not there yet, impossible to say which ds it will actually
            # be in, if any
            unavailable_paths.append(path)
            continue
        # the path exists in some shape or form
        if isdir(path):
            # this could contain all types of additional content
            d = path
        else:
            # for everything else we are interested in the container
            d = dirname(path)
            if not d:
                d = curdir
        # this could be `None` if there is no git repo
        dspath = dir_lookup.get(d, get_dataset_root(d))
        dir_lookup[d] = dspath
        if not dspath:
            nondataset_paths.append(path)
            continue
        if isdir(path):
            ds = Dataset(dspath)
            # we need to doublecheck that this is not a subdataset mount
            # point, in which case get_toppath() would point to the parent
            smpath = ds.get_containing_subdataset(
                path, recursion_limit=1).path
            if smpath != dspath:
                # fix entry
                dir_lookup[d] = smpath
                # submodule still needs to be obtained
                unavailable_paths.append(path)
                continue
            if recursive:
                # make sure we get everything relevant in all _checked out_
                # subdatasets, obtaining of previously unavailable subdataset
                # else done elsewhere
                subs = ds.get_subdatasets(fulfilled=True,
                                          recursive=recursive,
                                          recursion_limit=recursion_limit)
                for sub in subs:
                    subdspath = opj(dspath, sub)
                    if subdspath.startswith(_with_sep(path)):
                        # this subdatasets is underneath the search path
                        # we want it all
                        # be careful to not overwrite anything, in case
                        # this subdataset has been processed before
                        out[subdspath] = out.get(
                            subdspath,
                            [subdspath])
        out[dspath] = out.get(dspath, []) + [path]
    return out, unavailable_paths, nondataset_paths


def get_normalized_path_arguments(paths, dataset=None, default=None):
    """Apply standard resolution to path arguments

    This is nothing more than a helper to standardize path argument
    preprocessing.

    Parameter
    ---------
    paths : sequence or single path
      Path(s) to normalize
    dataset : path or Dataset or None
      Optional dataset identifying something against which to resolve input
      path arguments
    default: sequence of paths or single path or None
      If `paths` is empty, use this instead

    Returns
    -------
    tuple(list(paths), path)
      Normalized paths and path to a potential dataset against which paths were
      resolved.
    """
    dataset_path = dataset.path if isinstance(dataset, Dataset) else dataset
    if not paths and default:
        paths = default
    paths = assure_list(paths)
    # resolve path(s):
    resolved_paths = [resolve_path(p, dataset) for p in paths]
    if dataset:
        # guarantee absolute paths
        resolved_paths = [opj(dataset_path, p) for p in resolved_paths]
    lgr.debug('Resolved input path arguments: %s', resolved_paths)
    return resolved_paths, dataset_path


def path_is_under(values, path=None):
    """Whether a given path is a subdirectory of any of the given test values

    Parameters
    ----------
    values : sequence or dict
      Paths to be tested against. This can be a dictionary in which case
      all values from all keys will be tested against.
    path : path or None
      Test path. If None is given, the process' working directory is
      used.

    Returns
    -------
    bool
    """
    if path is None:
        from datalad.utils import getpwd
        pwd = getpwd()
    if isinstance(values, dict):
        values = chain(*values.values())
    for p in values:
        rpath = relpath(p, start=pwd)
        if rpath == curdir \
                or rpath == pardir \
                or set(psplit(rpath)) == {pardir}:
            # first match is enough
            return True
    return False


def get_dataset_directories(top, ignore_datalad=True):
    """Return a list of directories in the same dataset under a given path

    Parameters
    ----------
    top : path
      Top-level path
    ignore_datalad : bool
      Whether to exlcude the '.datalad' directory of a dataset and its content
      from the results.

    Returns
    -------
    list
      List of directories matching the top-level path, regardless of whether
      these directories are known to Git (i.e. contain tracked files). The
      list does not include the top-level path itself, but it does include
      any subdataset mount points (regardless of whether the particular
      subdatasets are installed or not).
    """
    def func(arg, top, names):
        refpath, ignore, dirs = arg
        legit_names = []
        for n in names:
            path = opj(top, n)
            if not isdir(path) or path in ignore:
                pass
            elif path != refpath and GitRepo.is_valid_repo(path):
                # mount point, keep but don't dive into
                dirs.append(path)
            else:
                legit_names.append(n)
                dirs.append(path)
        names[:] = legit_names

    # collects the directories
    refpath = get_dataset_root(top)
    if not refpath:
        raise ValueError("`top` path {} is not in a dataset".format(top))
    ignore = [opj(refpath, get_git_dir(refpath))]
    if ignore_datalad:
        ignore.append(opj(refpath, '.datalad'))
    d = []
    walk(top, func, (refpath, ignore, d))
    return d


# XXX the following present a different approach to
# amend_pathspec_with_superdatasets() for discovering datasets between
# processed ones and a base
# let it simmer for a while and RF to use one or the other
# this one here seems more leightweight and less convoluted
def _discover_trace_to_known(path, trace, spec):
    # this beast walks the directory tree from a given `path` until
    # it discoveres a known dataset (i.e. recorded in the spec)
    # if it finds one, it commits any accummulated trace of visited
    # datasets on this edge to the spec
    valid_repo = GitRepo.is_valid_repo(path)
    if valid_repo:
        trace = trace + [path]
        if path in spec:
            # found a known repo, commit the trace
            for i, p in enumerate(trace[:-1]):
                spec[p] = list(set(spec.get(p, []) + [trace[i + 1]]))
            # this edge is not done, we need to try to reach any downstream
            # dataset
    for p in listdir(path):
        if valid_repo and p == '.git':
            # ignore gitdir to steed things up
            continue
        p = opj(path, p)
        if not isdir(p):
            continue
        _discover_trace_to_known(p, trace, spec)


def filter_unmodified(content_by_ds, refds, since):
    """Filter per-dataset path specifications based on modification history.

    This function takes a path specification dictionary, as produced by
    `Interface._prep()` and filters it such that only that subset of paths
    remains in the dictionary that corresponding to the set of changes in
    the given reference dataset since a given state.

    The change set is traced across all related subdatasets, i.e. if a submodule
    in the reference dataset is reported as modified then all paths for any given
    subdataset in the modified one are tested for changes too (based on the
    state at which the parent dataset reports a change in the subdataset), and so
    on.

    In doing so, not only unmodified given paths are removed, but also modified
    given paths are replaced by the set of actually modified paths within them.

    Only committed changes are considered!

    Parameters
    ----------
    content_by_ds : dict
      Per-dataset path specifications, as produced ,for example, by
      `Interface._prep()`
    refds : Dataset or *Repo or path
      Reference dataset for which to determine the initial change set
    since : state
      Any commit-ish/tree-ish supported by Git (tag, commit, branch, ...).
      Changes between this given state and the most recent commit are
      evaluated.

    Returns
    -------
    dict
      Filtered path spec dictionary. If `since` is not None, the output is
      guaranteed to only contain paths to modified, and presently existing
      components of subdatasets of the given reference dataset (and itself).
    """
    if since is None:
        # we want all, subds not matching the ref are assumed to have been
        # sorted out before (e.g. one level up)
        return content_by_ds
    # turn refds argument into a usable repo instance
    if not hasattr(refds, 'path'):
        # not a Repo or Dataset
        refds_path = refds
        refds = GitRepo(refds, create=False)
    else:
        refds_path = refds.path
    repo = refds.repo
    if hasattr(repo, 'repo'):
        # TODO use GitRepo.diff() when available (gh-1217)
        repo = repo.repo

    # life is simple: we diff the base dataset, and kill anything that
    # does not start with something that is in the diff
    # we cannot really limit the diff paths easily because we might get
    # or miss content (e.g. subdatasets) if we don't figure out which ones
    # are known -- and we don't want that
    diff = repo.commit().diff(since)
    # get all modified paths (with original? commit) that are still
    # present
    modified = dict((opj(refds_path, d.b_path),
                    d.b_blob.hexsha if d.b_blob else None)
                    for d in diff)
    if not modified:
        # nothing modified nothing to report
        return {}
    # determine the subset that is a directory and hence is relevant for possible
    # subdatasets
    modified_dirs = {_with_sep(d) for d in modified if isdir(d)}
    # find the subdatasets matching modified paths, this will also kick out
    # any paths that are not in the dataset sub-hierarchy
    mod_subs = {candds: paths
                for candds, paths in content_by_ds.items()
                if candds != refds_path and
                any(_with_sep(candds).startswith(md) for md in modified_dirs)}
    # now query the next level down
    keep_subs = \
        [filter_unmodified(mod_subs, subds_path, modified[subds_path])
         for subds_path in mod_subs
         if subds_path in modified]
    # merge result list into a single dict
    keep = {k: v for d in keep_subs for k, v in d.items()}

    paths_refds = content_by_ds[refds_path]
    keep[refds_path] = [m for m in modified
                        if lexists(m)  # still around
                        and (m in paths_refds  # listed file, or subds
                        # or a modified path under a given directory
                        or any(m.startswith(_with_sep(p)) for p in paths_refds))]
    return keep

import wrapt
@wrapt.decorator
def eval_results(wrapped, instance, args, kwargs):
    """Decorator providing functionality to evaluate return values of datalad
    commands
    """
    from inspect import isgenerator

    # rudimentary wrapper to harvest generators
    results = wrapped(*args, **kwargs)
    if isgenerator(results):
        return list(results)
    else:
        return results

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user