{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport os\nimport sys\nimport re\nimport textwrap\nfrom importlib import import_module\nimport inspect\nimport string\nfrom collections import (\n    defaultdict,\n    OrderedDict,\n)\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureChoice,\n)\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.plugin import _get_plugins\nfrom datalad.plugin import _load_plugin\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups(include_plugins=False):\n    \"\"\"Return a list of command groups.\n\n    Parameters\n    ----------\n    include_plugins : bool, optional\n        Whether to include a group named 'plugins' that has a list of\n        discovered plugin commands.\n\n    Returns\n    -------\n    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).\n    \"\"\"\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    # TODO(yoh): see if we could retain \"generator\" for plugins\n    # ATM we need to make it explicit so we could check the command(s) below\n    # It could at least follow the same destiny as extensions so we would\n    # just do more iterative \"load ups\"\n\n    if include_plugins:\n        grps.append(('plugins', 'Plugins', list(_get_plugins())))\n    return grps\n\n\ndef get_cmd_summaries(descriptions, groups, width=79):\n    \"\"\"Return summaries for the commands in `groups`.\n\n    Parameters\n    ----------\n    descriptions : dict\n        A map of group names to summaries.\n    groups : list of tuples\n        A list of groups and commands in the form described by\n        `get_interface_groups`.\n    width : int, optional\n        The maximum width of each line in the summary text.\n\n    Returns\n    -------\n    A list with a formatted entry for each command. The first command of each\n    group is preceded by an entry describing the group.\n    \"\"\"\n    cmd_summary = []\n    for grp in sorted(groups, key=lambda x: x[1]):\n        grp_descr = grp[1]\n        grp_cmds = descriptions[grp[0]]\n\n        cmd_summary.append('\\n*%s*\\n' % (grp_descr,))\n        for cd in grp_cmds:\n            cmd_summary.append('  %s\\n%s'\n                               % ((cd[0],\n                                   textwrap.fill(\n                                       cd[1].rstrip(' .'),\n                                       width - 5,\n                                       initial_indent=' ' * 6,\n                                       subsequent_indent=' ' * 6))))\n    return cmd_summary\n\n\ndef load_interface(spec):\n    \"\"\"Load and return the class for `spec`.\n\n    Parameters\n    ----------\n    spec : tuple\n        For a standard interface, the first item is the datalad source module\n        and the second object name for the interface. For a plugin, the second\n        item should be a dictionary that maps 'file' to the path the of module.\n\n    Returns\n    -------\n    The interface class or, if importing the module fails, None.\n    \"\"\"\n    if isinstance(spec[1], dict):\n        intf = _load_plugin(spec[1]['file'], fail=False)\n    else:\n        lgr.log(5, \"Importing module %s \" % spec[0])\n        try:\n            mod = import_module(spec[0], package='datalad')\n        except Exception as e:\n            lgr.error(\"Internal error, cannot import interface '%s': %s\",\n                      spec[0], exc_str(e))\n            intf = None\n        else:\n            intf = getattr(mod, spec[1])\n    return intf\n\n\ndef get_cmd_doc(interface):\n    \"\"\"Return the documentation for the command defined by `interface`.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()\n    if hasattr(interface, '_docs_'):\n        # expand docs\n        intf_doc = intf_doc.format(**interface._docs_)\n    return intf_doc\n\n\ndef get_cmd_ex(interface):\n    \"\"\"Return the examples for the command defined by 'interface'.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_ex = \"\\n\\n*Examples*\\n\\n\"\n    for example in interface._examples_:\n        intf_ex += build_example(example, api='cmdline')\n    return intf_ex\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>.*?\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>(.*?)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    if 'DATALAD_SPHINX_RUN' not in os.environ:\n        # remove :role:`...` RST markup for cmdline docs\n        docs = re.sub(\n            r':\\S+:`[^`]*`[\\\\]*',\n            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n            docs,\n            flags=re.MULTILINE | re.DOTALL)\n        # make the handbook doc references more accessible\n        # the URL is a redirect configured at readthedocs\n        docs = re.sub(\n            r'(handbook:[0-9]-[0-9]*)',\n            '\\\\1 (http://handbook.datalad.org/symbols)',\n            docs)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>.*?\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>(.*?)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # make the handbook doc references more accessible\n    # the URL is a redirect configured at readthedocs\n    docs = re.sub(\n        r'(handbook:[0-9]-[0-9]*)',\n        '\\\\1 (http://handbook.datalad.org/symbols)',\n        docs)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        r',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        r'`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        r'\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from datalad.utils import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_example(example, api='python'):\n    \"\"\"Build a code example.\n\n    Take a dict from a classes _example_ specification (list of dicts) and\n    build a string with an api or cmd example (for use in cmd help or\n    docstring).\n\n    Parameters\n    ----------\n    api : {'python', 'cmdline'}\n        If 'python', build Python example for docstring. If 'cmdline', build\n        cmd example.\n\n    Returns\n    -------\n    ex : str\n        Concatenated examples for the given class.\n    \"\"\"\n    if api == 'python' :\n        code_field='code_py'\n        indicator='>'\n    elif api == 'cmdline':\n        code_field='code_cmd'\n        indicator='%'\n    else:\n        raise ValueError(\"unknown API selection: {}\".format(api))\n    if code_field not in example:\n        # only show an example if it exist for the API\n        return ''\n    description = textwrap.fill(example.get('text'))\n    # this indent the code snippet to get it properly rendered as code\n    # we are not using textwrap.fill(), because it would not acknowledge\n    # any meaningful structure/formatting of code snippets. Instead, we\n    # maintain line content as is.\n    code = dedent_docstring(example.get(code_field))\n    needs_indicator = not code.startswith(indicator)\n    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()\n\n    ex = \"\"\"{}::\\n\\n   {}{}\\n\\n\"\"\".format(\n        description,\n        # disable automatic prefixing, if the example already has one\n        # this enables providing more complex examples without having\n        # to infer its inner structure\n        '{} '.format(indicator)\n        if needs_indicator\n        # maintain spacing to avoid undesired relative indentation\n        else '',\n        code)\n\n    return ex\n\n\ndef update_docstring_with_examples(cls_doc, ex):\n    \"\"\"Update a commands docstring with examples.\n\n    Take _examples_ of a command, build the Python examples, and append\n    them to the docstring.\n\n    cls_doc: docstring\n    ex: list\n        list of dicts with examples\n    \"\"\"\n    from textwrap import indent\n    if len(cls_doc):\n        cls_doc += \"\\n\"\n    cls_doc += \"    Examples\\n    --------\\n\"\n    # loop though provided examples\n    for example in ex:\n        cls_doc += indent(build_example(example, api='python'), ' '*4)\n\n    return cls_doc\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None\n    )\n\n    # return original\n    return cls\n\n\nNA_STRING = 'N/A'  # we might want to make it configurable via config\n\n\nclass nagen(object):\n    \"\"\"A helper to provide a desired missing value if no value is known\n\n    Usecases\n    - could be used as a generator for `defaultdict`\n    - since it returns itself upon getitem, should work even for complex\n      nested dictionaries/lists .format templates\n    \"\"\"\n    def __init__(self, missing=NA_STRING):\n        self.missing = missing\n\n    def __repr__(self):\n        cls = self.__class__.__name__\n        args = str(self.missing) if self.missing != NA_STRING else ''\n        return '%s(%s)' % (cls, args)\n\n    def __str__(self):\n        return self.missing\n\n    def __getitem__(self, *args):\n        return self\n\n    def __getattr__(self, item):\n        return self\n\n\ndef nadict(*items):\n    \"\"\"A generator of default dictionary with the default nagen\"\"\"\n    dd = defaultdict(nagen)\n    dd.update(*items)\n    return dd\n\n\nclass DefaultOutputFormatter(string.Formatter):\n    \"\"\"A custom formatter for default output rendering using .format\n    \"\"\"\n    # TODO: make missing configurable?\n    def __init__(self, missing=nagen()):\n        \"\"\"\n        Parameters\n        ----------\n        missing: string, optional\n          What to output for the missing values\n        \"\"\"\n        super(DefaultOutputFormatter, self).__init__()\n        self.missing = missing\n\n    def _d(self, msg, *args):\n        # print(\"   HERE %s\" % (msg % args))\n        pass\n\n    def get_value(self, key, args, kwds):\n        assert not args\n        self._d(\"get_value: %r %r %r\", key, args, kwds)\n        return kwds.get(key, self.missing)\n\n    # def get_field(self, field_name, args, kwds):\n    #     assert not args\n    #     self._d(\"get_field: %r args=%r kwds=%r\" % (field_name, args, kwds))\n    #     try:\n    #         out = string.Formatter.get_field(self, field_name, args, kwds)\n    #     except Exception as exc:\n    #         # TODO needs more than just a value\n    #         return \"!ERR %s\" % exc\n\n\nclass DefaultOutputRenderer(object):\n    \"\"\"A default renderer for .format'ed output line\n    \"\"\"\n    def __init__(self, format):\n        self.format = format\n        # We still need custom output formatter since at the \"first level\"\n        # within .format template all items there is no `nadict`\n        self.formatter = DefaultOutputFormatter()\n\n    @classmethod\n    def _dict_to_nadict(cls, v):\n        \"\"\"Traverse datastructure and replace any regular dict with nadict\"\"\"\n        if isinstance(v, list):\n            return [cls._dict_to_nadict(x) for x in v]\n        elif isinstance(v, dict):\n            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())\n        else:\n            return v\n\n    def __call__(self, x, **kwargs):\n        dd = nadict(\n            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)\n                for k_, v_ in v.items()})\n                if isinstance(v, dict) else v)\n            for k, v in x.items()\n        )\n\n        msg = self.formatter.format(self.format, **dd)\n        return ui.message(msg)\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from datalad.utils import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args == tuple():\n                # explicitly provided an empty sequence of argument names\n                # this shall not appear in the parser\n                continue\n            elif cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help.rstrip()[-1] != '.':\n                help = help.rstrip() + '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n                if 'metavar' not in parser_kwargs and \\\n                        isinstance(param.constraints, EnsureChoice):\n                    parser_kwargs['metavar'] = \\\n                        '{%s}' % '|'.join(\n                            # don't use short_description(), because\n                            # it also needs to give valid output for\n                            # Python syntax (quotes...), but here we\n                            # can simplify to shell syntax where everything\n                            # is a string\n                            p for p in param.constraints._allowed\n                            # in the cmdline None pretty much means\n                            # don't give the options, so listing it\n                            # doesn't make sense. Moreover, any non-string\n                            # value cannot be given and very likely only\n                            # serves a special purpose in the Python API\n                            # or implementation details\n                            if isinstance(p, str))\n            if defaults_idx >= 0:\n                # if it is a flag, in commandline it makes little sense to show\n                # showing the Default: (likely boolean).\n                #   See https://github.com/datalad/datalad/issues/3203\n                if not parser_kwargs.get('action', '').startswith('store_'):\n                    # [Default: None] also makes little sense for cmdline\n                    if defaults[defaults_idx] is not None:\n                        help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from datalad.utils import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k)\n                  for k in argnames\n                  # some arguments might be Python-only and do not appear in the\n                  # parser Namespace\n                  if hasattr(args, k) and is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)\n\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = str(resolve_path(refds_path))\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from datalad.utils import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport os\nimport sys\nimport re\nimport textwrap\nfrom importlib import import_module\nimport inspect\nimport string\nfrom collections import (\n    defaultdict,\n    OrderedDict,\n)\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureChoice,\n)\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.plugin import _get_plugins\nfrom datalad.plugin import _load_plugin\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups(include_plugins=False):\n    \"\"\"Return a list of command groups.\n\n    Parameters\n    ----------\n    include_plugins : bool, optional\n        Whether to include a group named 'plugins' that has a list of\n        discovered plugin commands.\n\n    Returns\n    -------\n    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).\n    \"\"\"\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    # TODO(yoh): see if we could retain \"generator\" for plugins\n    # ATM we need to make it explicit so we could check the command(s) below\n    # It could at least follow the same destiny as extensions so we would\n    # just do more iterative \"load ups\"\n\n    if include_plugins:\n        grps.append(('plugins', 'Plugins', list(_get_plugins())))\n    return grps\n\n\ndef get_cmd_summaries(descriptions, groups, width=79):\n    \"\"\"Return summaries for the commands in `groups`.\n\n    Parameters\n    ----------\n    descriptions : dict\n        A map of group names to summaries.\n    groups : list of tuples\n        A list of groups and commands in the form described by\n        `get_interface_groups`.\n    width : int, optional\n        The maximum width of each line in the summary text.\n\n    Returns\n    -------\n    A list with a formatted entry for each command. The first command of each\n    group is preceded by an entry describing the group.\n    \"\"\"\n    cmd_summary = []\n    for grp in sorted(groups, key=lambda x: x[1]):\n        grp_descr = grp[1]\n        grp_cmds = descriptions[grp[0]]\n\n        cmd_summary.append('\\n*%s*\\n' % (grp_descr,))\n        for cd in grp_cmds:\n            cmd_summary.append('  %s\\n%s'\n                               % ((cd[0],\n                                   textwrap.fill(\n                                       cd[1].rstrip(' .'),\n                                       width - 5,\n                                       initial_indent=' ' * 6,\n                                       subsequent_indent=' ' * 6))))\n    return cmd_summary\n\n\ndef load_interface(spec):\n    \"\"\"Load and return the class for `spec`.\n\n    Parameters\n    ----------\n    spec : tuple\n        For a standard interface, the first item is the datalad source module\n        and the second object name for the interface. For a plugin, the second\n        item should be a dictionary that maps 'file' to the path the of module.\n\n    Returns\n    -------\n    The interface class or, if importing the module fails, None.\n    \"\"\"\n    if isinstance(spec[1], dict):\n        intf = _load_plugin(spec[1]['file'], fail=False)\n    else:\n        lgr.log(5, \"Importing module %s \" % spec[0])\n        try:\n            mod = import_module(spec[0], package='datalad')\n        except Exception as e:\n            lgr.error(\"Internal error, cannot import interface '%s': %s\",\n                      spec[0], exc_str(e))\n            intf = None\n        else:\n            intf = getattr(mod, spec[1])\n    return intf\n\n\ndef get_cmd_doc(interface):\n    \"\"\"Return the documentation for the command defined by `interface`.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()\n    if hasattr(interface, '_docs_'):\n        # expand docs\n        intf_doc = intf_doc.format(**interface._docs_)\n    return intf_doc\n\n\ndef get_cmd_ex(interface):\n    \"\"\"Return the examples for the command defined by 'interface'.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_ex = \"\\n\\n*Examples*\\n\\n\"\n    for example in interface._examples_:\n        intf_ex += build_example(example, api='cmdline')\n    return intf_ex\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>.*?\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>(.*?)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    if 'DATALAD_SPHINX_RUN' not in os.environ:\n        # remove :role:`...` RST markup for cmdline docs\n        docs = re.sub(\n            r':\\S+:`[^`]*`[\\\\]*',\n            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n            docs,\n            flags=re.MULTILINE | re.DOTALL)\n        # make the handbook doc references more accessible\n        # the URL is a redirect configured at readthedocs\n        docs = re.sub(\n            r'(handbook:[0-9]-[0-9]*)',\n            '\\\\1 (http://handbook.datalad.org/symbols)',\n            docs)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>.*?\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>(.*?)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # make the handbook doc references more accessible\n    # the URL is a redirect configured at readthedocs\n    docs = re.sub(\n        r'(handbook:[0-9]-[0-9]*)',\n        '\\\\1 (http://handbook.datalad.org/symbols)',\n        docs)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        r',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        r'`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        r'\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from datalad.utils import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_example(example, api='python'):\n    \"\"\"Build a code example.\n\n    Take a dict from a classes _example_ specification (list of dicts) and\n    build a string with an api or cmd example (for use in cmd help or\n    docstring).\n\n    Parameters\n    ----------\n    api : {'python', 'cmdline'}\n        If 'python', build Python example for docstring. If 'cmdline', build\n        cmd example.\n\n    Returns\n    -------\n    ex : str\n        Concatenated examples for the given class.\n    \"\"\"\n    if api == 'python' :\n        code_field='code_py'\n        indicator='>'\n    elif api == 'cmdline':\n        code_field='code_cmd'\n        indicator='%'\n    else:\n        raise ValueError(\"unknown API selection: {}\".format(api))\n    if code_field not in example:\n        # only show an example if it exist for the API\n        return ''\n    description = textwrap.fill(example.get('text'))\n    # this indent the code snippet to get it properly rendered as code\n    # we are not using textwrap.fill(), because it would not acknowledge\n    # any meaningful structure/formatting of code snippets. Instead, we\n    # maintain line content as is.\n    code = dedent_docstring(example.get(code_field))\n    needs_indicator = not code.startswith(indicator)\n    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()\n\n    ex = \"\"\"{}::\\n\\n   {}{}\\n\\n\"\"\".format(\n        description,\n        # disable automatic prefixing, if the example already has one\n        # this enables providing more complex examples without having\n        # to infer its inner structure\n        '{} '.format(indicator)\n        if needs_indicator\n        # maintain spacing to avoid undesired relative indentation\n        else '',\n        code)\n\n    return ex\n\n\ndef update_docstring_with_examples(cls_doc, ex):\n    \"\"\"Update a commands docstring with examples.\n\n    Take _examples_ of a command, build the Python examples, and append\n    them to the docstring.\n\n    cls_doc: docstring\n    ex: list\n        list of dicts with examples\n    \"\"\"\n    from textwrap import indent\n    if len(cls_doc):\n        cls_doc += \"\\n\"\n    cls_doc += \"    Examples\\n    --------\\n\"\n    # loop though provided examples\n    for example in ex:\n        cls_doc += indent(build_example(example, api='python'), ' '*4)\n\n    return cls_doc\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n\n\n    # update class attributes that may override defaults \n    if hasattr(cls, '_no_eval_results'):\n        add_args = None\n    else:\n        add_args = {k: getattr(cls, k, v) for k, v in eval_defaults.items()}\n\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=add_args\n    )\n\n    # return original\n    return cls\n\n\nNA_STRING = 'N/A'  # we might want to make it configurable via config\n\n\nclass nagen(object):\n    \"\"\"A helper to provide a desired missing value if no value is known\n\n    Usecases\n    - could be used as a generator for `defaultdict`\n    - since it returns itself upon getitem, should work even for complex\n      nested dictionaries/lists .format templates\n    \"\"\"\n    def __init__(self, missing=NA_STRING):\n        self.missing = missing\n\n    def __repr__(self):\n        cls = self.__class__.__name__\n        args = str(self.missing) if self.missing != NA_STRING else ''\n        return '%s(%s)' % (cls, args)\n\n    def __str__(self):\n        return self.missing\n\n    def __getitem__(self, *args):\n        return self\n\n    def __getattr__(self, item):\n        return self\n\n\ndef nadict(*items):\n    \"\"\"A generator of default dictionary with the default nagen\"\"\"\n    dd = defaultdict(nagen)\n    dd.update(*items)\n    return dd\n\n\nclass DefaultOutputFormatter(string.Formatter):\n    \"\"\"A custom formatter for default output rendering using .format\n    \"\"\"\n    # TODO: make missing configurable?\n    def __init__(self, missing=nagen()):\n        \"\"\"\n        Parameters\n        ----------\n        missing: string, optional\n          What to output for the missing values\n        \"\"\"\n        super(DefaultOutputFormatter, self).__init__()\n        self.missing = missing\n\n    def _d(self, msg, *args):\n        # print(\"   HERE %s\" % (msg % args))\n        pass\n\n    def get_value(self, key, args, kwds):\n        assert not args\n        self._d(\"get_value: %r %r %r\", key, args, kwds)\n        return kwds.get(key, self.missing)\n\n    # def get_field(self, field_name, args, kwds):\n    #     assert not args\n    #     self._d(\"get_field: %r args=%r kwds=%r\" % (field_name, args, kwds))\n    #     try:\n    #         out = string.Formatter.get_field(self, field_name, args, kwds)\n    #     except Exception as exc:\n    #         # TODO needs more than just a value\n    #         return \"!ERR %s\" % exc\n\n\nclass DefaultOutputRenderer(object):\n    \"\"\"A default renderer for .format'ed output line\n    \"\"\"\n    def __init__(self, format):\n        self.format = format\n        # We still need custom output formatter since at the \"first level\"\n        # within .format template all items there is no `nadict`\n        self.formatter = DefaultOutputFormatter()\n\n    @classmethod\n    def _dict_to_nadict(cls, v):\n        \"\"\"Traverse datastructure and replace any regular dict with nadict\"\"\"\n        if isinstance(v, list):\n            return [cls._dict_to_nadict(x) for x in v]\n        elif isinstance(v, dict):\n            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())\n        else:\n            return v\n\n    def __call__(self, x, **kwargs):\n        dd = nadict(\n            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)\n                for k_, v_ in v.items()})\n                if isinstance(v, dict) else v)\n            for k, v in x.items()\n        )\n\n        msg = self.formatter.format(self.format, **dd)\n        return ui.message(msg)\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from datalad.utils import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args == tuple():\n                # explicitly provided an empty sequence of argument names\n                # this shall not appear in the parser\n                continue\n            elif cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help.rstrip()[-1] != '.':\n                help = help.rstrip() + '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n                if 'metavar' not in parser_kwargs and \\\n                        isinstance(param.constraints, EnsureChoice):\n                    parser_kwargs['metavar'] = \\\n                        '{%s}' % '|'.join(\n                            # don't use short_description(), because\n                            # it also needs to give valid output for\n                            # Python syntax (quotes...), but here we\n                            # can simplify to shell syntax where everything\n                            # is a string\n                            p for p in param.constraints._allowed\n                            # in the cmdline None pretty much means\n                            # don't give the options, so listing it\n                            # doesn't make sense. Moreover, any non-string\n                            # value cannot be given and very likely only\n                            # serves a special purpose in the Python API\n                            # or implementation details\n                            if isinstance(p, str))\n            if defaults_idx >= 0:\n                # if it is a flag, in commandline it makes little sense to show\n                # showing the Default: (likely boolean).\n                #   See https://github.com/datalad/datalad/issues/3203\n                if not parser_kwargs.get('action', '').startswith('store_'):\n                    # [Default: None] also makes little sense for cmdline\n                    if defaults[defaults_idx] is not None:\n                        help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from datalad.utils import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k)\n                  for k in argnames\n                  # some arguments might be Python-only and do not appear in the\n                  # parser Namespace\n                  if hasattr(args, k) and is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)\n\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = str(resolve_path(refds_path))\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from datalad.utils import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n",
            "header": {
                "file": {
                    "path": "datalad/interface/base.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "f9be2c566",
                "new_commit_hash": "62abc6e72",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/base.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/base.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 514,
                            "length": 10
                        },
                        "new_range": {
                            "start": 514,
                            "length": 19
                        },
                        "section": "def build_doc(cls, **kwargs):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # build standard doc and insert eval_doc",
                                "type": "context"
                            },
                            {
                                "content": "    spec = getattr(cls, '_params_', dict())",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # update class attributes that may override defaults ",
                                "type": "added"
                            },
                            {
                                "content": "    if hasattr(cls, '_no_eval_results'):",
                                "type": "added"
                            },
                            {
                                "content": "        add_args = None",
                                "type": "added"
                            },
                            {
                                "content": "    else:",
                                "type": "added"
                            },
                            {
                                "content": "        add_args = {k: getattr(cls, k, v) for k, v in eval_defaults.items()}",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # ATTN: An important consequence of this update() call is that it",
                                "type": "context"
                            },
                            {
                                "content": "    # fulfills the docstring's promise of overriding any existing",
                                "type": "context"
                            },
                            {
                                "content": "    # values for eval_params keys in _params_.",
                                "type": "context"
                            },
                            {
                                "content": "    #",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # get docs for eval_results parameters:",
                                "type": "context"
                            },
                            {
                                "content": "    spec.update(eval_params)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/base.py",
                            "type": "function",
                            "name": "build_doc",
                            "content": "def build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n\n\n    # update class attributes that may override defaults \n    if hasattr(cls, '_no_eval_results'):\n        add_args = None\n    else:\n        add_args = {k: getattr(cls, k, v) for k, v in eval_defaults.items()}\n\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=add_args\n    )\n\n    # return original\n    return cls",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 469,
                            "end_lineno": 541,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 525,
                            "length": 7
                        },
                        "new_range": {
                            "start": 534,
                            "length": 7
                        },
                        "section": "def build_doc(cls, **kwargs):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        cls.__call__, spec,",
                                "type": "context"
                            },
                            {
                                "content": "        prefix=alter_interface_docs_for_api(cls_doc),",
                                "type": "context"
                            },
                            {
                                "content": "        suffix=alter_interface_docs_for_api(call_doc),",
                                "type": "context"
                            },
                            {
                                "content": "        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None",
                                "type": "deleted"
                            },
                            {
                                "content": "        add_args=add_args",
                                "type": "added"
                            },
                            {
                                "content": "    )",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # return original",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/base.py",
                            "type": "function",
                            "name": "build_doc",
                            "content": "def build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None\n    )\n\n    # return original\n    return cls",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 469,
                            "end_lineno": 532,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom ..base import Interface\nfrom ..utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom ..base import Interface\nfrom ..utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    result_renderer = 'tailored' # overrides None default\n    return_type = 'item-or-list' # overrides 'list'\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring shows correct override values of defaults in eval_params\n    assert_in(\"Default: 'tailored'\", doc1)\n    assert_in(\"Default: 'item-or-list'\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "ebb42e630",
                "new_commit_hash": "87c46738c",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 206,
                            "length": 6
                        },
                        "new_range": {
                            "start": 206,
                            "length": 9
                        },
                        "section": "def test_save_hierarchy(path):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "class TestUtils(Interface):",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"TestUtil's fake command\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    result_renderer = 'tailored' # overrides None default",
                                "type": "added"
                            },
                            {
                                "content": "    return_type = 'item-or-list' # overrides 'list'",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    _params_ = dict(",
                                "type": "context"
                            },
                            {
                                "content": "        number=Parameter(",
                                "type": "context"
                            },
                            {
                                "content": "            args=(\"-n\", \"--number\",),",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "class",
                            "name": "TestUtils",
                            "content": "class TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    result_renderer = 'tailored' # overrides None default\n    return_type = 'item-or-list' # overrides 'list'\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 206,
                            "end_lineno": 240,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 255,
                            "length": 6
                        },
                        "new_range": {
                            "start": 258,
                            "length": 10
                        },
                        "section": "def test_eval_results_plus_build_doc():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    assert_in(\"Parameters\", doc1)",
                                "type": "context"
                            },
                            {
                                "content": "    assert_in(\"It's a number\", doc1)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # docstring shows correct override values of defaults in eval_params",
                                "type": "added"
                            },
                            {
                                "content": "    assert_in(\"Default: 'tailored'\", doc1)",
                                "type": "added"
                            },
                            {
                                "content": "    assert_in(\"Default: 'item-or-list'\", doc1)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # docstring also contains eval_result's parameters:",
                                "type": "context"
                            },
                            {
                                "content": "    assert_in(\"result_filter\", doc1)",
                                "type": "context"
                            },
                            {
                                "content": "    assert_in(\"return_type\", doc1)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "function",
                            "name": "test_eval_results_plus_build_doc",
                            "content": "def test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring shows correct override values of defaults in eval_params\n    assert_in(\"Default: 'tailored'\", doc1)\n    assert_in(\"Default: 'item-or-list'\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 243,
                            "end_lineno": 296,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "47614e0b75cefc0ba59b2efe579c5ce540ebdcd9^",
    "new_commit_hash": "47614e0b75cefc0ba59b2efe579c5ce540ebdcd9",
    "commit_message": "Merge branch 'iss2762': DOC: Fixed documentation builder to allow for overriding opts...\n\n* iss2762:\n  [DATALAD RUNCMD] Minor typo fix (over[r]ide)\n  [DOC] Add test for eval_params overrids\n  DOC: corrects previous fix for defaults\n  DOC: Fixed documentation builder to allow for overriding opts in eval_defaults. Closes #2762",
    "commit_date": "2020-06-19T11:45:14-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "47614e0b75cefc0ba59b2efe579c5ce540ebdcd9",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom datalad.interface.base import Interface\nfrom datalad.interface.utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    result_renderer = 'tailored' # overrides None default\n    return_type = 'item-or-list' # overrides 'list'\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring shows correct override values of defaults in eval_params\n    assert_in(\"Default: 'tailored'\", doc1)\n    assert_in(\"Default: 'item-or-list'\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.13.0rc2.dev76\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 90ms\nInstalled 25 packages in 3.37s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 63 packages in 4.22s\nPrepared 1 package in 3.67s\nInstalled 57 packages in 526ms\n + annexremote==1.6.6\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.13.0rc2 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.0.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 97 packages in 298ms\nInstalled 32 packages in 814ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9\nplugins: cov-6.0.0\ncollected 6 items\n\nr2e_tests/test_1.py FF..E.                                               [100%]\n\n==================================== ERRORS ====================================\n___________________ ERROR at setup of test_discover_ds_trace ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9/r2e_tests/test_1.py, line 336\n  @with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n  @with_tempfile(mkdir=True)\n  def test_discover_ds_trace(path, otherdir):\nE       fixture 'otherdir' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9/r2e_tests/test_1.py:336\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexac1x8bn9'}\nfilename = '/tmp/datalad_temp_test_dirtyljc764q8'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:714: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexjh0t49t3'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchye09ri9ge'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:541: TypeError\n==================================== PASSES ====================================\n_______________________ test_eval_results_plus_build_doc _______________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.utils:utils.py:2493 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\nDEBUG    datalad.utils:utils.py:2493 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\n_________________________ test_utils_suppress_similar __________________________\n------------------------------ Captured log call -------------------------------\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_eval_results_plus_build_doc\nPASSED r2e_tests/test_1.py::test_result_filter\nPASSED r2e_tests/test_1.py::test_utils_suppress_similar\nERROR r2e_tests/test_1.py::test_discover_ds_trace\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\n===================== 2 failed, 3 passed, 1 error in 2.81s =====================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9\nplugins: cov-6.0.0\ncollected 6 items\n\nr2e_tests/test_1.py FFF.E.                                               [100%]\n\n==================================== ERRORS ====================================\n___________________ ERROR at setup of test_discover_ds_trace ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9/r2e_tests/test_1.py, line 336\n  @with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n  @with_tempfile(mkdir=True)\n  def test_discover_ds_trace(path, otherdir):\nE       fixture 'otherdir' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_47614e0b75cefc0ba59b2efe579c5ce540ebdcd9/r2e_tests/test_1.py:336\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexjbbnmq5p'}\nfilename = '/tmp/datalad_temp_test_dirty50mbe2_b'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:714: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvfsqrsf_'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchyjnu1h78j'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:541: TypeError\n_______________________ test_eval_results_plus_build_doc _______________________\n\n    def test_eval_results_plus_build_doc():\n    \n        # test docs\n    \n        # docstring was build already:\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            TestUtils().__call__(1)\n            assert_not_in(\"Building doc for\", cml.out)\n        # docstring accessible both ways:\n        doc1 = Dataset.fake_command.__doc__\n        doc2 = TestUtils().__call__.__doc__\n    \n        # docstring was built from Test_Util's definition:\n        assert_equal(doc1, doc2)\n        assert_in(\"TestUtil's fake command\", doc1)\n        assert_in(\"Parameters\", doc1)\n        assert_in(\"It's a number\", doc1)\n    \n        # docstring shows correct override values of defaults in eval_params\n>       assert_in(\"Default: 'tailored'\", doc1)\n\nr2e_tests/test_1.py:262: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../../../../.local/share/uv/python/cpython-3.9.21-linux-x86_64-gnu/lib/python3.9/unittest/case.py:1104: in assertIn\n    self.fail(self._formatMessage(msg, standardMsg))\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <nose.tools.trivial.Dummy testMethod=nop>\nmsg = '\"Default: \\'tailored\\'\" not found in \\'TestUtil\\\\\\'s fake command\\\\nParameters\\\\n----------\\\\nnumber : str or None\\\\n...ist in case\\\\n  of multiple return values. `None` is return in case of an empty\\\\n  list. [Default: \\\\\\'list\\\\\\']\\\\n\\''\n\n    def fail(self, msg=None):\n        \"\"\"Fail immediately, with the given message.\"\"\"\n>       raise self.failureException(msg)\nE       AssertionError: \"Default: 'tailored'\" not found in 'TestUtil\\'s fake command\\nParameters\\n----------\\nnumber : str or None\\n  It\\'s a number.\\ndataset : Dataset or None, optional\\n  \"specify the dataset to update.  If no dataset is given, an attempt\\n  is made to identify the dataset based on the input and/or the\\n  current working directory. [Default: None]\\nresult_fn\\n  Generate the result records with this function rather than using the\\n  default logic. `number` will be passed as an argument. [Default:\\n  None]\\non_failure : {\\'ignore\\', \\'continue\\', \\'stop\\'}, optional\\n  behavior to perform on failure: \\'ignore\\' any failure is reported,\\n  but does not cause an exception; \\'continue\\' if any failure occurs an\\n  exception will be raised at the end, but processing other actions\\n  will continue for as long as possible; \\'stop\\': processing will stop\\n  on first failure and an exception is raised. A failure is any result\\n  with status \\'impossible\\' or \\'error\\'. Raised exception is an\\n  IncompleteResultsError that carries the result dictionaries of the\\n  failures in its `failed` attribute. [Default: \\'continue\\']\\nresult_filter : callable or None, optional\\n  if given, each to-be-returned status dictionary is passed to this\\n  callable, and is only returned if the callable\\'s return value does\\n  not evaluate to False or a ValueError exception is raised. If the\\n  given callable supports `**kwargs` it will additionally be passed\\n  the keyword arguments of the original API call. [Default: None]\\nresult_renderer : {\\'default\\', \\'json\\', \\'json_pp\\', \\'tailored\\'} or None, optional\\n  format of return value rendering on stdout. [Default: None]\\nresult_xfm : {\\'datasets\\', \\'successdatasets-or-none\\', \\'paths\\', \\'relpaths\\', \\'metadata\\'} or callable or None, optional\\n  if given, each to-be-returned result status dictionary is passed to\\n  this callable, and its return value becomes the result instead. This\\n  is different from `result_filter`, as it can perform arbitrary\\n  transformation of the result value. This is mostly useful for top-\\n  level command invocations that need to provide the results in a\\n  particular format. Instead of a callable, a label for a pre-crafted\\n  result transformation can be given. [Default: None]\\nreturn_type : {\\'generator\\', \\'list\\', \\'item-or-list\\'}, optional\\n  return value behavior switch. If \\'item-or-list\\' a single value is\\n  returned instead of a one-item return value list, or a list in case\\n  of multiple return values. `None` is return in case of an empty\\n  list. [Default: \\'list\\']\\n'\n\n../../../../.local/share/uv/python/cpython-3.9.21-linux-x86_64-gnu/lib/python3.9/unittest/case.py:676: AssertionError\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.utils:utils.py:2493 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\n==================================== PASSES ====================================\n_________________________ test_utils_suppress_similar __________________________\n------------------------------ Captured log call -------------------------------\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_result_filter\nPASSED r2e_tests/test_1.py::test_utils_suppress_similar\nERROR r2e_tests/test_1.py::test_discover_ds_trace\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\nFAILED r2e_tests/test_1.py::test_eval_results_plus_build_doc - AssertionError...\n===================== 3 failed, 2 passed, 1 error in 0.82s =====================\n",
    "old_commit_res_stderr": ""
}
datalad/interface/base.py datalad/interface/tests/test_utils.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Documentation Does Not Reflect Overridden Default Options in `eval_params`

**Description:**
When overriding default options in `eval_params`, the generated documentation fails to display the updated default values. Specifically, overriding attributes like `result_renderer` and `return_type` should update their default values in the documentation, but they remain unchanged.

**Example Code:**
```python
class TestUtils(Interface):
    result_renderer = 'tailored'  # Overrides the default value
    return_type = 'item-or-list'  # Overrides the default value

    _params_ = dict(
        number=Parameter(
            args=("-n", "--number"),
            doc="It's a number.",
            default='list'
        )
    )
```

**Expected Behavior:**
The documentation should display the overridden default values for `result_renderer` and `return_type`. For example:
- `result_renderer`: Default: `'tailored'`
- `return_type`: Default: `'item-or-list'`

**Actual Behavior:**
The documentation continues to show the original default values instead of the overridden ones:
- `result_renderer`: Default: `None`
- `return_type`: Default: `'list'`

This discrepancy causes confusion and leads to failing tests that expect the documentation to reflect the overridden defaults.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/base.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""High-level interface generation

"""

__docformat__ = 'restructuredtext'

import logging
lgr = logging.getLogger('datalad.interface.base')

import os
import sys
import re
import textwrap
from importlib import import_module
import inspect
import string
from collections import (
    defaultdict,
    OrderedDict,
)

from ..ui import ui
from ..dochelpers import exc_str

from datalad.interface.common_opts import eval_params
from datalad.interface.common_opts import eval_defaults
from datalad.support.constraints import (
    EnsureKeyChoice,
    EnsureChoice,
)
from datalad.distribution.dataset import Dataset
from datalad.distribution.dataset import resolve_path
from datalad.plugin import _get_plugins
from datalad.plugin import _load_plugin


default_logchannels = {
    '': 'debug',
    'ok': 'debug',
    'notneeded': 'debug',
    'impossible': 'warning',
    'error': 'error',
}


def get_api_name(intfspec):
    """Given an interface specification return an API name for it"""
    if len(intfspec) > 3:
        name = intfspec[3]
    else:
        name = intfspec[0].split('.')[-1]
    return name


def get_cmdline_command_name(intfspec):
    """Given an interface specification return a cmdline command name"""
    if len(intfspec) > 2:
        name = intfspec[2]
    else:
        name = intfspec[0].split('.')[-1].replace('_', '-')
    return name


def get_interface_groups(include_plugins=False):
    """Return a list of command groups.

    Parameters
    ----------
    include_plugins : bool, optional
        Whether to include a group named 'plugins' that has a list of
        discovered plugin commands.

    Returns
    -------
    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).
    """
    from .. import interface as _interfaces

    grps = []
    # auto detect all available interfaces and generate a function-based
    # API from them
    for _item in _interfaces.__dict__:
        if not _item.startswith('_group_'):
            continue
        grp_name = _item[7:]
        grp = getattr(_interfaces, _item)
        grps.append((grp_name,) + grp)
    # TODO(yoh): see if we could retain "generator" for plugins
    # ATM we need to make it explicit so we could check the command(s) below
    # It could at least follow the same destiny as extensions so we would
    # just do more iterative "load ups"

    if include_plugins:
        grps.append(('plugins', 'Plugins', list(_get_plugins())))
    return grps


def get_cmd_summaries(descriptions, groups, width=79):
    """Return summaries for the commands in `groups`.

    Parameters
    ----------
    descriptions : dict
        A map of group names to summaries.
    groups : list of tuples
        A list of groups and commands in the form described by
        `get_interface_groups`.
    width : int, optional
        The maximum width of each line in the summary text.

    Returns
    -------
    A list with a formatted entry for each command. The first command of each
    group is preceded by an entry describing the group.
    """
    cmd_summary = []
    for grp in sorted(groups, key=lambda x: x[1]):
        grp_descr = grp[1]
        grp_cmds = descriptions[grp[0]]

        cmd_summary.append('\n*%s*\n' % (grp_descr,))
        for cd in grp_cmds:
            cmd_summary.append('  %s\n%s'
                               % ((cd[0],
                                   textwrap.fill(
                                       cd[1].rstrip(' .'),
                                       width - 5,
                                       initial_indent=' ' * 6,
                                       subsequent_indent=' ' * 6))))
    return cmd_summary


def load_interface(spec):
    """Load and return the class for `spec`.

    Parameters
    ----------
    spec : tuple
        For a standard interface, the first item is the datalad source module
        and the second object name for the interface. For a plugin, the second
        item should be a dictionary that maps 'file' to the path the of module.

    Returns
    -------
    The interface class or, if importing the module fails, None.
    """
    if isinstance(spec[1], dict):
        intf = _load_plugin(spec[1]['file'], fail=False)
    else:
        lgr.log(5, "Importing module %s " % spec[0])
        try:
            mod = import_module(spec[0], package='datalad')
        except Exception as e:
            lgr.error("Internal error, cannot import interface '%s': %s",
                      spec[0], exc_str(e))
            intf = None
        else:
            intf = getattr(mod, spec[1])
    return intf


def get_cmd_doc(interface):
    """Return the documentation for the command defined by `interface`.

    Parameters
    ----------
    interface : subclass of Interface
    """
    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()
    if hasattr(interface, '_docs_'):
        # expand docs
        intf_doc = intf_doc.format(**interface._docs_)
    return intf_doc


def get_cmd_ex(interface):
    """Return the examples for the command defined by 'interface'.

    Parameters
    ----------
    interface : subclass of Interface
    """
    intf_ex = "\n\n*Examples*\n\n"
    for example in interface._examples_:
        intf_ex += build_example(example, api='cmdline')
    return intf_ex


def dedent_docstring(text):
    """Remove uniform indentation from a multiline docstring"""
    # Problem is that first line might often have no offset, so might
    # need to be ignored from dedent call
    if text is None:
        return None
    if not text.startswith(' '):
        lines = text.split('\n')
        if len(lines) == 1:
            # single line, no indentation, nothing to do
            return text
        text2 = '\n'.join(lines[1:])
        return lines[0] + "\n" + textwrap.dedent(text2)
    else:
        return textwrap.dedent(text)


def alter_interface_docs_for_api(docs):
    """Apply modifications to interface docstrings for Python API use."""
    # central place to alter the impression of docstrings,
    # like removing cmdline specific sections
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        r'\|\| CMDLINE \>\>.*?\<\< CMDLINE \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        r'\[CMD:\s[^\[\]]*\sCMD\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    docs = re.sub(
        r'\[PY:\s([^\[\]]*)\sPY\]',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE)
    docs = re.sub(
        r'\|\| PYTHON \>\>(.*?)\<\< PYTHON \|\|',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    if 'DATALAD_SPHINX_RUN' not in os.environ:
        # remove :role:`...` RST markup for cmdline docs
        docs = re.sub(
            r':\S+:`[^`]*`[\\]*',
            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\'),
            docs,
            flags=re.MULTILINE | re.DOTALL)
        # make the handbook doc references more accessible
        # the URL is a redirect configured at readthedocs
        docs = re.sub(
            r'(handbook:[0-9]-[0-9]*)',
            '\\1 (http://handbook.datalad.org/symbols)',
            docs)
    docs = re.sub(
        r'\|\| REFLOW \>\>\n(.*?)\<\< REFLOW \|\|',
        lambda match: textwrap.fill(match.group(1)),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    return docs


def alter_interface_docs_for_cmdline(docs):
    """Apply modifications to interface docstrings for cmdline doc use."""
    # central place to alter the impression of docstrings,
    # like removing Python API specific sections, and argument markup
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        r'\|\| PYTHON \>\>.*?\<\< PYTHON \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        r'\[PY:\s[^\[\]]*\sPY\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    docs = re.sub(
        r'\[CMD:\s([^\[\]]*)\sCMD\]',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE)
    docs = re.sub(
        r'\|\| CMDLINE \>\>(.*?)\<\< CMDLINE \|\|',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # remove :role:`...` RST markup for cmdline docs
    docs = re.sub(
        r':\S+:`[^`]*`[\\]*',
        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\'),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # make the handbook doc references more accessible
    # the URL is a redirect configured at readthedocs
    docs = re.sub(
        r'(handbook:[0-9]-[0-9]*)',
        '\\1 (http://handbook.datalad.org/symbols)',
        docs)
    # remove None constraint. In general, `None` on the cmdline means don't
    # give option at all, but specifying `None` explicitly is practically
    # impossible
    docs = re.sub(
        r',\sor\svalue\smust\sbe\s`None`',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # capitalize variables and remove backticks to uniformize with
    # argparse output
    docs = re.sub(
        r'`\S*`',
        lambda match: match.group(0).strip('`').upper(),
        docs)
    # clean up sphinx API refs
    docs = re.sub(
        r'\~datalad\.api\.\S*',
        lambda match: "`{0}`".format(match.group(0)[13:]),
        docs)
    # Remove RST paragraph markup
    docs = re.sub(
        r'^.. \S+::',
        lambda match: match.group(0)[3:-2].upper(),
        docs,
        flags=re.MULTILINE)
    docs = re.sub(
        r'\|\| REFLOW \>\>\n(.*?)\<\< REFLOW \|\|',
        lambda match: textwrap.fill(match.group(1)),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    return docs


def is_api_arg(arg):
    """Return True if argument is our API argument or self or used for internal
    purposes
    """
    return arg != 'self' and not arg.startswith('_')


def update_docstring_with_parameters(func, params, prefix=None, suffix=None,
                                     add_args=None):
    """Generate a useful docstring from a parameter spec

    Amends any existing docstring of a callable with a textual
    description of its parameters. The Parameter spec needs to match
    the number and names of the callables arguments.
    """
    from datalad.utils import getargspec
    # get the signature
    ndefaults = 0
    args, varargs, varkw, defaults = getargspec(func)
    if add_args:
        add_argnames = sorted(add_args.keys())
        args.extend(add_argnames)
        defaults = defaults + tuple(add_args[k] for k in add_argnames)
    if defaults is not None:
        ndefaults = len(defaults)
    # start documentation with what the callable brings with it
    doc = prefix if prefix else u''
    if len(args) > 1:
        if len(doc):
            doc += '\n'
        doc += "Parameters\n----------\n"
        for i, arg in enumerate(args):
            if not is_api_arg(arg):
                continue
            # we need a parameter spec for each argument
            if not arg in params:
                raise ValueError("function has argument '%s' not described as a parameter" % arg)
            param = params[arg]
            # validate the default -- to make sure that the parameter description is
            # somewhat OK
            defaults_idx = ndefaults - len(args) + i
            if defaults_idx >= 0:
                if param.constraints is not None:
                    param.constraints(defaults[defaults_idx])
            orig_docs = param._doc
            param._doc = alter_interface_docs_for_api(param._doc)
            doc += param.get_autodoc(
                arg,
                default=defaults[defaults_idx] if defaults_idx >= 0 else None,
                has_default=defaults_idx >= 0)
            param._doc = orig_docs
            doc += '\n'
    doc += suffix if suffix else u""
    # assign the amended docs
    func.__doc__ = doc
    return func


def build_example(example, api='python'):
    """Build a code example.

    Take a dict from a classes _example_ specification (list of dicts) and
    build a string with an api or cmd example (for use in cmd help or
    docstring).

    Parameters
    ----------
    api : {'python', 'cmdline'}
        If 'python', build Python example for docstring. If 'cmdline', build
        cmd example.

    Returns
    -------
    ex : str
        Concatenated examples for the given class.
    """
    if api == 'python' :
        code_field='code_py'
        indicator='>'
    elif api == 'cmdline':
        code_field='code_cmd'
        indicator='%'
    else:
        raise ValueError("unknown API selection: {}".format(api))
    if code_field not in example:
        # only show an example if it exist for the API
        return ''
    description = textwrap.fill(example.get('text'))
    # this indent the code snippet to get it properly rendered as code
    # we are not using textwrap.fill(), because it would not acknowledge
    # any meaningful structure/formatting of code snippets. Instead, we
    # maintain line content as is.
    code = dedent_docstring(example.get(code_field))
    needs_indicator = not code.startswith(indicator)
    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()

    ex = """{}::\n\n   {}{}\n\n""".format(
        description,
        # disable automatic prefixing, if the example already has one
        # this enables providing more complex examples without having
        # to infer its inner structure
        '{} '.format(indicator)
        if needs_indicator
        # maintain spacing to avoid undesired relative indentation
        else '',
        code)

    return ex


def update_docstring_with_examples(cls_doc, ex):
    """Update a commands docstring with examples.

    Take _examples_ of a command, build the Python examples, and append
    them to the docstring.

    cls_doc: docstring
    ex: list
        list of dicts with examples
    """
    from textwrap import indent
    if len(cls_doc):
        cls_doc += "\n"
    cls_doc += "    Examples\n    --------\n"
    # loop though provided examples
    for example in ex:
        cls_doc += indent(build_example(example, api='python'), ' '*4)

    return cls_doc


def build_doc(cls, **kwargs):
    """Decorator to build docstrings for datalad commands

    It's intended to decorate the class, the __call__-method of which is the
    actual command. It expects that __call__-method to be decorated by
    eval_results.

    Note that values for any `eval_params` keys in `cls._params_` are
    ignored.  This means one class may extend another's `_params_`
    without worrying about filtering out `eval_params`.

    Parameters
    ----------
    cls: Interface
      class defining a datalad command
    """

    # Note, that this is a class decorator, which is executed only once when the
    # class is imported. It builds the docstring for the class' __call__ method
    # and returns the original class.
    #
    # This is because a decorator for the actual function would not be able to
    # behave like this. To build the docstring we need to access the attribute
    # _params of the class. From within a function decorator we cannot do this
    # during import time, since the class is being built in this very moment and
    # is not yet available in the module. And if we do it from within the part
    # of a function decorator, that is executed when the function is called, we
    # would need to actually call the command once in order to build this
    # docstring.

    lgr.debug("Building doc for {}".format(cls))

    cls_doc = cls.__doc__
    if hasattr(cls, '_docs_'):
        # expand docs
        cls_doc = cls_doc.format(**cls._docs_)
    # get examples
    ex = getattr(cls, '_examples_', [])
    if ex:
        cls_doc = update_docstring_with_examples(cls_doc, ex)

    call_doc = None
    # suffix for update_docstring_with_parameters:
    if cls.__call__.__doc__:
        call_doc = cls.__call__.__doc__

    # build standard doc and insert eval_doc
    spec = getattr(cls, '_params_', dict())
    # ATTN: An important consequence of this update() call is that it
    # fulfills the docstring's promise of overriding any existing
    # values for eval_params keys in _params_.
    #
    # get docs for eval_results parameters:
    spec.update(eval_params)

    update_docstring_with_parameters(
        cls.__call__, spec,
        prefix=alter_interface_docs_for_api(cls_doc),
        suffix=alter_interface_docs_for_api(call_doc),
        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None
    )

    # return original
    return cls


NA_STRING = 'N/A'  # we might want to make it configurable via config


class nagen(object):
    """A helper to provide a desired missing value if no value is known

    Usecases
    - could be used as a generator for `defaultdict`
    - since it returns itself upon getitem, should work even for complex
      nested dictionaries/lists .format templates
    """
    def __init__(self, missing=NA_STRING):
        self.missing = missing

    def __repr__(self):
        cls = self.__class__.__name__
        args = str(self.missing) if self.missing != NA_STRING else ''
        return '%s(%s)' % (cls, args)

    def __str__(self):
        return self.missing

    def __getitem__(self, *args):
        return self

    def __getattr__(self, item):
        return self


def nadict(*items):
    """A generator of default dictionary with the default nagen"""
    dd = defaultdict(nagen)
    dd.update(*items)
    return dd


class DefaultOutputFormatter(string.Formatter):
    """A custom formatter for default output rendering using .format
    """
    # TODO: make missing configurable?
    def __init__(self, missing=nagen()):
        """
        Parameters
        ----------
        missing: string, optional
          What to output for the missing values
        """
        super(DefaultOutputFormatter, self).__init__()
        self.missing = missing

    def _d(self, msg, *args):
        # print("   HERE %s" % (msg % args))
        pass

    def get_value(self, key, args, kwds):
        assert not args
        self._d("get_value: %r %r %r", key, args, kwds)
        return kwds.get(key, self.missing)

    # def get_field(self, field_name, args, kwds):
    #     assert not args
    #     self._d("get_field: %r args=%r kwds=%r" % (field_name, args, kwds))
    #     try:
    #         out = string.Formatter.get_field(self, field_name, args, kwds)
    #     except Exception as exc:
    #         # TODO needs more than just a value
    #         return "!ERR %s" % exc


class DefaultOutputRenderer(object):
    """A default renderer for .format'ed output line
    """
    def __init__(self, format):
        self.format = format
        # We still need custom output formatter since at the "first level"
        # within .format template all items there is no `nadict`
        self.formatter = DefaultOutputFormatter()

    @classmethod
    def _dict_to_nadict(cls, v):
        """Traverse datastructure and replace any regular dict with nadict"""
        if isinstance(v, list):
            return [cls._dict_to_nadict(x) for x in v]
        elif isinstance(v, dict):
            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())
        else:
            return v

    def __call__(self, x, **kwargs):
        dd = nadict(
            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)
                for k_, v_ in v.items()})
                if isinstance(v, dict) else v)
            for k, v in x.items()
        )

        msg = self.formatter.format(self.format, **dd)
        return ui.message(msg)


class Interface(object):
    """Base class for interface implementations"""

    # exit code to return if user-interrupted
    # if None, would just reraise the Exception, so if in --dbg
    # mode would fall into the debugger
    _interrupted_exit_code = 1

    _OLDSTYLE_COMMANDS = (
        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',
        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')

    @classmethod
    def setup_parser(cls, parser):
        # XXX needs safety check for name collisions
        # XXX allow for parser kwargs customization
        parser_kwargs = {}
        from datalad.utils import getargspec
        # get the signature
        ndefaults = 0
        args, varargs, varkw, defaults = getargspec(cls.__call__)
        if not defaults is None:
            ndefaults = len(defaults)
        for i, arg in enumerate(args):
            if not is_api_arg(arg):
                continue
            param = cls._params_[arg]
            defaults_idx = ndefaults - len(args) + i
            cmd_args = param.cmd_args
            if cmd_args == tuple():
                # explicitly provided an empty sequence of argument names
                # this shall not appear in the parser
                continue
            elif cmd_args is None:
                cmd_args = []
            if not len(cmd_args):
                if defaults_idx >= 0:
                    # dealing with a kwarg
                    template = '--%s'
                else:
                    # positional arg
                    template = '%s'
                # use parameter name as default argument name
                parser_args = (template % arg.replace('_', '-'),)
            else:
                parser_args = [c.replace('_', '-') for c in cmd_args]
            parser_kwargs = param.cmd_kwargs
            if defaults_idx >= 0:
                parser_kwargs['default'] = defaults[defaults_idx]
            help = alter_interface_docs_for_cmdline(param._doc)
            if help and help.rstrip()[-1] != '.':
                help = help.rstrip() + '.'
            if param.constraints is not None:
                parser_kwargs['type'] = param.constraints
                # include value constraint description and default
                # into the help string
                cdoc = alter_interface_docs_for_cmdline(
                    param.constraints.long_description())
                if cdoc[0] == '(' and cdoc[-1] == ')':
                    cdoc = cdoc[1:-1]
                help += '  Constraints: %s' % cdoc
                if 'metavar' not in parser_kwargs and \
                        isinstance(param.constraints, EnsureChoice):
                    parser_kwargs['metavar'] = \
                        '{%s}' % '|'.join(
                            # don't use short_description(), because
                            # it also needs to give valid output for
                            # Python syntax (quotes...), but here we
                            # can simplify to shell syntax where everything
                            # is a string
                            p for p in param.constraints._allowed
                            # in the cmdline None pretty much means
                            # don't give the options, so listing it
                            # doesn't make sense. Moreover, any non-string
                            # value cannot be given and very likely only
                            # serves a special purpose in the Python API
                            # or implementation details
                            if isinstance(p, str))
            if defaults_idx >= 0:
                # if it is a flag, in commandline it makes little sense to show
                # showing the Default: (likely boolean).
                #   See https://github.com/datalad/datalad/issues/3203
                if not parser_kwargs.get('action', '').startswith('store_'):
                    # [Default: None] also makes little sense for cmdline
                    if defaults[defaults_idx] is not None:
                        help += " [Default: %r]" % (defaults[defaults_idx],)
            # create the parameter, using the constraint instance for type
            # conversion
            parser.add_argument(*parser_args, help=help,
                                **parser_kwargs)

    @classmethod
    def call_from_parser(cls, args):
        # XXX needs safety check for name collisions
        from datalad.utils import getargspec
        argspec = getargspec(cls.__call__)
        if argspec[2] is None:
            # no **kwargs in the call receiver, pull argnames from signature
            argnames = getargspec(cls.__call__)[0]
        else:
            # common options
            # XXX define or better get from elsewhere
            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',
                           'help', 'log_level', 'logger', 'pbs_runner',
                           'result_renderer', 'subparser')
            argnames = [name for name in dir(args)
                        if not (name.startswith('_') or name in common_opts)]
        kwargs = {k: getattr(args, k)
                  for k in argnames
                  # some arguments might be Python-only and do not appear in the
                  # parser Namespace
                  if hasattr(args, k) and is_api_arg(k)}
        # we are coming from the entry point, this is the toplevel command,
        # let it run like generator so we can act on partial results quicker
        # TODO remove following condition test when transition is complete and
        # run indented code unconditionally
        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:
            # set all common args explicitly  to override class defaults
            # that are tailored towards the the Python API
            kwargs['return_type'] = 'generator'
            kwargs['result_xfm'] = None
            # allow commands to override the default, unless something other than
            # default is requested
            kwargs['result_renderer'] = \
                args.common_output_format if args.common_output_format != 'default' \
                else getattr(cls, 'result_renderer', args.common_output_format)
            if '{' in args.common_output_format:
                # stupid hack, could and should become more powerful
                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)

            if args.common_on_failure:
                kwargs['on_failure'] = args.common_on_failure
            # compose filter function from to be invented cmdline options
            res_filter = cls._get_result_filter(args)
            if res_filter is not None:
                # Don't add result_filter if it's None because then
                # eval_results can't distinguish between --report-{status,type}
                # not specified via the CLI and None passed via the Python API.
                kwargs['result_filter'] = res_filter
        try:
            ret = cls.__call__(**kwargs)
            if inspect.isgenerator(ret):
                ret = list(ret)
            return ret
        except KeyboardInterrupt as exc:
            ui.error("\nInterrupted by user while doing magic: %s" % exc_str(exc))
            if cls._interrupted_exit_code is not None:
                sys.exit(cls._interrupted_exit_code)
            else:
                raise

    @classmethod
    def _get_result_filter(cls, args):
        from datalad import cfg
        result_filter = None
        if args.common_report_status or 'datalad.runtime.report-status' in cfg:
            report_status = args.common_report_status or \
                            cfg.obtain('datalad.runtime.report-status')
            if report_status == "all":
                pass  # no filter
            elif report_status == 'success':
                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))
            elif report_status == 'failure':
                result_filter = EnsureKeyChoice('status',
                                                ('impossible', 'error'))
            else:
                result_filter = EnsureKeyChoice('status', (report_status,))
        if args.common_report_type:
            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))
            result_filter = result_filter & tfilt if result_filter else tfilt
        return result_filter

    @classmethod
    def get_refds_path(cls, dataset):
        """Return a resolved reference dataset path from a `dataset` argument"""
        # theoretically a dataset could come in as a relative path -> resolve
        if dataset is None:
            return dataset
        refds_path = dataset.path if isinstance(dataset, Dataset) \
            else Dataset(dataset).path
        if refds_path:
            refds_path = str(resolve_path(refds_path))
        return refds_path


def get_allargs_as_kwargs(call, args, kwargs):
    """Generate a kwargs dict from a call signature and *args, **kwargs

    Basically resolving the argnames for all positional arguments, and
    resolvin the defaults for all kwargs that are not given in a kwargs
    dict
    """
    from datalad.utils import getargspec
    argspec = getargspec(call)
    defaults = argspec.defaults
    nargs = len(argspec.args)
    assert (nargs >= len(defaults))
    # map any args to their name
    argmap = list(zip(argspec.args[:len(args)], args))
    kwargs_ = OrderedDict(argmap)
    # map defaults of kwargs to their names (update below)
    for k, v in zip(argspec.args[-len(defaults):], defaults):
        if k not in kwargs_:
            kwargs_[k] = v
    # update with provided kwarg args
    kwargs_.update(kwargs)
    # XXX we cannot assert the following, because our own highlevel
    # API commands support more kwargs than what is discoverable
    # from their signature...
    #assert (nargs == len(kwargs_))
    return kwargs_

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Documentation Does Not Reflect Overridden Default Options in `eval_params`

**Description:**
When overriding default options in `eval_params`, the generated documentation fails to display the updated default values. Specifically, overriding attributes like `result_renderer` and `return_type` should update their default values in the documentation, but they remain unchanged.

**Example Code:**
```python
class TestUtils(Interface):
    result_renderer = 'tailored'  # Overrides the default value
    return_type = 'item-or-list'  # Overrides the default value

    _params_ = dict(
        number=Parameter(
            args=("-n", "--number"),
            doc="It's a number.",
            default='list'
        )
    )
```

**Expected Behavior:**
The documentation should display the overridden default values for `result_renderer` and `return_type`. For example:
- `result_renderer`: Default: `'tailored'`
- `return_type`: Default: `'item-or-list'`

**Actual Behavior:**
The documentation continues to show the original default values instead of the overridden ones:
- `result_renderer`: Default: `None`
- `return_type`: Default: `'list'`

This discrepancy causes confusion and leads to failing tests that expect the documentation to reflect the overridden defaults.
[/ISSUE]
{
    "test_eval_results_plus_build_doc": "PASSED",
    "test_result_filter": "PASSED",
    "test_utils_suppress_similar": "PASSED",
    "test_discover_ds_trace": "ERROR",
    "test_dirty": "FAILED",
    "test_save_hierarchy": "FAILED"
}
diff --git a/datalad/interface/base.py b/datalad/interface/base.py
index f9be2c566..62abc6e72 100644
--- a/datalad/interface/base.py
+++ b/datalad/interface/base.py
@@ -514,10 +514,19 @@ def build_doc(cls, **kwargs):
 
     # build standard doc and insert eval_doc
     spec = getattr(cls, '_params_', dict())
+
+
+    # update class attributes that may override defaults 
+    if hasattr(cls, '_no_eval_results'):
+        add_args = None
+    else:
+        add_args = {k: getattr(cls, k, v) for k, v in eval_defaults.items()}
+
     # ATTN: An important consequence of this update() call is that it
     # fulfills the docstring's promise of overriding any existing
     # values for eval_params keys in _params_.
     #
+
     # get docs for eval_results parameters:
     spec.update(eval_params)
 
@@ -525,7 +534,7 @@ def build_doc(cls, **kwargs):
         cls.__call__, spec,
         prefix=alter_interface_docs_for_api(cls_doc),
         suffix=alter_interface_docs_for_api(call_doc),
-        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None
+        add_args=add_args
     )
 
     # return original

{"datalad/interface/base.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport os\nimport sys\nimport re\nimport textwrap\nfrom importlib import import_module\nimport inspect\nimport string\nfrom collections import (\n    defaultdict,\n    OrderedDict,\n)\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureChoice,\n)\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.plugin import _get_plugins\nfrom datalad.plugin import _load_plugin\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups(include_plugins=False):\n    \"\"\"Return a list of command groups.\n\n    Parameters\n    ----------\n    include_plugins : bool, optional\n        Whether to include a group named 'plugins' that has a list of\n        discovered plugin commands.\n\n    Returns\n    -------\n    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).\n    \"\"\"\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    # TODO(yoh): see if we could retain \"generator\" for plugins\n    # ATM we need to make it explicit so we could check the command(s) below\n    # It could at least follow the same destiny as extensions so we would\n    # just do more iterative \"load ups\"\n\n    if include_plugins:\n        grps.append(('plugins', 'Plugins', list(_get_plugins())))\n    return grps\n\n\ndef get_cmd_summaries(descriptions, groups, width=79):\n    \"\"\"Return summaries for the commands in `groups`.\n\n    Parameters\n    ----------\n    descriptions : dict\n        A map of group names to summaries.\n    groups : list of tuples\n        A list of groups and commands in the form described by\n        `get_interface_groups`.\n    width : int, optional\n        The maximum width of each line in the summary text.\n\n    Returns\n    -------\n    A list with a formatted entry for each command. The first command of each\n    group is preceded by an entry describing the group.\n    \"\"\"\n    cmd_summary = []\n    for grp in sorted(groups, key=lambda x: x[1]):\n        grp_descr = grp[1]\n        grp_cmds = descriptions[grp[0]]\n\n        cmd_summary.append('\\n*%s*\\n' % (grp_descr,))\n        for cd in grp_cmds:\n            cmd_summary.append('  %s\\n%s'\n                               % ((cd[0],\n                                   textwrap.fill(\n                                       cd[1].rstrip(' .'),\n                                       width - 5,\n                                       initial_indent=' ' * 6,\n                                       subsequent_indent=' ' * 6))))\n    return cmd_summary\n\n\ndef load_interface(spec):\n    \"\"\"Load and return the class for `spec`.\n\n    Parameters\n    ----------\n    spec : tuple\n        For a standard interface, the first item is the datalad source module\n        and the second object name for the interface. For a plugin, the second\n        item should be a dictionary that maps 'file' to the path the of module.\n\n    Returns\n    -------\n    The interface class or, if importing the module fails, None.\n    \"\"\"\n    if isinstance(spec[1], dict):\n        intf = _load_plugin(spec[1]['file'], fail=False)\n    else:\n        lgr.log(5, \"Importing module %s \" % spec[0])\n        try:\n            mod = import_module(spec[0], package='datalad')\n        except Exception as e:\n            lgr.error(\"Internal error, cannot import interface '%s': %s\",\n                      spec[0], exc_str(e))\n            intf = None\n        else:\n            intf = getattr(mod, spec[1])\n    return intf\n\n\ndef get_cmd_doc(interface):\n    \"\"\"Return the documentation for the command defined by `interface`.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()\n    if hasattr(interface, '_docs_'):\n        # expand docs\n        intf_doc = intf_doc.format(**interface._docs_)\n    return intf_doc\n\n\ndef get_cmd_ex(interface):\n    \"\"\"Return the examples for the command defined by 'interface'.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_ex = \"\\n\\n*Examples*\\n\\n\"\n    for example in interface._examples_:\n        intf_ex += build_example(example, api='cmdline')\n    return intf_ex\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>.*?\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>(.*?)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    if 'DATALAD_SPHINX_RUN' not in os.environ:\n        # remove :role:`...` RST markup for cmdline docs\n        docs = re.sub(\n            r':\\S+:`[^`]*`[\\\\]*',\n            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n            docs,\n            flags=re.MULTILINE | re.DOTALL)\n        # make the handbook doc references more accessible\n        # the URL is a redirect configured at readthedocs\n        docs = re.sub(\n            r'(handbook:[0-9]-[0-9]*)',\n            '\\\\1 (http://handbook.datalad.org/symbols)',\n            docs)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>.*?\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>(.*?)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # make the handbook doc references more accessible\n    # the URL is a redirect configured at readthedocs\n    docs = re.sub(\n        r'(handbook:[0-9]-[0-9]*)',\n        '\\\\1 (http://handbook.datalad.org/symbols)',\n        docs)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        r',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        r'`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        r'\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from datalad.utils import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_example(example, api='python'):\n    \"\"\"Build a code example.\n\n    Take a dict from a classes _example_ specification (list of dicts) and\n    build a string with an api or cmd example (for use in cmd help or\n    docstring).\n\n    Parameters\n    ----------\n    api : {'python', 'cmdline'}\n        If 'python', build Python example for docstring. If 'cmdline', build\n        cmd example.\n\n    Returns\n    -------\n    ex : str\n        Concatenated examples for the given class.\n    \"\"\"\n    if api == 'python' :\n        code_field='code_py'\n        indicator='>'\n    elif api == 'cmdline':\n        code_field='code_cmd'\n        indicator='%'\n    else:\n        raise ValueError(\"unknown API selection: {}\".format(api))\n    if code_field not in example:\n        # only show an example if it exist for the API\n        return ''\n    description = textwrap.fill(example.get('text'))\n    # this indent the code snippet to get it properly rendered as code\n    # we are not using textwrap.fill(), because it would not acknowledge\n    # any meaningful structure/formatting of code snippets. Instead, we\n    # maintain line content as is.\n    code = dedent_docstring(example.get(code_field))\n    needs_indicator = not code.startswith(indicator)\n    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()\n\n    ex = \"\"\"{}::\\n\\n   {}{}\\n\\n\"\"\".format(\n        description,\n        # disable automatic prefixing, if the example already has one\n        # this enables providing more complex examples without having\n        # to infer its inner structure\n        '{} '.format(indicator)\n        if needs_indicator\n        # maintain spacing to avoid undesired relative indentation\n        else '',\n        code)\n\n    return ex\n\n\ndef update_docstring_with_examples(cls_doc, ex):\n    \"\"\"Update a commands docstring with examples.\n\n    Take _examples_ of a command, build the Python examples, and append\n    them to the docstring.\n\n    cls_doc: docstring\n    ex: list\n        list of dicts with examples\n    \"\"\"\n    from textwrap import indent\n    if len(cls_doc):\n        cls_doc += \"\\n\"\n    cls_doc += \"    Examples\\n    --------\\n\"\n    # loop though provided examples\n    for example in ex:\n        cls_doc += indent(build_example(example, api='python'), ' '*4)\n\n    return cls_doc\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None\n    )\n\n    # return original\n    return cls\n\n\nNA_STRING = 'N/A'  # we might want to make it configurable via config\n\n\nclass nagen(object):\n    \"\"\"A helper to provide a desired missing value if no value is known\n\n    Usecases\n    - could be used as a generator for `defaultdict`\n    - since it returns itself upon getitem, should work even for complex\n      nested dictionaries/lists .format templates\n    \"\"\"\n    def __init__(self, missing=NA_STRING):\n        self.missing = missing\n\n    def __repr__(self):\n        cls = self.__class__.__name__\n        args = str(self.missing) if self.missing != NA_STRING else ''\n        return '%s(%s)' % (cls, args)\n\n    def __str__(self):\n        return self.missing\n\n    def __getitem__(self, *args):\n        return self\n\n    def __getattr__(self, item):\n        return self\n\n\ndef nadict(*items):\n    \"\"\"A generator of default dictionary with the default nagen\"\"\"\n    dd = defaultdict(nagen)\n    dd.update(*items)\n    return dd\n\n\nclass DefaultOutputFormatter(string.Formatter):\n    \"\"\"A custom formatter for default output rendering using .format\n    \"\"\"\n    # TODO: make missing configurable?\n    def __init__(self, missing=nagen()):\n        \"\"\"\n        Parameters\n        ----------\n        missing: string, optional\n          What to output for the missing values\n        \"\"\"\n        super(DefaultOutputFormatter, self).__init__()\n        self.missing = missing\n\n    def _d(self, msg, *args):\n        # print(\"   HERE %s\" % (msg % args))\n        pass\n\n    def get_value(self, key, args, kwds):\n        assert not args\n        self._d(\"get_value: %r %r %r\", key, args, kwds)\n        return kwds.get(key, self.missing)\n\n    # def get_field(self, field_name, args, kwds):\n    #     assert not args\n    #     self._d(\"get_field: %r args=%r kwds=%r\" % (field_name, args, kwds))\n    #     try:\n    #         out = string.Formatter.get_field(self, field_name, args, kwds)\n    #     except Exception as exc:\n    #         # TODO needs more than just a value\n    #         return \"!ERR %s\" % exc\n\n\nclass DefaultOutputRenderer(object):\n    \"\"\"A default renderer for .format'ed output line\n    \"\"\"\n    def __init__(self, format):\n        self.format = format\n        # We still need custom output formatter since at the \"first level\"\n        # within .format template all items there is no `nadict`\n        self.formatter = DefaultOutputFormatter()\n\n    @classmethod\n    def _dict_to_nadict(cls, v):\n        \"\"\"Traverse datastructure and replace any regular dict with nadict\"\"\"\n        if isinstance(v, list):\n            return [cls._dict_to_nadict(x) for x in v]\n        elif isinstance(v, dict):\n            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())\n        else:\n            return v\n\n    def __call__(self, x, **kwargs):\n        dd = nadict(\n            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)\n                for k_, v_ in v.items()})\n                if isinstance(v, dict) else v)\n            for k, v in x.items()\n        )\n\n        msg = self.formatter.format(self.format, **dd)\n        return ui.message(msg)\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from datalad.utils import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args == tuple():\n                # explicitly provided an empty sequence of argument names\n                # this shall not appear in the parser\n                continue\n            elif cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help.rstrip()[-1] != '.':\n                help = help.rstrip() + '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n                if 'metavar' not in parser_kwargs and \\\n                        isinstance(param.constraints, EnsureChoice):\n                    parser_kwargs['metavar'] = \\\n                        '{%s}' % '|'.join(\n                            # don't use short_description(), because\n                            # it also needs to give valid output for\n                            # Python syntax (quotes...), but here we\n                            # can simplify to shell syntax where everything\n                            # is a string\n                            p for p in param.constraints._allowed\n                            # in the cmdline None pretty much means\n                            # don't give the options, so listing it\n                            # doesn't make sense. Moreover, any non-string\n                            # value cannot be given and very likely only\n                            # serves a special purpose in the Python API\n                            # or implementation details\n                            if isinstance(p, str))\n            if defaults_idx >= 0:\n                # if it is a flag, in commandline it makes little sense to show\n                # showing the Default: (likely boolean).\n                #   See https://github.com/datalad/datalad/issues/3203\n                if not parser_kwargs.get('action', '').startswith('store_'):\n                    # [Default: None] also makes little sense for cmdline\n                    if defaults[defaults_idx] is not None:\n                        help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from datalad.utils import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k)\n                  for k in argnames\n                  # some arguments might be Python-only and do not appear in the\n                  # parser Namespace\n                  if hasattr(args, k) and is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)\n\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = str(resolve_path(refds_path))\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from datalad.utils import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Documentation Does Not Reflect Overridden Default Options in `eval_params`

**Description:**
When overriding default options in `eval_params`, the generated documentation fails to display the updated default values. Specifically, overriding attributes like `result_renderer` and `return_type` should update their default values in the documentation, but they remain unchanged.

**Example Code:**
```python
class TestUtils(Interface):
    result_renderer = 'tailored'  # Overrides the default value
    return_type = 'item-or-list'  # Overrides the default value

    _params_ = dict(
        number=Parameter(
            args=("-n", "--number"),
            doc="It's a number.",
            default='list'
        )
    )
```

**Expected Behavior:**
The documentation should display the overridden default values for `result_renderer` and `return_type`. For example:
- `result_renderer`: Default: `'tailored'`
- `return_type`: Default: `'item-or-list'`

**Actual Behavior:**
The documentation continues to show the original default values instead of the overridden ones:
- `result_renderer`: Default: `None`
- `return_type`: Default: `'list'`

This discrepancy causes confusion and leads to failing tests that expect the documentation to reflect the overridden defaults.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/base.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""High-level interface generation

"""

__docformat__ = 'restructuredtext'

import logging
lgr = logging.getLogger('datalad.interface.base')

import os
import sys
import re
import textwrap
from importlib import import_module
import inspect
import string
from collections import (
    defaultdict,
    OrderedDict,
)

from ..ui import ui
from ..dochelpers import exc_str

from datalad.interface.common_opts import eval_params
from datalad.interface.common_opts import eval_defaults
from datalad.support.constraints import (
    EnsureKeyChoice,
    EnsureChoice,
)
from datalad.distribution.dataset import Dataset
from datalad.distribution.dataset import resolve_path
from datalad.plugin import _get_plugins
from datalad.plugin import _load_plugin


default_logchannels = {
    '': 'debug',
    'ok': 'debug',
    'notneeded': 'debug',
    'impossible': 'warning',
    'error': 'error',
}


def get_api_name(intfspec):
    """Given an interface specification return an API name for it"""
    if len(intfspec) > 3:
        name = intfspec[3]
    else:
        name = intfspec[0].split('.')[-1]
    return name


def get_cmdline_command_name(intfspec):
    """Given an interface specification return a cmdline command name"""
    if len(intfspec) > 2:
        name = intfspec[2]
    else:
        name = intfspec[0].split('.')[-1].replace('_', '-')
    return name


def get_interface_groups(include_plugins=False):
    """Return a list of command groups.

    Parameters
    ----------
    include_plugins : bool, optional
        Whether to include a group named 'plugins' that has a list of
        discovered plugin commands.

    Returns
    -------
    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).
    """
    from .. import interface as _interfaces

    grps = []
    # auto detect all available interfaces and generate a function-based
    # API from them
    for _item in _interfaces.__dict__:
        if not _item.startswith('_group_'):
            continue
        grp_name = _item[7:]
        grp = getattr(_interfaces, _item)
        grps.append((grp_name,) + grp)
    # TODO(yoh): see if we could retain "generator" for plugins
    # ATM we need to make it explicit so we could check the command(s) below
    # It could at least follow the same destiny as extensions so we would
    # just do more iterative "load ups"

    if include_plugins:
        grps.append(('plugins', 'Plugins', list(_get_plugins())))
    return grps


def get_cmd_summaries(descriptions, groups, width=79):
    """Return summaries for the commands in `groups`.

    Parameters
    ----------
    descriptions : dict
        A map of group names to summaries.
    groups : list of tuples
        A list of groups and commands in the form described by
        `get_interface_groups`.
    width : int, optional
        The maximum width of each line in the summary text.

    Returns
    -------
    A list with a formatted entry for each command. The first command of each
    group is preceded by an entry describing the group.
    """
    cmd_summary = []
    for grp in sorted(groups, key=lambda x: x[1]):
        grp_descr = grp[1]
        grp_cmds = descriptions[grp[0]]

        cmd_summary.append('\n*%s*\n' % (grp_descr,))
        for cd in grp_cmds:
            cmd_summary.append('  %s\n%s'
                               % ((cd[0],
                                   textwrap.fill(
                                       cd[1].rstrip(' .'),
                                       width - 5,
                                       initial_indent=' ' * 6,
                                       subsequent_indent=' ' * 6))))
    return cmd_summary


def load_interface(spec):
    """Load and return the class for `spec`.

    Parameters
    ----------
    spec : tuple
        For a standard interface, the first item is the datalad source module
        and the second object name for the interface. For a plugin, the second
        item should be a dictionary that maps 'file' to the path the of module.

    Returns
    -------
    The interface class or, if importing the module fails, None.
    """
    if isinstance(spec[1], dict):
        intf = _load_plugin(spec[1]['file'], fail=False)
    else:
        lgr.log(5, "Importing module %s " % spec[0])
        try:
            mod = import_module(spec[0], package='datalad')
        except Exception as e:
            lgr.error("Internal error, cannot import interface '%s': %s",
                      spec[0], exc_str(e))
            intf = None
        else:
            intf = getattr(mod, spec[1])
    return intf


def get_cmd_doc(interface):
    """Return the documentation for the command defined by `interface`.

    Parameters
    ----------
    interface : subclass of Interface
    """
    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()
    if hasattr(interface, '_docs_'):
        # expand docs
        intf_doc = intf_doc.format(**interface._docs_)
    return intf_doc


def get_cmd_ex(interface):
    """Return the examples for the command defined by 'interface'.

    Parameters
    ----------
    interface : subclass of Interface
    """
    intf_ex = "\n\n*Examples*\n\n"
    for example in interface._examples_:
        intf_ex += build_example(example, api='cmdline')
    return intf_ex


def dedent_docstring(text):
    """Remove uniform indentation from a multiline docstring"""
    # Problem is that first line might often have no offset, so might
    # need to be ignored from dedent call
    if text is None:
        return None
    if not text.startswith(' '):
        lines = text.split('\n')
        if len(lines) == 1:
            # single line, no indentation, nothing to do
            return text
        text2 = '\n'.join(lines[1:])
        return lines[0] + "\n" + textwrap.dedent(text2)
    else:
        return textwrap.dedent(text)


def alter_interface_docs_for_api(docs):
    """Apply modifications to interface docstrings for Python API use."""
    # central place to alter the impression of docstrings,
    # like removing cmdline specific sections
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        r'\|\| CMDLINE \>\>.*?\<\< CMDLINE \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        r'\[CMD:\s[^\[\]]*\sCMD\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    docs = re.sub(
        r'\[PY:\s([^\[\]]*)\sPY\]',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE)
    docs = re.sub(
        r'\|\| PYTHON \>\>(.*?)\<\< PYTHON \|\|',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    if 'DATALAD_SPHINX_RUN' not in os.environ:
        # remove :role:`...` RST markup for cmdline docs
        docs = re.sub(
            r':\S+:`[^`]*`[\\]*',
            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\'),
            docs,
            flags=re.MULTILINE | re.DOTALL)
        # make the handbook doc references more accessible
        # the URL is a redirect configured at readthedocs
        docs = re.sub(
            r'(handbook:[0-9]-[0-9]*)',
            '\\1 (http://handbook.datalad.org/symbols)',
            docs)
    docs = re.sub(
        r'\|\| REFLOW \>\>\n(.*?)\<\< REFLOW \|\|',
        lambda match: textwrap.fill(match.group(1)),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    return docs


def alter_interface_docs_for_cmdline(docs):
    """Apply modifications to interface docstrings for cmdline doc use."""
    # central place to alter the impression of docstrings,
    # like removing Python API specific sections, and argument markup
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        r'\|\| PYTHON \>\>.*?\<\< PYTHON \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        r'\[PY:\s[^\[\]]*\sPY\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    docs = re.sub(
        r'\[CMD:\s([^\[\]]*)\sCMD\]',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE)
    docs = re.sub(
        r'\|\| CMDLINE \>\>(.*?)\<\< CMDLINE \|\|',
        lambda match: match.group(1),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # remove :role:`...` RST markup for cmdline docs
    docs = re.sub(
        r':\S+:`[^`]*`[\\]*',
        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\'),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # make the handbook doc references more accessible
    # the URL is a redirect configured at readthedocs
    docs = re.sub(
        r'(handbook:[0-9]-[0-9]*)',
        '\\1 (http://handbook.datalad.org/symbols)',
        docs)
    # remove None constraint. In general, `None` on the cmdline means don't
    # give option at all, but specifying `None` explicitly is practically
    # impossible
    docs = re.sub(
        r',\sor\svalue\smust\sbe\s`None`',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # capitalize variables and remove backticks to uniformize with
    # argparse output
    docs = re.sub(
        r'`\S*`',
        lambda match: match.group(0).strip('`').upper(),
        docs)
    # clean up sphinx API refs
    docs = re.sub(
        r'\~datalad\.api\.\S*',
        lambda match: "`{0}`".format(match.group(0)[13:]),
        docs)
    # Remove RST paragraph markup
    docs = re.sub(
        r'^.. \S+::',
        lambda match: match.group(0)[3:-2].upper(),
        docs,
        flags=re.MULTILINE)
    docs = re.sub(
        r'\|\| REFLOW \>\>\n(.*?)\<\< REFLOW \|\|',
        lambda match: textwrap.fill(match.group(1)),
        docs,
        flags=re.MULTILINE | re.DOTALL)
    return docs


def is_api_arg(arg):
    """Return True if argument is our API argument or self or used for internal
    purposes
    """
    return arg != 'self' and not arg.startswith('_')


def update_docstring_with_parameters(func, params, prefix=None, suffix=None,
                                     add_args=None):
    """Generate a useful docstring from a parameter spec

    Amends any existing docstring of a callable with a textual
    description of its parameters. The Parameter spec needs to match
    the number and names of the callables arguments.
    """
    from datalad.utils import getargspec
    # get the signature
    ndefaults = 0
    args, varargs, varkw, defaults = getargspec(func)
    if add_args:
        add_argnames = sorted(add_args.keys())
        args.extend(add_argnames)
        defaults = defaults + tuple(add_args[k] for k in add_argnames)
    if defaults is not None:
        ndefaults = len(defaults)
    # start documentation with what the callable brings with it
    doc = prefix if prefix else u''
    if len(args) > 1:
        if len(doc):
            doc += '\n'
        doc += "Parameters\n----------\n"
        for i, arg in enumerate(args):
            if not is_api_arg(arg):
                continue
            # we need a parameter spec for each argument
            if not arg in params:
                raise ValueError("function has argument '%s' not described as a parameter" % arg)
            param = params[arg]
            # validate the default -- to make sure that the parameter description is
            # somewhat OK
            defaults_idx = ndefaults - len(args) + i
            if defaults_idx >= 0:
                if param.constraints is not None:
                    param.constraints(defaults[defaults_idx])
            orig_docs = param._doc
            param._doc = alter_interface_docs_for_api(param._doc)
            doc += param.get_autodoc(
                arg,
                default=defaults[defaults_idx] if defaults_idx >= 0 else None,
                has_default=defaults_idx >= 0)
            param._doc = orig_docs
            doc += '\n'
    doc += suffix if suffix else u""
    # assign the amended docs
    func.__doc__ = doc
    return func


def build_example(example, api='python'):
    """Build a code example.

    Take a dict from a classes _example_ specification (list of dicts) and
    build a string with an api or cmd example (for use in cmd help or
    docstring).

    Parameters
    ----------
    api : {'python', 'cmdline'}
        If 'python', build Python example for docstring. If 'cmdline', build
        cmd example.

    Returns
    -------
    ex : str
        Concatenated examples for the given class.
    """
    if api == 'python' :
        code_field='code_py'
        indicator='>'
    elif api == 'cmdline':
        code_field='code_cmd'
        indicator='%'
    else:
        raise ValueError("unknown API selection: {}".format(api))
    if code_field not in example:
        # only show an example if it exist for the API
        return ''
    description = textwrap.fill(example.get('text'))
    # this indent the code snippet to get it properly rendered as code
    # we are not using textwrap.fill(), because it would not acknowledge
    # any meaningful structure/formatting of code snippets. Instead, we
    # maintain line content as is.
    code = dedent_docstring(example.get(code_field))
    needs_indicator = not code.startswith(indicator)
    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()

    ex = """{}::\n\n   {}{}\n\n""".format(
        description,
        # disable automatic prefixing, if the example already has one
        # this enables providing more complex examples without having
        # to infer its inner structure
        '{} '.format(indicator)
        if needs_indicator
        # maintain spacing to avoid undesired relative indentation
        else '',
        code)

    return ex


def update_docstring_with_examples(cls_doc, ex):
    """Update a commands docstring with examples.

    Take _examples_ of a command, build the Python examples, and append
    them to the docstring.

    cls_doc: docstring
    ex: list
        list of dicts with examples
    """
    from textwrap import indent
    if len(cls_doc):
        cls_doc += "\n"
    cls_doc += "    Examples\n    --------\n"
    # loop though provided examples
    for example in ex:
        cls_doc += indent(build_example(example, api='python'), ' '*4)

    return cls_doc


def build_doc(cls, **kwargs):
    """Decorator to build docstrings for datalad commands

    It's intended to decorate the class, the __call__-method of which is the
    actual command. It expects that __call__-method to be decorated by
    eval_results.

    Note that values for any `eval_params` keys in `cls._params_` are
    ignored.  This means one class may extend another's `_params_`
    without worrying about filtering out `eval_params`.

    Parameters
    ----------
    cls: Interface
      class defining a datalad command
    """

    # Note, that this is a class decorator, which is executed only once when the
    # class is imported. It builds the docstring for the class' __call__ method
    # and returns the original class.
    #
    # This is because a decorator for the actual function would not be able to
    # behave like this. To build the docstring we need to access the attribute
    # _params of the class. From within a function decorator we cannot do this
    # during import time, since the class is being built in this very moment and
    # is not yet available in the module. And if we do it from within the part
    # of a function decorator, that is executed when the function is called, we
    # would need to actually call the command once in order to build this
    # docstring.

    lgr.debug("Building doc for {}".format(cls))

    cls_doc = cls.__doc__
    if hasattr(cls, '_docs_'):
        # expand docs
        cls_doc = cls_doc.format(**cls._docs_)
    # get examples
    ex = getattr(cls, '_examples_', [])
    if ex:
        cls_doc = update_docstring_with_examples(cls_doc, ex)

    call_doc = None
    # suffix for update_docstring_with_parameters:
    if cls.__call__.__doc__:
        call_doc = cls.__call__.__doc__

    # build standard doc and insert eval_doc
    spec = getattr(cls, '_params_', dict())
    # ATTN: An important consequence of this update() call is that it
    # fulfills the docstring's promise of overriding any existing
    # values for eval_params keys in _params_.
    #
    # get docs for eval_results parameters:
    spec.update(eval_params)

    update_docstring_with_parameters(
        cls.__call__, spec,
        prefix=alter_interface_docs_for_api(cls_doc),
        suffix=alter_interface_docs_for_api(call_doc),
        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None
    )

    # return original
    return cls


NA_STRING = 'N/A'  # we might want to make it configurable via config


class nagen(object):
    """A helper to provide a desired missing value if no value is known

    Usecases
    - could be used as a generator for `defaultdict`
    - since it returns itself upon getitem, should work even for complex
      nested dictionaries/lists .format templates
    """
    def __init__(self, missing=NA_STRING):
        self.missing = missing

    def __repr__(self):
        cls = self.__class__.__name__
        args = str(self.missing) if self.missing != NA_STRING else ''
        return '%s(%s)' % (cls, args)

    def __str__(self):
        return self.missing

    def __getitem__(self, *args):
        return self

    def __getattr__(self, item):
        return self


def nadict(*items):
    """A generator of default dictionary with the default nagen"""
    dd = defaultdict(nagen)
    dd.update(*items)
    return dd


class DefaultOutputFormatter(string.Formatter):
    """A custom formatter for default output rendering using .format
    """
    # TODO: make missing configurable?
    def __init__(self, missing=nagen()):
        """
        Parameters
        ----------
        missing: string, optional
          What to output for the missing values
        """
        super(DefaultOutputFormatter, self).__init__()
        self.missing = missing

    def _d(self, msg, *args):
        # print("   HERE %s" % (msg % args))
        pass

    def get_value(self, key, args, kwds):
        assert not args
        self._d("get_value: %r %r %r", key, args, kwds)
        return kwds.get(key, self.missing)

    # def get_field(self, field_name, args, kwds):
    #     assert not args
    #     self._d("get_field: %r args=%r kwds=%r" % (field_name, args, kwds))
    #     try:
    #         out = string.Formatter.get_field(self, field_name, args, kwds)
    #     except Exception as exc:
    #         # TODO needs more than just a value
    #         return "!ERR %s" % exc


class DefaultOutputRenderer(object):
    """A default renderer for .format'ed output line
    """
    def __init__(self, format):
        self.format = format
        # We still need custom output formatter since at the "first level"
        # within .format template all items there is no `nadict`
        self.formatter = DefaultOutputFormatter()

    @classmethod
    def _dict_to_nadict(cls, v):
        """Traverse datastructure and replace any regular dict with nadict"""
        if isinstance(v, list):
            return [cls._dict_to_nadict(x) for x in v]
        elif isinstance(v, dict):
            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())
        else:
            return v

    def __call__(self, x, **kwargs):
        dd = nadict(
            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)
                for k_, v_ in v.items()})
                if isinstance(v, dict) else v)
            for k, v in x.items()
        )

        msg = self.formatter.format(self.format, **dd)
        return ui.message(msg)


class Interface(object):
    """Base class for interface implementations"""

    # exit code to return if user-interrupted
    # if None, would just reraise the Exception, so if in --dbg
    # mode would fall into the debugger
    _interrupted_exit_code = 1

    _OLDSTYLE_COMMANDS = (
        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',
        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')

    @classmethod
    def setup_parser(cls, parser):
        # XXX needs safety check for name collisions
        # XXX allow for parser kwargs customization
        parser_kwargs = {}
        from datalad.utils import getargspec
        # get the signature
        ndefaults = 0
        args, varargs, varkw, defaults = getargspec(cls.__call__)
        if not defaults is None:
            ndefaults = len(defaults)
        for i, arg in enumerate(args):
            if not is_api_arg(arg):
                continue
            param = cls._params_[arg]
            defaults_idx = ndefaults - len(args) + i
            cmd_args = param.cmd_args
            if cmd_args == tuple():
                # explicitly provided an empty sequence of argument names
                # this shall not appear in the parser
                continue
            elif cmd_args is None:
                cmd_args = []
            if not len(cmd_args):
                if defaults_idx >= 0:
                    # dealing with a kwarg
                    template = '--%s'
                else:
                    # positional arg
                    template = '%s'
                # use parameter name as default argument name
                parser_args = (template % arg.replace('_', '-'),)
            else:
                parser_args = [c.replace('_', '-') for c in cmd_args]
            parser_kwargs = param.cmd_kwargs
            if defaults_idx >= 0:
                parser_kwargs['default'] = defaults[defaults_idx]
            help = alter_interface_docs_for_cmdline(param._doc)
            if help and help.rstrip()[-1] != '.':
                help = help.rstrip() + '.'
            if param.constraints is not None:
                parser_kwargs['type'] = param.constraints
                # include value constraint description and default
                # into the help string
                cdoc = alter_interface_docs_for_cmdline(
                    param.constraints.long_description())
                if cdoc[0] == '(' and cdoc[-1] == ')':
                    cdoc = cdoc[1:-1]
                help += '  Constraints: %s' % cdoc
                if 'metavar' not in parser_kwargs and \
                        isinstance(param.constraints, EnsureChoice):
                    parser_kwargs['metavar'] = \
                        '{%s}' % '|'.join(
                            # don't use short_description(), because
                            # it also needs to give valid output for
                            # Python syntax (quotes...), but here we
                            # can simplify to shell syntax where everything
                            # is a string
                            p for p in param.constraints._allowed
                            # in the cmdline None pretty much means
                            # don't give the options, so listing it
                            # doesn't make sense. Moreover, any non-string
                            # value cannot be given and very likely only
                            # serves a special purpose in the Python API
                            # or implementation details
                            if isinstance(p, str))
            if defaults_idx >= 0:
                # if it is a flag, in commandline it makes little sense to show
                # showing the Default: (likely boolean).
                #   See https://github.com/datalad/datalad/issues/3203
                if not parser_kwargs.get('action', '').startswith('store_'):
                    # [Default: None] also makes little sense for cmdline
                    if defaults[defaults_idx] is not None:
                        help += " [Default: %r]" % (defaults[defaults_idx],)
            # create the parameter, using the constraint instance for type
            # conversion
            parser.add_argument(*parser_args, help=help,
                                **parser_kwargs)

    @classmethod
    def call_from_parser(cls, args):
        # XXX needs safety check for name collisions
        from datalad.utils import getargspec
        argspec = getargspec(cls.__call__)
        if argspec[2] is None:
            # no **kwargs in the call receiver, pull argnames from signature
            argnames = getargspec(cls.__call__)[0]
        else:
            # common options
            # XXX define or better get from elsewhere
            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',
                           'help', 'log_level', 'logger', 'pbs_runner',
                           'result_renderer', 'subparser')
            argnames = [name for name in dir(args)
                        if not (name.startswith('_') or name in common_opts)]
        kwargs = {k: getattr(args, k)
                  for k in argnames
                  # some arguments might be Python-only and do not appear in the
                  # parser Namespace
                  if hasattr(args, k) and is_api_arg(k)}
        # we are coming from the entry point, this is the toplevel command,
        # let it run like generator so we can act on partial results quicker
        # TODO remove following condition test when transition is complete and
        # run indented code unconditionally
        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:
            # set all common args explicitly  to override class defaults
            # that are tailored towards the the Python API
            kwargs['return_type'] = 'generator'
            kwargs['result_xfm'] = None
            # allow commands to override the default, unless something other than
            # default is requested
            kwargs['result_renderer'] = \
                args.common_output_format if args.common_output_format != 'default' \
                else getattr(cls, 'result_renderer', args.common_output_format)
            if '{' in args.common_output_format:
                # stupid hack, could and should become more powerful
                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)

            if args.common_on_failure:
                kwargs['on_failure'] = args.common_on_failure
            # compose filter function from to be invented cmdline options
            res_filter = cls._get_result_filter(args)
            if res_filter is not None:
                # Don't add result_filter if it's None because then
                # eval_results can't distinguish between --report-{status,type}
                # not specified via the CLI and None passed via the Python API.
                kwargs['result_filter'] = res_filter
        try:
            ret = cls.__call__(**kwargs)
            if inspect.isgenerator(ret):
                ret = list(ret)
            return ret
        except KeyboardInterrupt as exc:
            ui.error("\nInterrupted by user while doing magic: %s" % exc_str(exc))
            if cls._interrupted_exit_code is not None:
                sys.exit(cls._interrupted_exit_code)
            else:
                raise

    @classmethod
    def _get_result_filter(cls, args):
        from datalad import cfg
        result_filter = None
        if args.common_report_status or 'datalad.runtime.report-status' in cfg:
            report_status = args.common_report_status or \
                            cfg.obtain('datalad.runtime.report-status')
            if report_status == "all":
                pass  # no filter
            elif report_status == 'success':
                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))
            elif report_status == 'failure':
                result_filter = EnsureKeyChoice('status',
                                                ('impossible', 'error'))
            else:
                result_filter = EnsureKeyChoice('status', (report_status,))
        if args.common_report_type:
            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))
            result_filter = result_filter & tfilt if result_filter else tfilt
        return result_filter

    @classmethod
    def get_refds_path(cls, dataset):
        """Return a resolved reference dataset path from a `dataset` argument"""
        # theoretically a dataset could come in as a relative path -> resolve
        if dataset is None:
            return dataset
        refds_path = dataset.path if isinstance(dataset, Dataset) \
            else Dataset(dataset).path
        if refds_path:
            refds_path = str(resolve_path(refds_path))
        return refds_path


def get_allargs_as_kwargs(call, args, kwargs):
    """Generate a kwargs dict from a call signature and *args, **kwargs

    Basically resolving the argnames for all positional arguments, and
    resolvin the defaults for all kwargs that are not given in a kwargs
    dict
    """
    from datalad.utils import getargspec
    argspec = getargspec(call)
    defaults = argspec.defaults
    nargs = len(argspec.args)
    assert (nargs >= len(defaults))
    # map any args to their name
    argmap = list(zip(argspec.args[:len(args)], args))
    kwargs_ = OrderedDict(argmap)
    # map defaults of kwargs to their names (update below)
    for k, v in zip(argspec.args[-len(defaults):], defaults):
        if k not in kwargs_:
            kwargs_[k] = v
    # update with provided kwarg args
    kwargs_.update(kwargs)
    # XXX we cannot assert the following, because our own highlevel
    # API commands support more kwargs than what is discoverable
    # from their signature...
    #assert (nargs == len(kwargs_))
    return kwargs_

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user