{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport os\nimport sys\nimport re\nimport textwrap\nfrom importlib import import_module\nimport inspect\nimport string\nfrom collections import (\n    defaultdict,\n    OrderedDict,\n)\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureChoice,\n)\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.plugin import _get_plugins\nfrom datalad.plugin import _load_plugin\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups(include_plugins=False):\n    \"\"\"Return a list of command groups.\n\n    Parameters\n    ----------\n    include_plugins : bool, optional\n        Whether to include a group named 'plugins' that has a list of\n        discovered plugin commands.\n\n    Returns\n    -------\n    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).\n    \"\"\"\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    # TODO(yoh): see if we could retain \"generator\" for plugins\n    # ATM we need to make it explicit so we could check the command(s) below\n    # It could at least follow the same destiny as extensions so we would\n    # just do more iterative \"load ups\"\n\n    if include_plugins:\n        grps.append(('plugins', 'Plugins', list(_get_plugins())))\n    return grps\n\n\ndef get_cmd_summaries(descriptions, groups, width=79):\n    \"\"\"Return summaries for the commands in `groups`.\n\n    Parameters\n    ----------\n    descriptions : dict\n        A map of group names to summaries.\n    groups : list of tuples\n        A list of groups and commands in the form described by\n        `get_interface_groups`.\n    width : int, optional\n        The maximum width of each line in the summary text.\n\n    Returns\n    -------\n    A list with a formatted entry for each command. The first command of each\n    group is preceded by an entry describing the group.\n    \"\"\"\n    cmd_summary = []\n    for grp in sorted(groups, key=lambda x: x[1]):\n        grp_descr = grp[1]\n        grp_cmds = descriptions[grp[0]]\n\n        cmd_summary.append('\\n*%s*\\n' % (grp_descr,))\n        for cd in grp_cmds:\n            cmd_summary.append('  %s\\n%s'\n                               % ((cd[0],\n                                   textwrap.fill(\n                                       cd[1].rstrip(' .'),\n                                       width - 5,\n                                       initial_indent=' ' * 6,\n                                       subsequent_indent=' ' * 6))))\n    return cmd_summary\n\n\ndef load_interface(spec):\n    \"\"\"Load and return the class for `spec`.\n\n    Parameters\n    ----------\n    spec : tuple\n        For a standard interface, the first item is the datalad source module\n        and the second object name for the interface. For a plugin, the second\n        item should be a dictionary that maps 'file' to the path the of module.\n\n    Returns\n    -------\n    The interface class or, if importing the module fails, None.\n    \"\"\"\n    if isinstance(spec[1], dict):\n        intf = _load_plugin(spec[1]['file'], fail=False)\n    else:\n        lgr.log(5, \"Importing module %s \" % spec[0])\n        try:\n            mod = import_module(spec[0], package='datalad')\n        except Exception as e:\n            lgr.error(\"Internal error, cannot import interface '%s': %s\",\n                      spec[0], exc_str(e))\n            intf = None\n        else:\n            intf = getattr(mod, spec[1])\n    return intf\n\n\ndef get_cmd_doc(interface):\n    \"\"\"Return the documentation for the command defined by `interface`.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()\n    if hasattr(interface, '_docs_'):\n        # expand docs\n        intf_doc = intf_doc.format(**interface._docs_)\n    return intf_doc\n\n\ndef get_cmd_ex(interface):\n    \"\"\"Return the examples for the command defined by 'interface'.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_ex = \"\\n\\n*Examples*\\n\\n\"\n    for example in interface._examples_:\n        intf_ex += build_example(example, api='cmdline')\n    return intf_ex\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>.*?\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>(.*?)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    if 'DATALAD_SPHINX_RUN' not in os.environ:\n        # remove :role:`...` RST markup for cmdline docs\n        docs = re.sub(\n            r':\\S+:`[^`]*`[\\\\]*',\n            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n            docs,\n            flags=re.MULTILINE | re.DOTALL)\n        # make the handbook doc references more accessible\n        # the URL is a redirect configured at readthedocs\n        docs = re.sub(\n            r'(handbook:[0-9]-[0-9]*)',\n            '\\\\1 (http://handbook.datalad.org/symbols)',\n            docs)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>.*?\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>(.*?)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # make the handbook doc references more accessible\n    # the URL is a redirect configured at readthedocs\n    docs = re.sub(\n        r'(handbook:[0-9]-[0-9]*)',\n        '\\\\1 (http://handbook.datalad.org/symbols)',\n        docs)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        r',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        r'`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        r'\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from datalad.utils import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_example(example, api='python'):\n    \"\"\"Build a code example.\n\n    Take a dict from a classes _example_ specification (list of dicts) and\n    build a string with an api or cmd example (for use in cmd help or\n    docstring).\n\n    Parameters\n    ----------\n    api : {'python', 'cmdline'}\n        If 'python', build Python example for docstring. If 'cmdline', build\n        cmd example.\n\n    Returns\n    -------\n    ex : str\n        Concatenated examples for the given class.\n    \"\"\"\n    if api == 'python' :\n        code_field='code_py'\n        indicator='>'\n    elif api == 'cmdline':\n        code_field='code_cmd'\n        indicator='%'\n    else:\n        raise ValueError(\"unknown API selection: {}\".format(api))\n    if code_field not in example:\n        # only show an example if it exist for the API\n        return ''\n    description = textwrap.fill(example.get('text'))\n    # this indent the code snippet to get it properly rendered as code\n    # we are not using textwrap.fill(), because it would not acknowledge\n    # any meaningful structure/formatting of code snippets. Instead, we\n    # maintain line content as is.\n    code = dedent_docstring(example.get(code_field))\n    needs_indicator = not code.startswith(indicator)\n    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()\n\n    ex = \"\"\"{}::\\n\\n   {}{}\\n\\n\"\"\".format(\n        description,\n        # disable automatic prefixing, if the example already has one\n        # this enables providing more complex examples without having\n        # to infer its inner structure\n        '{} '.format(indicator)\n        if needs_indicator\n        # maintain spacing to avoid undesired relative indentation\n        else '',\n        code)\n\n    return ex\n\n\ndef update_docstring_with_examples(cls_doc, ex):\n    \"\"\"Update a commands docstring with examples.\n\n    Take _examples_ of a command, build the Python examples, and append\n    them to the docstring.\n\n    cls_doc: docstring\n    ex: list\n        list of dicts with examples\n    \"\"\"\n    from textwrap import indent\n    if len(cls_doc):\n        cls_doc += \"\\n\"\n    cls_doc += \"    Examples\\n    --------\\n\"\n    # loop though provided examples\n    for example in ex:\n        cls_doc += indent(build_example(example, api='python'), ' '*4)\n\n    return cls_doc\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n\n\n    # update class attributes that may override defaults \n    if hasattr(cls, '_no_eval_results'):\n        add_args = None\n    else:\n        add_args = {k: getattr(cls, k, v) for k, v in eval_defaults.items()}\n\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=add_args\n    )\n\n    # return original\n    return cls\n\n\nNA_STRING = 'N/A'  # we might want to make it configurable via config\n\n\nclass nagen(object):\n    \"\"\"A helper to provide a desired missing value if no value is known\n\n    Usecases\n    - could be used as a generator for `defaultdict`\n    - since it returns itself upon getitem, should work even for complex\n      nested dictionaries/lists .format templates\n    \"\"\"\n    def __init__(self, missing=NA_STRING):\n        self.missing = missing\n\n    def __repr__(self):\n        cls = self.__class__.__name__\n        args = str(self.missing) if self.missing != NA_STRING else ''\n        return '%s(%s)' % (cls, args)\n\n    def __str__(self):\n        return self.missing\n\n    def __getitem__(self, *args):\n        return self\n\n    def __getattr__(self, item):\n        return self\n\n\ndef nadict(*items):\n    \"\"\"A generator of default dictionary with the default nagen\"\"\"\n    dd = defaultdict(nagen)\n    dd.update(*items)\n    return dd\n\n\nclass DefaultOutputFormatter(string.Formatter):\n    \"\"\"A custom formatter for default output rendering using .format\n    \"\"\"\n    # TODO: make missing configurable?\n    def __init__(self, missing=nagen()):\n        \"\"\"\n        Parameters\n        ----------\n        missing: string, optional\n          What to output for the missing values\n        \"\"\"\n        super(DefaultOutputFormatter, self).__init__()\n        self.missing = missing\n\n    def _d(self, msg, *args):\n        # print(\"   HERE %s\" % (msg % args))\n        pass\n\n    def get_value(self, key, args, kwds):\n        assert not args\n        self._d(\"get_value: %r %r %r\", key, args, kwds)\n        return kwds.get(key, self.missing)\n\n    # def get_field(self, field_name, args, kwds):\n    #     assert not args\n    #     self._d(\"get_field: %r args=%r kwds=%r\" % (field_name, args, kwds))\n    #     try:\n    #         out = string.Formatter.get_field(self, field_name, args, kwds)\n    #     except Exception as exc:\n    #         # TODO needs more than just a value\n    #         return \"!ERR %s\" % exc\n\n\nclass DefaultOutputRenderer(object):\n    \"\"\"A default renderer for .format'ed output line\n    \"\"\"\n    def __init__(self, format):\n        self.format = format\n        # We still need custom output formatter since at the \"first level\"\n        # within .format template all items there is no `nadict`\n        self.formatter = DefaultOutputFormatter()\n\n    @classmethod\n    def _dict_to_nadict(cls, v):\n        \"\"\"Traverse datastructure and replace any regular dict with nadict\"\"\"\n        if isinstance(v, list):\n            return [cls._dict_to_nadict(x) for x in v]\n        elif isinstance(v, dict):\n            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())\n        else:\n            return v\n\n    def __call__(self, x, **kwargs):\n        dd = nadict(\n            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)\n                for k_, v_ in v.items()})\n                if isinstance(v, dict) else v)\n            for k, v in x.items()\n        )\n\n        msg = self.formatter.format(self.format, **dd)\n        return ui.message(msg)\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from datalad.utils import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args == tuple():\n                # explicitly provided an empty sequence of argument names\n                # this shall not appear in the parser\n                continue\n            elif cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help.rstrip()[-1] != '.':\n                help = help.rstrip() + '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n                if 'metavar' not in parser_kwargs and \\\n                        isinstance(param.constraints, EnsureChoice):\n                    parser_kwargs['metavar'] = \\\n                        '{%s}' % '|'.join(\n                            # don't use short_description(), because\n                            # it also needs to give valid output for\n                            # Python syntax (quotes...), but here we\n                            # can simplify to shell syntax where everything\n                            # is a string\n                            p for p in param.constraints._allowed\n                            # in the cmdline None pretty much means\n                            # don't give the options, so listing it\n                            # doesn't make sense. Moreover, any non-string\n                            # value cannot be given and very likely only\n                            # serves a special purpose in the Python API\n                            # or implementation details\n                            if isinstance(p, str))\n            if defaults_idx >= 0:\n                # if it is a flag, in commandline it makes little sense to show\n                # showing the Default: (likely boolean).\n                #   See https://github.com/datalad/datalad/issues/3203\n                if not parser_kwargs.get('action', '').startswith('store_'):\n                    # [Default: None] also makes little sense for cmdline\n                    if defaults[defaults_idx] is not None:\n                        help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from datalad.utils import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k)\n                  for k in argnames\n                  # some arguments might be Python-only and do not appear in the\n                  # parser Namespace\n                  if hasattr(args, k) and is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)\n\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = str(resolve_path(refds_path))\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from datalad.utils import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport os\nimport sys\nimport re\nimport textwrap\nfrom importlib import import_module\nimport inspect\nimport string\nfrom collections import (\n    defaultdict,\n    OrderedDict,\n)\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureChoice,\n)\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.plugin import _get_plugins\nfrom datalad.plugin import _load_plugin\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups(include_plugins=False):\n    \"\"\"Return a list of command groups.\n\n    Parameters\n    ----------\n    include_plugins : bool, optional\n        Whether to include a group named 'plugins' that has a list of\n        discovered plugin commands.\n\n    Returns\n    -------\n    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).\n    \"\"\"\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    # TODO(yoh): see if we could retain \"generator\" for plugins\n    # ATM we need to make it explicit so we could check the command(s) below\n    # It could at least follow the same destiny as extensions so we would\n    # just do more iterative \"load ups\"\n\n    if include_plugins:\n        grps.append(('plugins', 'Plugins', list(_get_plugins())))\n    return grps\n\n\ndef get_cmd_summaries(descriptions, groups, width=79):\n    \"\"\"Return summaries for the commands in `groups`.\n\n    Parameters\n    ----------\n    descriptions : dict\n        A map of group names to summaries.\n    groups : list of tuples\n        A list of groups and commands in the form described by\n        `get_interface_groups`.\n    width : int, optional\n        The maximum width of each line in the summary text.\n\n    Returns\n    -------\n    A list with a formatted entry for each command. The first command of each\n    group is preceded by an entry describing the group.\n    \"\"\"\n    cmd_summary = []\n    for grp in sorted(groups, key=lambda x: x[1]):\n        grp_descr = grp[1]\n        grp_cmds = descriptions[grp[0]]\n\n        cmd_summary.append('\\n*%s*\\n' % (grp_descr,))\n        for cd in grp_cmds:\n            cmd_summary.append('  %s\\n%s'\n                               % ((cd[0],\n                                   textwrap.fill(\n                                       cd[1].rstrip(' .'),\n                                       width - 5,\n                                       initial_indent=' ' * 6,\n                                       subsequent_indent=' ' * 6))))\n    return cmd_summary\n\n\ndef load_interface(spec):\n    \"\"\"Load and return the class for `spec`.\n\n    Parameters\n    ----------\n    spec : tuple\n        For a standard interface, the first item is the datalad source module\n        and the second object name for the interface. For a plugin, the second\n        item should be a dictionary that maps 'file' to the path the of module.\n\n    Returns\n    -------\n    The interface class or, if importing the module fails, None.\n    \"\"\"\n    if isinstance(spec[1], dict):\n        intf = _load_plugin(spec[1]['file'], fail=False)\n    else:\n        lgr.log(5, \"Importing module %s \" % spec[0])\n        try:\n            mod = import_module(spec[0], package='datalad')\n        except Exception as e:\n            lgr.error(\"Internal error, cannot import interface '%s': %s\",\n                      spec[0], exc_str(e))\n            intf = None\n        else:\n            intf = getattr(mod, spec[1])\n    return intf\n\n\ndef get_cmd_doc(interface):\n    \"\"\"Return the documentation for the command defined by `interface`.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()\n    if hasattr(interface, '_docs_'):\n        # expand docs\n        intf_doc = intf_doc.format(**interface._docs_)\n    return intf_doc\n\n\ndef get_cmd_ex(interface):\n    \"\"\"Return the examples for the command defined by 'interface'.\n\n    Parameters\n    ----------\n    interface : subclass of Interface\n    \"\"\"\n    intf_ex = \"\\n\\n*Examples*\\n\\n\"\n    for example in interface._examples_:\n        intf_ex += build_example(example, api='cmdline')\n    return intf_ex\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>.*?\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>(.*?)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    if 'DATALAD_SPHINX_RUN' not in os.environ:\n        # remove :role:`...` RST markup for cmdline docs\n        docs = re.sub(\n            r':\\S+:`[^`]*`[\\\\]*',\n            lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n            docs,\n            flags=re.MULTILINE | re.DOTALL)\n        # make the handbook doc references more accessible\n        # the URL is a redirect configured at readthedocs\n        docs = re.sub(\n            r'(handbook:[0-9]-[0-9]*)',\n            '\\\\1 (http://handbook.datalad.org/symbols)',\n            docs)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        r'\\|\\| PYTHON \\>\\>.*?\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        r'\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        r'\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| CMDLINE \\>\\>(.*?)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # make the handbook doc references more accessible\n    # the URL is a redirect configured at readthedocs\n    docs = re.sub(\n        r'(handbook:[0-9]-[0-9]*)',\n        '\\\\1 (http://handbook.datalad.org/symbols)',\n        docs)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        r',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        r'`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        r'\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        r'\\|\\| REFLOW \\>\\>\\n(.*?)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from datalad.utils import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_example(example, api='python'):\n    \"\"\"Build a code example.\n\n    Take a dict from a classes _example_ specification (list of dicts) and\n    build a string with an api or cmd example (for use in cmd help or\n    docstring).\n\n    Parameters\n    ----------\n    api : {'python', 'cmdline'}\n        If 'python', build Python example for docstring. If 'cmdline', build\n        cmd example.\n\n    Returns\n    -------\n    ex : str\n        Concatenated examples for the given class.\n    \"\"\"\n    if api == 'python' :\n        code_field='code_py'\n        indicator='>'\n    elif api == 'cmdline':\n        code_field='code_cmd'\n        indicator='%'\n    else:\n        raise ValueError(\"unknown API selection: {}\".format(api))\n    if code_field not in example:\n        # only show an example if it exist for the API\n        return ''\n    description = textwrap.fill(example.get('text'))\n    # this indent the code snippet to get it properly rendered as code\n    # we are not using textwrap.fill(), because it would not acknowledge\n    # any meaningful structure/formatting of code snippets. Instead, we\n    # maintain line content as is.\n    code = dedent_docstring(example.get(code_field))\n    needs_indicator = not code.startswith(indicator)\n    code = textwrap.indent(code, ' ' * (5 if needs_indicator else 3)).lstrip()\n\n    ex = \"\"\"{}::\\n\\n   {}{}\\n\\n\"\"\".format(\n        description,\n        # disable automatic prefixing, if the example already has one\n        # this enables providing more complex examples without having\n        # to infer its inner structure\n        '{} '.format(indicator)\n        if needs_indicator\n        # maintain spacing to avoid undesired relative indentation\n        else '',\n        code)\n\n    return ex\n\n\ndef update_docstring_with_examples(cls_doc, ex):\n    \"\"\"Update a commands docstring with examples.\n\n    Take _examples_ of a command, build the Python examples, and append\n    them to the docstring.\n\n    cls_doc: docstring\n    ex: list\n        list of dicts with examples\n    \"\"\"\n    from textwrap import indent\n    if len(cls_doc):\n        cls_doc += \"\\n\"\n    cls_doc += \"    Examples\\n    --------\\n\"\n    # loop though provided examples\n    for example in ex:\n        cls_doc += indent(build_example(example, api='python'), ' '*4)\n\n    return cls_doc\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Note that values for any `eval_params` keys in `cls._params_` are\n    ignored.  This means one class may extend another's `_params_`\n    without worrying about filtering out `eval_params`.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n    # get examples\n    ex = getattr(cls, '_examples_', [])\n    if ex:\n        cls_doc = update_docstring_with_examples(cls_doc, ex)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n\n\n    # update class attributes that may override defaults \n    if hasattr(cls, '_no_eval_results'):\n        add_args = None\n    else:\n        add_args = {k: getattr(cls, k, v) for k, v in eval_defaults.items()}\n\n    # ATTN: An important consequence of this update() call is that it\n    # fulfills the docstring's promise of overriding any existing\n    # values for eval_params keys in _params_.\n    #\n\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=add_args\n    )\n\n    # return original\n    return cls\n\n\nNA_STRING = 'N/A'  # we might want to make it configurable via config\n\n\nclass nagen(object):\n    \"\"\"A helper to provide a desired missing value if no value is known\n\n    Usecases\n    - could be used as a generator for `defaultdict`\n    - since it returns itself upon getitem, should work even for complex\n      nested dictionaries/lists .format templates\n    \"\"\"\n    def __init__(self, missing=NA_STRING):\n        self.missing = missing\n\n    def __repr__(self):\n        cls = self.__class__.__name__\n        args = str(self.missing) if self.missing != NA_STRING else ''\n        return '%s(%s)' % (cls, args)\n\n    def __str__(self):\n        return self.missing\n\n    def __getitem__(self, *args):\n        return self\n\n    def __getattr__(self, item):\n        return self\n\n\ndef nadict(*items):\n    \"\"\"A generator of default dictionary with the default nagen\"\"\"\n    dd = defaultdict(nagen)\n    dd.update(*items)\n    return dd\n\n\nclass DefaultOutputFormatter(string.Formatter):\n    \"\"\"A custom formatter for default output rendering using .format\n    \"\"\"\n    # TODO: make missing configurable?\n    def __init__(self, missing=nagen()):\n        \"\"\"\n        Parameters\n        ----------\n        missing: string, optional\n          What to output for the missing values\n        \"\"\"\n        super(DefaultOutputFormatter, self).__init__()\n        self.missing = missing\n\n    def _d(self, msg, *args):\n        # print(\"   HERE %s\" % (msg % args))\n        pass\n\n    def get_value(self, key, args, kwds):\n        assert not args\n        self._d(\"get_value: %r %r %r\", key, args, kwds)\n        return kwds.get(key, self.missing)\n\n    # def get_field(self, field_name, args, kwds):\n    #     assert not args\n    #     self._d(\"get_field: %r args=%r kwds=%r\" % (field_name, args, kwds))\n    #     try:\n    #         out = string.Formatter.get_field(self, field_name, args, kwds)\n    #     except Exception as exc:\n    #         # TODO needs more than just a value\n    #         return \"!ERR %s\" % exc\n\n\nclass DefaultOutputRenderer(object):\n    \"\"\"A default renderer for .format'ed output line\n    \"\"\"\n    def __init__(self, format):\n        self.format = format\n        # We still need custom output formatter since at the \"first level\"\n        # within .format template all items there is no `nadict`\n        self.formatter = DefaultOutputFormatter()\n\n    @classmethod\n    def _dict_to_nadict(cls, v):\n        \"\"\"Traverse datastructure and replace any regular dict with nadict\"\"\"\n        if isinstance(v, list):\n            return [cls._dict_to_nadict(x) for x in v]\n        elif isinstance(v, dict):\n            return nadict((k, cls._dict_to_nadict(x)) for k, x in v.items())\n        else:\n            return v\n\n    def __call__(self, x, **kwargs):\n        dd = nadict(\n            (k, nadict({k_.replace(':', '#'): self._dict_to_nadict(v_)\n                for k_, v_ in v.items()})\n                if isinstance(v, dict) else v)\n            for k, v in x.items()\n        )\n\n        msg = self.formatter.format(self.format, **dd)\n        return ui.message(msg)\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'Export', 'Ls', 'SSHRun', 'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from datalad.utils import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args == tuple():\n                # explicitly provided an empty sequence of argument names\n                # this shall not appear in the parser\n                continue\n            elif cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help.rstrip()[-1] != '.':\n                help = help.rstrip() + '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n                if 'metavar' not in parser_kwargs and \\\n                        isinstance(param.constraints, EnsureChoice):\n                    parser_kwargs['metavar'] = \\\n                        '{%s}' % '|'.join(\n                            # don't use short_description(), because\n                            # it also needs to give valid output for\n                            # Python syntax (quotes...), but here we\n                            # can simplify to shell syntax where everything\n                            # is a string\n                            p for p in param.constraints._allowed\n                            # in the cmdline None pretty much means\n                            # don't give the options, so listing it\n                            # doesn't make sense. Moreover, any non-string\n                            # value cannot be given and very likely only\n                            # serves a special purpose in the Python API\n                            # or implementation details\n                            if isinstance(p, str))\n            if defaults_idx >= 0:\n                # if it is a flag, in commandline it makes little sense to show\n                # showing the Default: (likely boolean).\n                #   See https://github.com/datalad/datalad/issues/3203\n                if not parser_kwargs.get('action', '').startswith('store_'):\n                    # [Default: None] also makes little sense for cmdline\n                    if defaults[defaults_idx] is not None:\n                        help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from datalad.utils import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k)\n                  for k in argnames\n                  # some arguments might be Python-only and do not appear in the\n                  # parser Namespace\n                  if hasattr(args, k) and is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)\n\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = str(resolve_path(refds_path))\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from datalad.utils import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    defaults = defaults or []  # ensure it is a list and not None\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n",
            "header": {
                "file": {
                    "path": "datalad/interface/base.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "62abc6e72",
                "new_commit_hash": "fde19b4b0",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/base.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/base.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 837,
                            "length": 6
                        },
                        "new_range": {
                            "start": 837,
                            "length": 7
                        },
                        "section": "def get_allargs_as_kwargs(call, args, kwargs):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    argspec = getargspec(call)",
                                "type": "context"
                            },
                            {
                                "content": "    defaults = argspec.defaults",
                                "type": "context"
                            },
                            {
                                "content": "    nargs = len(argspec.args)",
                                "type": "context"
                            },
                            {
                                "content": "    defaults = defaults or []  # ensure it is a list and not None",
                                "type": "added"
                            },
                            {
                                "content": "    assert (nargs >= len(defaults))",
                                "type": "context"
                            },
                            {
                                "content": "    # map any args to their name",
                                "type": "context"
                            },
                            {
                                "content": "    argmap = list(zip(argspec.args[:len(args)], args))",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/base.py",
                            "type": "function",
                            "name": "get_allargs_as_kwargs",
                            "content": "def get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from datalad.utils import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    defaults = defaults or []  # ensure it is a list and not None\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 829,
                            "end_lineno": 855,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom ..base import Interface\nfrom ..utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    result_renderer = 'tailored' # overrides None default\n    return_type = 'item-or-list' # overrides 'list'\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring shows correct override values of defaults in eval_params\n    assert_in(\"Default: 'tailored'\", doc1)\n    assert_in(\"Default: 'item-or-list'\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom ..base import Interface\nfrom ..utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom ..results import get_status_dict\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    result_renderer = 'tailored' # overrides None default\n    return_type = 'item-or-list' # overrides 'list'\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring shows correct override values of defaults in eval_params\n    assert_in(\"Default: 'tailored'\", doc1)\n    assert_in(\"Default: 'item-or-list'\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n\nclass TestUtils2(Interface):\n    # result_renderer = custom_renderer\n    _params_ = dict(\n        number=Parameter(\n            args=(\"--path\",),\n            constraints=EnsureStr() | EnsureNone()),\n    )\n    @staticmethod\n    @eval_results\n    def __call__(path=None):\n        def logger(msg, *args):\n            return msg % args\n        if path:\n            # we will be testing for path %s\n            message = (\"all good %s\", \"my friend\")\n        else:\n            message = (\"kaboom %s %s\", \"greedy\")\n        yield get_status_dict(\n            action=\"test\",\n            status=\"ok\",\n            message=message,\n            logger=logger,\n            path=path or ''\n        )\n\n\ndef test_incorrect_msg_interpolation():\n    with assert_raises(TypeError) as cme:\n        TestUtils2().__call__()\n    # this must be our custom exception\n    assert_re_in(\"Failed to render.*kaboom.*not enough arguments\", str(cme.exception))\n\n    # there should be no exception if reported in the record path contains %\n    TestUtils2().__call__(\"%eatthis\")",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "87c46738c",
                "new_commit_hash": "120f61f54",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 50,
                            "length": 6
                        },
                        "new_range": {
                            "start": 50,
                            "length": 7
                        },
                        "section": "from ..utils import ("
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    eval_results,",
                                "type": "context"
                            },
                            {
                                "content": "    handle_dirty_dataset,",
                                "type": "context"
                            },
                            {
                                "content": ")",
                                "type": "context"
                            },
                            {
                                "content": "from ..results import get_status_dict",
                                "type": "added"
                            },
                            {
                                "content": "from datalad.interface.base import build_doc",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ..results import get_status_dict",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 53,
                            "end_lineno": 53,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 460,
                            "length": 3
                        },
                        "new_range": {
                            "start": 461,
                            "length": 39
                        },
                        "section": "def test_utils_suppress_similar():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        assert_in(\"path10\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_not_in(\"path20\", cmo.out)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "class TestUtils2(Interface):",
                                "type": "added"
                            },
                            {
                                "content": "    # result_renderer = custom_renderer",
                                "type": "added"
                            },
                            {
                                "content": "    _params_ = dict(",
                                "type": "added"
                            },
                            {
                                "content": "        number=Parameter(",
                                "type": "added"
                            },
                            {
                                "content": "            args=(\"--path\",),",
                                "type": "added"
                            },
                            {
                                "content": "            constraints=EnsureStr() | EnsureNone()),",
                                "type": "added"
                            },
                            {
                                "content": "    )",
                                "type": "added"
                            },
                            {
                                "content": "    @staticmethod",
                                "type": "added"
                            },
                            {
                                "content": "    @eval_results",
                                "type": "added"
                            },
                            {
                                "content": "    def __call__(path=None):",
                                "type": "added"
                            },
                            {
                                "content": "        def logger(msg, *args):",
                                "type": "added"
                            },
                            {
                                "content": "            return msg % args",
                                "type": "added"
                            },
                            {
                                "content": "        if path:",
                                "type": "added"
                            },
                            {
                                "content": "            # we will be testing for path %s",
                                "type": "added"
                            },
                            {
                                "content": "            message = (\"all good %s\", \"my friend\")",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            message = (\"kaboom %s %s\", \"greedy\")",
                                "type": "added"
                            },
                            {
                                "content": "        yield get_status_dict(",
                                "type": "added"
                            },
                            {
                                "content": "            action=\"test\",",
                                "type": "added"
                            },
                            {
                                "content": "            status=\"ok\",",
                                "type": "added"
                            },
                            {
                                "content": "            message=message,",
                                "type": "added"
                            },
                            {
                                "content": "            logger=logger,",
                                "type": "added"
                            },
                            {
                                "content": "            path=path or ''",
                                "type": "added"
                            },
                            {
                                "content": "        )",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def test_incorrect_msg_interpolation():",
                                "type": "added"
                            },
                            {
                                "content": "    with assert_raises(TypeError) as cme:",
                                "type": "added"
                            },
                            {
                                "content": "        TestUtils2().__call__()",
                                "type": "added"
                            },
                            {
                                "content": "    # this must be our custom exception",
                                "type": "added"
                            },
                            {
                                "content": "    assert_re_in(\"Failed to render.*kaboom.*not enough arguments\", str(cme.exception))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # there should be no exception if reported in the record path contains %",
                                "type": "added"
                            },
                            {
                                "content": "    TestUtils2().__call__(\"%eatthis\")",
                                "type": "added"
                            },
                            {
                                "content": "No newline at end of file",
                                "type": "note"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "method",
                            "name": "TestUtils2.__call__",
                            "content": "class TestUtils2(Interface):\n    # result_renderer = custom_renderer\n    _params_ = dict(\n        number=Parameter(\n            args=(\"--path\",),\n            constraints=EnsureStr() | EnsureNone()),\n    )\n    @staticmethod\n    @eval_results\n    def __call__(path=None):\n        def logger(msg, *args):\n            return msg % args\n        if path:\n            # we will be testing for path %s\n            message = (\"all good %s\", \"my friend\")\n        else:\n            message = (\"kaboom %s %s\", \"greedy\")\n        yield get_status_dict(\n            action=\"test\",\n            status=\"ok\",\n            message=message,\n            logger=logger,\n            path=path or ''\n        )",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 475,
                            "end_lineno": 489,
                            "parent": {
                                "file_name": "datalad/interface/tests/test_utils.py",
                                "type": "class",
                                "name": "TestUtils2",
                                "content": "class TestUtils2(Interface):\n    # result_renderer = custom_renderer\n    _params_ = dict(\n        number=Parameter(\n            args=(\"--path\",),\n            constraints=EnsureStr() | EnsureNone()),\n    )\n    @staticmethod\n    @eval_results\n    def __call__(path=None):\n        def logger(msg, *args):\n            return msg % args\n        if path:\n            # we will be testing for path %s\n            message = (\"all good %s\", \"my friend\")\n        else:\n            message = (\"kaboom %s %s\", \"greedy\")\n        yield get_status_dict(\n            action=\"test\",\n            status=\"ok\",\n            message=message,\n            logger=logger,\n            path=path or ''\n        )",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 466,
                                "end_lineno": 489,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "class",
                            "name": "TestUtils2",
                            "content": "class TestUtils2(Interface):\n    # result_renderer = custom_renderer\n    _params_ = dict(\n        number=Parameter(\n            args=(\"--path\",),\n            constraints=EnsureStr() | EnsureNone()),\n    )\n    @staticmethod\n    @eval_results\n    def __call__(path=None):\n        def logger(msg, *args):\n            return msg % args\n        if path:\n            # we will be testing for path %s\n            message = (\"all good %s\", \"my friend\")\n        else:\n            message = (\"kaboom %s %s\", \"greedy\")\n        yield get_status_dict(\n            action=\"test\",\n            status=\"ok\",\n            message=message,\n            logger=logger,\n            path=path or ''\n        )",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 466,
                            "end_lineno": 489,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/interface/tests/test_utils.py",
                            "type": "function",
                            "name": "test_incorrect_msg_interpolation",
                            "content": "def test_incorrect_msg_interpolation():\n    with assert_raises(TypeError) as cme:\n        TestUtils2().__call__()\n    # this must be our custom exception\n    assert_re_in(\"Failed to render.*kaboom.*not enough arguments\", str(cme.exception))\n\n    # there should be no exception if reported in the record path contains %\n    TestUtils2().__call__(\"%eatthis\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 492,
                            "end_lineno": 499,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport inspect\nimport logging\nimport wrapt\nimport sys\nimport re\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nimport os.path as op\nfrom os.path import join as opj\nfrom os.path import isdir\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\nimport json\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import (\n    path_startswith,\n    path_is_subpath,\n    assure_unicode,\n    getargspec,\n    get_wrapped_class,\n)\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad import cfg as dlcfg\nfrom datalad.dochelpers import (\n    exc_str,\n    single_or_plural,\n)\n\nfrom datalad.ui import ui\nimport datalad.support.ansi_colors as ac\n\nfrom datalad.interface.base import default_logchannels\nfrom datalad.interface.base import get_allargs_as_kwargs\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom .results import known_result_xfms\nfrom datalad.config import ConfigManager\nfrom datalad.core.local.resulthooks import (\n    get_jsonhooks_from_config,\n    match_jsonhook2result,\n    run_jsonhook,\n)\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef cls2cmdlinename(cls):\n    \"Return the cmdline command name from an Interface class\"\n    r = re.compile(r'([a-z0-9])([A-Z])')\n    return r.sub('\\\\1-\\\\2', cls.__name__).lower()\n\n\n# TODO remove\n# only `drop` and `uninstall` are still using this\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.dirty:\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.core.local.save import Save\n        Save.__call__(dataset=ds, message=msg, updated=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\n# TODO remove\n# only `remove` and `uninstall` use this, the uses path `path_is_subpath`\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        path = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    path_drive, _ = op.splitdrive(path)\n    for p in values:\n        p_drive, _ = op.splitdrive(p)\n        # need to protect against unsupported use of relpath() with\n        # abspaths on windows from different drives (gh-3724)\n        if path_drive != p_drive:\n        # different drives, enough evidence for \"not under\"\n            continue\n        rpath = relpath(p, start=path)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,\n                                      spec, includeds=None):\n    \"\"\"Discover the edges and nodes in a dataset tree to given target paths\n\n    Parameters\n    ----------\n    basepath : path\n      Path to a start or top-level dataset. Really has to be a path to a\n      dataset!\n    targetpaths : list(path)\n      Any non-zero number of paths that are termination points for the\n      search algorithm. Can be paths to datasets, directories, or files\n      (and any combination thereof).\n    current_trace : list\n      For a top-level call this should probably always be `[]`\n    spec : dict\n      `content_by_ds`-style dictionary that will receive information about the\n      discovered datasets. Specifically, for each discovered dataset there\n      will be in item with its path under the key (path) of the respective\n      superdataset.\n    includeds : sequence, optional\n      Any paths given are treated as existing subdatasets, regardless of\n      whether they can be found in the filesystem. Such subdatasets will appear\n      under the key of the closest existing dataset in the `spec`.\n\n    Returns\n    -------\n    None\n      Function calls itself recursively and populates `spec` dict in-place.\n      Keys are dataset paths, values are sets of subdataset paths\n    \"\"\"\n    # convert to set for faster lookup\n    includeds = includeds if isinstance(includeds, set) else \\\n        set() if includeds is None else set(includeds)\n    # this beast walks the directory tree from a given `basepath` until\n    # it discovers any of the given `targetpaths`\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(basepath)\n    if valid_repo:\n        # we are passing into a new dataset, extend the dataset trace\n        current_trace = current_trace + [basepath]\n    # this edge is not done, we need to try to reach any downstream\n    # dataset\n    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)\n    # whether anything in this directory matched a targetpath\n    filematch = False\n    if isdir(basepath):\n        for p in listdir(basepath):\n            p = assure_unicode(opj(basepath, p))\n            if not isdir(p):\n                if p in targetpaths:\n                    filematch = True\n                # we cannot have anything below this one\n                continue\n            # OPT listdir might be large and we could have only few items\n            # in `targetpaths` -- so traverse only those in spec which have\n            # leading dir basepath\n            # filter targets matching this downward path\n            downward_targets = set(\n                t for t in targetpaths if path_startswith(t, p))\n            if not downward_targets:\n                continue\n            # remove the matching ones from the \"todo\" list\n            undiscovered_ds.difference_update(downward_targets)\n            # go one deeper\n            discover_dataset_trace_to_targets(\n                p, downward_targets, current_trace, spec,\n                includeds=includeds if not includeds else includeds.intersection(\n                    downward_targets))\n    undiscovered_ds = [t for t in undiscovered_ds\n                       if includeds and\n                          path_is_subpath(t, current_trace[-1]) and\n                          t in includeds]\n    if filematch or basepath in targetpaths or undiscovered_ds:\n        for i, p in enumerate(current_trace[:-1]):\n            # TODO RF prepare proper annotated path dicts\n            subds = spec.get(p, set())\n            subds.add(current_trace[i + 1])\n            spec[p] = subds\n        if undiscovered_ds:\n            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(\n                undiscovered_ds)\n\n\ndef get_result_filter(fx):\n    \"\"\"Wrap a filter into a helper to be able to accept additional\n    arguments, if the filter doesn't support it already\"\"\"\n    _fx = fx\n    if fx and not getargspec(fx).keywords:\n        def _fx(res, **kwargs):\n            return fx(res)\n    return _fx\n\n\ndef eval_results(func):\n    \"\"\"Decorator for return value evaluation of datalad commands.\n\n    Note, this decorator is only compatible with commands that return\n    status dict sequences!\n\n    Two basic modes of operation are supported: 1) \"generator mode\" that\n    `yields` individual results, and 2) \"list mode\" that returns a sequence of\n    results. The behavior can be selected via the kwarg `return_type`.\n    Default is \"list mode\".\n\n    This decorator implements common functionality for result rendering/output,\n    error detection/handling, and logging.\n\n    Result rendering/output can be triggered via the\n    `datalad.api.result-renderer` configuration variable, or the\n    `result_renderer` keyword argument of each decorated command. Supported\n    modes are: 'default' (one line per result with action, status, path,\n    and an optional message); 'json' (one object per result, like git-annex),\n    'json_pp' (like 'json', but pretty-printed spanning multiple lines),\n    'tailored' custom output formatting provided by each command\n    class (if any).\n\n    Error detection works by inspecting the `status` item of all result\n    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'\n    will cause an IncompleteResultsError exception to be raised that carries\n    the failed actions' status dictionaries in its `failed` attribute.\n\n    Status messages will be logged automatically, by default the following\n    association of result status and log channel will be used: 'ok' (debug),\n    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger\n    instances included in the results are used to capture the origin of a\n    status report.\n\n    Parameters\n    ----------\n    func: function\n      __call__ method of a subclass of Interface,\n      i.e. a datalad command definition\n    \"\"\"\n\n    @wrapt.decorator\n    def eval_func(wrapped, instance, args, kwargs):\n        lgr.log(2, \"Entered eval_func for %s\", func)\n        # for result filters\n        # we need to produce a dict with argname/argvalue pairs for all args\n        # incl. defaults and args given as positionals\n        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)\n\n        # determine the command class associated with `wrapped`\n        wrapped_class = get_wrapped_class(wrapped)\n\n        # retrieve common options from kwargs, and fall back on the command\n        # class attributes, or general defaults if needed\n        kwargs = kwargs.copy()  # we will pop, which might cause side-effect\n        common_params = {\n            p_name: kwargs.pop(\n                # go with any explicitly given default\n                p_name,\n                # otherwise determine the command class and pull any\n                # default set in that class\n                getattr(\n                    wrapped_class,\n                    p_name,\n                    # or the common default\n                    eval_defaults[p_name]))\n            for p_name in eval_params}\n\n        # short cuts and configured setup for common options\n        return_type = common_params['return_type']\n        result_filter = get_result_filter(common_params['result_filter'])\n        # resolve string labels for transformers too\n        result_xfm = known_result_xfms.get(\n            common_params['result_xfm'],\n            # use verbatim, if not a known label\n            common_params['result_xfm'])\n        result_renderer = common_params['result_renderer']\n        # TODO remove this conditional branch entirely, done outside\n        if not result_renderer:\n            result_renderer = dlcfg.get('datalad.api.result-renderer', None)\n        # look for potential override of logging behavior\n        result_log_level = dlcfg.get('datalad.log.result-level', None)\n\n        # query cfg for defaults\n        # .is_installed and .config can be costly, so ensure we do\n        # it only once. See https://github.com/datalad/datalad/issues/3575\n        dataset_arg = allkwargs.get('dataset', None)\n        from datalad.distribution.dataset import Dataset\n        ds = dataset_arg if isinstance(dataset_arg, Dataset) \\\n            else Dataset(dataset_arg) if dataset_arg else None\n        # do not reuse a dataset's existing config manager here\n        # they are configured to read the committed dataset configuration\n        # too. That means a datalad update can silently bring in new\n        # procedure definitions from the outside, and in some sense enable\n        # remote code execution by a 3rd-party\n        # To avoid that, create a new config manager that only reads local\n        # config (system and .git/config), plus any overrides given to this\n        # datalad session\n        proc_cfg = ConfigManager(\n            ds, source='local', overrides=dlcfg.overrides\n        ) if ds and ds.is_installed() else dlcfg\n\n        # look for hooks\n        hooks = get_jsonhooks_from_config(proc_cfg)\n\n        # this internal helper function actually drives the command\n        # generator-style, it may generate an exception if desired,\n        # on incomplete results\n        def generator_func(*_args, **_kwargs):\n            # flag whether to raise an exception\n            incomplete_results = []\n            # track what actions were performed how many times\n            action_summary = {}\n\n            # if a custom summary is to be provided, collect the results\n            # of the command execution\n            results = []\n            do_custom_result_summary = result_renderer in ('tailored', 'default') \\\n                and hasattr(wrapped_class, 'custom_result_summary_renderer')\n\n            # process main results\n            for r in _process_results(\n                    # execution\n                    wrapped(*_args, **_kwargs),\n                    wrapped_class,\n                    common_params['on_failure'],\n                    # bookkeeping\n                    action_summary,\n                    incomplete_results,\n                    # communication\n                    result_renderer,\n                    result_log_level,\n                    # let renderers get to see how a command was called\n                    allkwargs):\n                for hook, spec in hooks.items():\n                    # run the hooks before we yield the result\n                    # this ensures that they are executed before\n                    # a potentially wrapper command gets to act\n                    # on them\n                    if match_jsonhook2result(hook, r, spec['match']):\n                        lgr.debug('Result %s matches hook %s', r, hook)\n                        # a hook is also a command that yields results\n                        # so yield them outside too\n                        # users need to pay attention to void infinite\n                        # loops, i.e. when a hook yields a result that\n                        # triggers that same hook again\n                        for hr in run_jsonhook(hook, spec, r, dataset_arg):\n                            # apply same logic as for main results, otherwise\n                            # any filters would only tackle the primary results\n                            # and a mixture of return values could happen\n                            if not keep_result(hr, result_filter, **allkwargs):\n                                continue\n                            hr = xfm_result(hr, result_xfm)\n                            # rationale for conditional is a few lines down\n                            if hr:\n                                yield hr\n                if not keep_result(r, result_filter, **allkwargs):\n                    continue\n                r = xfm_result(r, result_xfm)\n                # in case the result_xfm decided to not give us anything\n                # exclude it from the results. There is no particular reason\n                # to do so other than that it was established behavior when\n                # this comment was written. This will not affect any real\n                # result record\n                if r:\n                    yield r\n\n                # collect if summary is desired\n                if do_custom_result_summary:\n                    results.append(r)\n\n            # result summary before a potential exception\n            # custom first\n            if do_custom_result_summary:\n                wrapped_class.custom_result_summary_renderer(results)\n            elif result_renderer == 'default' and action_summary and \\\n                    sum(sum(s.values()) for s in action_summary.values()) > 1:\n                # give a summary in default mode, when there was more than one\n                # action performed\n                ui.message(\"action summary:\\n  {}\".format(\n                    '\\n  '.join('{} ({})'.format(\n                        act,\n                        ', '.join('{}: {}'.format(status, action_summary[act][status])\n                                  for status in sorted(action_summary[act])))\n                                for act in sorted(action_summary))))\n\n            if incomplete_results:\n                raise IncompleteResultsError(\n                    failed=incomplete_results,\n                    msg=\"Command did not complete successfully\")\n\n        if return_type == 'generator':\n            # hand over the generator\n            lgr.log(2, \"Returning generator_func from eval_func for %s\", wrapped_class)\n            return generator_func(*args, **kwargs)\n        else:\n            @wrapt.decorator\n            def return_func(wrapped_, instance_, args_, kwargs_):\n                results = wrapped_(*args_, **kwargs_)\n                if inspect.isgenerator(results):\n                    # unwind generator if there is one, this actually runs\n                    # any processing\n                    results = list(results)\n                # render summaries\n                if not result_xfm and result_renderer in ('tailored', 'default'):\n                    # cannot render transformed results\n                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):\n                        wrapped_class.custom_result_summary_renderer(results)\n                if return_type == 'item-or-list' and \\\n                        len(results) < 2:\n                    return results[0] if results else None\n                else:\n                    return results\n            lgr.log(2, \"Returning return_func from eval_func for %s\", wrapped_class)\n            return return_func(generator_func)(*args, **kwargs)\n\n    return eval_func(func)\n\n\ndef default_result_renderer(res):\n    if res.get('status', None) != 'notneeded':\n        path = res.get('path', None)\n        path = ' {}'.format(path) if path else ''\n        ui.message('{action}({status}):{path}{type}{msg}'.format(\n                action=ac.color_word(res['action'], ac.BOLD),\n                status=ac.color_status(res['status']),\n                path=relpath(path, res['refds']) if path and res.get('refds') else path,\n                type=' ({})'.format(\n                        ac.color_word(res['type'], ac.MAGENTA)\n                ) if 'type' in res else '',\n                msg=' [{}]'.format(\n                        res['message'][0] % res['message'][1:]\n                        if isinstance(res['message'], tuple) else res[\n                            'message'])\n                if res.get('message', None) else ''))\n\n\ndef _display_suppressed_message(nsimilar, ndisplayed, final=False):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)),\n                   cr=\"\\n\" if final else \"\\r\")\n\n\ndef _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10 if sys.stdout.isatty() else float(\"inf\")\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n                else:\n                    _display_suppressed_message(\n                        result_repetitions, render_n_repetitions)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions,\n                    final=True)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions, final=True)\n\n\ndef keep_result(res, rfilter, **kwargs):\n    if not rfilter:\n        return True\n    try:\n        if not rfilter(res, **kwargs):\n            # give the slightest indication which filter was employed\n            raise ValueError(\n                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))\n    except ValueError as e:\n        # make sure to report the excluded result to massively improve\n        # debugging experience\n        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)\n        return False\n    return True\n\n\ndef xfm_result(res, xfm):\n    if not xfm:\n        return res\n\n    return xfm(res)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport inspect\nimport logging\nimport wrapt\nimport sys\nimport re\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nimport os.path as op\nfrom os.path import join as opj\nfrom os.path import isdir\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\nimport json\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import (\n    path_startswith,\n    path_is_subpath,\n    assure_unicode,\n    getargspec,\n    get_wrapped_class,\n)\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad import cfg as dlcfg\nfrom datalad.dochelpers import (\n    exc_str,\n    single_or_plural,\n)\n\nfrom datalad.ui import ui\nimport datalad.support.ansi_colors as ac\n\nfrom datalad.interface.base import default_logchannels\nfrom datalad.interface.base import get_allargs_as_kwargs\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom .results import known_result_xfms\nfrom datalad.config import ConfigManager\nfrom datalad.core.local.resulthooks import (\n    get_jsonhooks_from_config,\n    match_jsonhook2result,\n    run_jsonhook,\n)\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef cls2cmdlinename(cls):\n    \"Return the cmdline command name from an Interface class\"\n    r = re.compile(r'([a-z0-9])([A-Z])')\n    return r.sub('\\\\1-\\\\2', cls.__name__).lower()\n\n\n# TODO remove\n# only `drop` and `uninstall` are still using this\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.dirty:\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.core.local.save import Save\n        Save.__call__(dataset=ds, message=msg, updated=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\n# TODO remove\n# only `remove` and `uninstall` use this, the uses path `path_is_subpath`\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        path = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    path_drive, _ = op.splitdrive(path)\n    for p in values:\n        p_drive, _ = op.splitdrive(p)\n        # need to protect against unsupported use of relpath() with\n        # abspaths on windows from different drives (gh-3724)\n        if path_drive != p_drive:\n        # different drives, enough evidence for \"not under\"\n            continue\n        rpath = relpath(p, start=path)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,\n                                      spec, includeds=None):\n    \"\"\"Discover the edges and nodes in a dataset tree to given target paths\n\n    Parameters\n    ----------\n    basepath : path\n      Path to a start or top-level dataset. Really has to be a path to a\n      dataset!\n    targetpaths : list(path)\n      Any non-zero number of paths that are termination points for the\n      search algorithm. Can be paths to datasets, directories, or files\n      (and any combination thereof).\n    current_trace : list\n      For a top-level call this should probably always be `[]`\n    spec : dict\n      `content_by_ds`-style dictionary that will receive information about the\n      discovered datasets. Specifically, for each discovered dataset there\n      will be in item with its path under the key (path) of the respective\n      superdataset.\n    includeds : sequence, optional\n      Any paths given are treated as existing subdatasets, regardless of\n      whether they can be found in the filesystem. Such subdatasets will appear\n      under the key of the closest existing dataset in the `spec`.\n\n    Returns\n    -------\n    None\n      Function calls itself recursively and populates `spec` dict in-place.\n      Keys are dataset paths, values are sets of subdataset paths\n    \"\"\"\n    # convert to set for faster lookup\n    includeds = includeds if isinstance(includeds, set) else \\\n        set() if includeds is None else set(includeds)\n    # this beast walks the directory tree from a given `basepath` until\n    # it discovers any of the given `targetpaths`\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(basepath)\n    if valid_repo:\n        # we are passing into a new dataset, extend the dataset trace\n        current_trace = current_trace + [basepath]\n    # this edge is not done, we need to try to reach any downstream\n    # dataset\n    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)\n    # whether anything in this directory matched a targetpath\n    filematch = False\n    if isdir(basepath):\n        for p in listdir(basepath):\n            p = assure_unicode(opj(basepath, p))\n            if not isdir(p):\n                if p in targetpaths:\n                    filematch = True\n                # we cannot have anything below this one\n                continue\n            # OPT listdir might be large and we could have only few items\n            # in `targetpaths` -- so traverse only those in spec which have\n            # leading dir basepath\n            # filter targets matching this downward path\n            downward_targets = set(\n                t for t in targetpaths if path_startswith(t, p))\n            if not downward_targets:\n                continue\n            # remove the matching ones from the \"todo\" list\n            undiscovered_ds.difference_update(downward_targets)\n            # go one deeper\n            discover_dataset_trace_to_targets(\n                p, downward_targets, current_trace, spec,\n                includeds=includeds if not includeds else includeds.intersection(\n                    downward_targets))\n    undiscovered_ds = [t for t in undiscovered_ds\n                       if includeds and\n                          path_is_subpath(t, current_trace[-1]) and\n                          t in includeds]\n    if filematch or basepath in targetpaths or undiscovered_ds:\n        for i, p in enumerate(current_trace[:-1]):\n            # TODO RF prepare proper annotated path dicts\n            subds = spec.get(p, set())\n            subds.add(current_trace[i + 1])\n            spec[p] = subds\n        if undiscovered_ds:\n            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(\n                undiscovered_ds)\n\n\ndef get_result_filter(fx):\n    \"\"\"Wrap a filter into a helper to be able to accept additional\n    arguments, if the filter doesn't support it already\"\"\"\n    _fx = fx\n    if fx and not getargspec(fx).keywords:\n        def _fx(res, **kwargs):\n            return fx(res)\n    return _fx\n\n\ndef eval_results(func):\n    \"\"\"Decorator for return value evaluation of datalad commands.\n\n    Note, this decorator is only compatible with commands that return\n    status dict sequences!\n\n    Two basic modes of operation are supported: 1) \"generator mode\" that\n    `yields` individual results, and 2) \"list mode\" that returns a sequence of\n    results. The behavior can be selected via the kwarg `return_type`.\n    Default is \"list mode\".\n\n    This decorator implements common functionality for result rendering/output,\n    error detection/handling, and logging.\n\n    Result rendering/output can be triggered via the\n    `datalad.api.result-renderer` configuration variable, or the\n    `result_renderer` keyword argument of each decorated command. Supported\n    modes are: 'default' (one line per result with action, status, path,\n    and an optional message); 'json' (one object per result, like git-annex),\n    'json_pp' (like 'json', but pretty-printed spanning multiple lines),\n    'tailored' custom output formatting provided by each command\n    class (if any).\n\n    Error detection works by inspecting the `status` item of all result\n    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'\n    will cause an IncompleteResultsError exception to be raised that carries\n    the failed actions' status dictionaries in its `failed` attribute.\n\n    Status messages will be logged automatically, by default the following\n    association of result status and log channel will be used: 'ok' (debug),\n    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger\n    instances included in the results are used to capture the origin of a\n    status report.\n\n    Parameters\n    ----------\n    func: function\n      __call__ method of a subclass of Interface,\n      i.e. a datalad command definition\n    \"\"\"\n\n    @wrapt.decorator\n    def eval_func(wrapped, instance, args, kwargs):\n        lgr.log(2, \"Entered eval_func for %s\", func)\n        # for result filters\n        # we need to produce a dict with argname/argvalue pairs for all args\n        # incl. defaults and args given as positionals\n        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)\n\n        # determine the command class associated with `wrapped`\n        wrapped_class = get_wrapped_class(wrapped)\n\n        # retrieve common options from kwargs, and fall back on the command\n        # class attributes, or general defaults if needed\n        kwargs = kwargs.copy()  # we will pop, which might cause side-effect\n        common_params = {\n            p_name: kwargs.pop(\n                # go with any explicitly given default\n                p_name,\n                # otherwise determine the command class and pull any\n                # default set in that class\n                getattr(\n                    wrapped_class,\n                    p_name,\n                    # or the common default\n                    eval_defaults[p_name]))\n            for p_name in eval_params}\n\n        # short cuts and configured setup for common options\n        return_type = common_params['return_type']\n        result_filter = get_result_filter(common_params['result_filter'])\n        # resolve string labels for transformers too\n        result_xfm = known_result_xfms.get(\n            common_params['result_xfm'],\n            # use verbatim, if not a known label\n            common_params['result_xfm'])\n        result_renderer = common_params['result_renderer']\n        # TODO remove this conditional branch entirely, done outside\n        if not result_renderer:\n            result_renderer = dlcfg.get('datalad.api.result-renderer', None)\n        # look for potential override of logging behavior\n        result_log_level = dlcfg.get('datalad.log.result-level', None)\n\n        # query cfg for defaults\n        # .is_installed and .config can be costly, so ensure we do\n        # it only once. See https://github.com/datalad/datalad/issues/3575\n        dataset_arg = allkwargs.get('dataset', None)\n        from datalad.distribution.dataset import Dataset\n        ds = dataset_arg if isinstance(dataset_arg, Dataset) \\\n            else Dataset(dataset_arg) if dataset_arg else None\n        # do not reuse a dataset's existing config manager here\n        # they are configured to read the committed dataset configuration\n        # too. That means a datalad update can silently bring in new\n        # procedure definitions from the outside, and in some sense enable\n        # remote code execution by a 3rd-party\n        # To avoid that, create a new config manager that only reads local\n        # config (system and .git/config), plus any overrides given to this\n        # datalad session\n        proc_cfg = ConfigManager(\n            ds, source='local', overrides=dlcfg.overrides\n        ) if ds and ds.is_installed() else dlcfg\n\n        # look for hooks\n        hooks = get_jsonhooks_from_config(proc_cfg)\n\n        # this internal helper function actually drives the command\n        # generator-style, it may generate an exception if desired,\n        # on incomplete results\n        def generator_func(*_args, **_kwargs):\n            # flag whether to raise an exception\n            incomplete_results = []\n            # track what actions were performed how many times\n            action_summary = {}\n\n            # if a custom summary is to be provided, collect the results\n            # of the command execution\n            results = []\n            do_custom_result_summary = result_renderer in ('tailored', 'default') \\\n                and hasattr(wrapped_class, 'custom_result_summary_renderer')\n\n            # process main results\n            for r in _process_results(\n                    # execution\n                    wrapped(*_args, **_kwargs),\n                    wrapped_class,\n                    common_params['on_failure'],\n                    # bookkeeping\n                    action_summary,\n                    incomplete_results,\n                    # communication\n                    result_renderer,\n                    result_log_level,\n                    # let renderers get to see how a command was called\n                    allkwargs):\n                for hook, spec in hooks.items():\n                    # run the hooks before we yield the result\n                    # this ensures that they are executed before\n                    # a potentially wrapper command gets to act\n                    # on them\n                    if match_jsonhook2result(hook, r, spec['match']):\n                        lgr.debug('Result %s matches hook %s', r, hook)\n                        # a hook is also a command that yields results\n                        # so yield them outside too\n                        # users need to pay attention to void infinite\n                        # loops, i.e. when a hook yields a result that\n                        # triggers that same hook again\n                        for hr in run_jsonhook(hook, spec, r, dataset_arg):\n                            # apply same logic as for main results, otherwise\n                            # any filters would only tackle the primary results\n                            # and a mixture of return values could happen\n                            if not keep_result(hr, result_filter, **allkwargs):\n                                continue\n                            hr = xfm_result(hr, result_xfm)\n                            # rationale for conditional is a few lines down\n                            if hr:\n                                yield hr\n                if not keep_result(r, result_filter, **allkwargs):\n                    continue\n                r = xfm_result(r, result_xfm)\n                # in case the result_xfm decided to not give us anything\n                # exclude it from the results. There is no particular reason\n                # to do so other than that it was established behavior when\n                # this comment was written. This will not affect any real\n                # result record\n                if r:\n                    yield r\n\n                # collect if summary is desired\n                if do_custom_result_summary:\n                    results.append(r)\n\n            # result summary before a potential exception\n            # custom first\n            if do_custom_result_summary:\n                wrapped_class.custom_result_summary_renderer(results)\n            elif result_renderer == 'default' and action_summary and \\\n                    sum(sum(s.values()) for s in action_summary.values()) > 1:\n                # give a summary in default mode, when there was more than one\n                # action performed\n                ui.message(\"action summary:\\n  {}\".format(\n                    '\\n  '.join('{} ({})'.format(\n                        act,\n                        ', '.join('{}: {}'.format(status, action_summary[act][status])\n                                  for status in sorted(action_summary[act])))\n                                for act in sorted(action_summary))))\n\n            if incomplete_results:\n                raise IncompleteResultsError(\n                    failed=incomplete_results,\n                    msg=\"Command did not complete successfully\")\n\n        if return_type == 'generator':\n            # hand over the generator\n            lgr.log(2, \"Returning generator_func from eval_func for %s\", wrapped_class)\n            return generator_func(*args, **kwargs)\n        else:\n            @wrapt.decorator\n            def return_func(wrapped_, instance_, args_, kwargs_):\n                results = wrapped_(*args_, **kwargs_)\n                if inspect.isgenerator(results):\n                    # unwind generator if there is one, this actually runs\n                    # any processing\n                    results = list(results)\n                # render summaries\n                if not result_xfm and result_renderer in ('tailored', 'default'):\n                    # cannot render transformed results\n                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):\n                        wrapped_class.custom_result_summary_renderer(results)\n                if return_type == 'item-or-list' and \\\n                        len(results) < 2:\n                    return results[0] if results else None\n                else:\n                    return results\n            lgr.log(2, \"Returning return_func from eval_func for %s\", wrapped_class)\n            return return_func(generator_func)(*args, **kwargs)\n\n    return eval_func(func)\n\n\ndef default_result_renderer(res):\n    if res.get('status', None) != 'notneeded':\n        path = res.get('path', None)\n        path = ' {}'.format(path) if path else ''\n        ui.message('{action}({status}):{path}{type}{msg}'.format(\n                action=ac.color_word(res['action'], ac.BOLD),\n                status=ac.color_status(res['status']),\n                path=relpath(path, res['refds']) if path and res.get('refds') else path,\n                type=' ({})'.format(\n                        ac.color_word(res['type'], ac.MAGENTA)\n                ) if 'type' in res else '',\n                msg=' [{}]'.format(\n                        res['message'][0] % res['message'][1:]\n                        if isinstance(res['message'], tuple) else res[\n                            'message'])\n                if res.get('message', None) else ''))\n\n\ndef _display_suppressed_message(nsimilar, ndisplayed, final=False):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)),\n                   cr=\"\\n\" if final else \"\\r\")\n\n\ndef _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10 if sys.stdout.isatty() else float(\"inf\")\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                path = res['path']\n                if msgargs:\n                    # we will pass the msg for %-polation, so % should be doubled\n                    path = path.replace('%', '%%')\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], path)\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                try:\n                    res_lgr(msg, *msgargs)\n                except TypeError as exc:\n                    raise TypeError(\n                        \"Failed to render %r with %r from %r: %s\"\n                        % (msg, msgargs, res, exc_str(exc))\n                    )\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n                else:\n                    _display_suppressed_message(\n                        result_repetitions, render_n_repetitions)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions,\n                    final=True)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions, final=True)\n\n\ndef keep_result(res, rfilter, **kwargs):\n    if not rfilter:\n        return True\n    try:\n        if not rfilter(res, **kwargs):\n            # give the slightest indication which filter was employed\n            raise ValueError(\n                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))\n    except ValueError as e:\n        # make sure to report the excluded result to massively improve\n        # debugging experience\n        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)\n        return False\n    return True\n\n\ndef xfm_result(res, xfm):\n    if not xfm:\n        return res\n\n    return xfm(res)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "696e600b6",
                "new_commit_hash": "42f51e0de",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/utils.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 579,
                            "length": 11
                        },
                        "new_range": {
                            "start": 579,
                            "length": 21
                        },
                        "section": "def _process_results("
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                msgargs = msg[1:]",
                                "type": "context"
                            },
                            {
                                "content": "                msg = msg[0]",
                                "type": "context"
                            },
                            {
                                "content": "            if 'path' in res:",
                                "type": "context"
                            },
                            {
                                "content": "                path = res['path']",
                                "type": "added"
                            },
                            {
                                "content": "                if msgargs:",
                                "type": "added"
                            },
                            {
                                "content": "                    # we will pass the msg for %-polation, so % should be doubled",
                                "type": "added"
                            },
                            {
                                "content": "                    path = path.replace('%', '%%')",
                                "type": "added"
                            },
                            {
                                "content": "                msg = '{} [{}({})]'.format(",
                                "type": "context"
                            },
                            {
                                "content": "                    msg, res['action'], res['path'])",
                                "type": "deleted"
                            },
                            {
                                "content": "                    msg, res['action'], path)",
                                "type": "added"
                            },
                            {
                                "content": "            if msgargs:",
                                "type": "context"
                            },
                            {
                                "content": "                # support string expansion of logging to avoid runtime cost",
                                "type": "context"
                            },
                            {
                                "content": "                res_lgr(msg, *msgargs)",
                                "type": "deleted"
                            },
                            {
                                "content": "                try:",
                                "type": "added"
                            },
                            {
                                "content": "                    res_lgr(msg, *msgargs)",
                                "type": "added"
                            },
                            {
                                "content": "                except TypeError as exc:",
                                "type": "added"
                            },
                            {
                                "content": "                    raise TypeError(",
                                "type": "added"
                            },
                            {
                                "content": "                        \"Failed to render %r with %r from %r: %s\"",
                                "type": "added"
                            },
                            {
                                "content": "                        % (msg, msgargs, res, exc_str(exc))",
                                "type": "added"
                            },
                            {
                                "content": "                    )",
                                "type": "added"
                            },
                            {
                                "content": "            else:",
                                "type": "context"
                            },
                            {
                                "content": "                res_lgr(msg)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/utils.py",
                            "type": "function",
                            "name": "_process_results",
                            "content": "def _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10 if sys.stdout.isatty() else float(\"inf\")\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n                else:\n                    _display_suppressed_message(\n                        result_repetitions, render_n_repetitions)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions,\n                    final=True)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions, final=True)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 529,
                            "end_lineno": 646,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Various handlers/functionality for different types of files (e.g. for archives)\n\n\"\"\"\n\nimport hashlib\nimport os\nimport tempfile\nfrom urllib.parse import unquote as urlunquote\nimport string\nimport random\nimport logging\n\nfrom datalad.support.path import (\n    join as opj,\n    exists,\n    abspath,\n    isabs,\n    normpath,\n    relpath,\n    pardir,\n    isdir,\n    sep as opsep,\n)\n\nfrom datalad.support.locking import lock_if_check_fails\nfrom datalad.support.external_versions import external_versions\nfrom datalad.consts import ARCHIVES_TEMP_DIR\nfrom datalad.utils import (\n    any_re_search,\n    ensure_bytes,\n    ensure_unicode,\n    unlink,\n    rmdir,\n    rmtemp,\n    rmtree,\n    get_tempfile_kwargs,\n    on_windows,\n    Path,\n)\nfrom datalad import cfg\nfrom datalad.config import anything2bool\n\n# fall back on patool, if a functional implementation is available\n# (i.e. not on windows), it is requested, or 7z is not found\nif not on_windows and (\n        cfg.obtain(\n            'datalad.runtime.use-patool', default=False,\n            valtype=anything2bool) or not external_versions['cmd:7z']):\n    from datalad.support.archive_utils_patool import (\n        decompress_file as _decompress_file,\n        # other code expects this to be here\n        compress_files\n    )\nelse:\n    from datalad.support.archive_utils_7z import (\n        decompress_file as _decompress_file,\n        # other code expects this to be here\n        compress_files\n    )\n\nlgr = logging.getLogger('datalad.support.archives')\n\n\ndef decompress_file(archive, dir_, leading_directories='strip'):\n    \"\"\"Decompress `archive` into a directory `dir_`\n\n    Parameters\n    ----------\n    archive: str\n    dir_: str\n    leading_directories: {'strip', None}\n      If `strip`, and archive contains a single leading directory under which\n      all content is stored, all the content will be moved one directory up\n      and that leading directory will be removed.\n    \"\"\"\n    if not exists(dir_):\n        lgr.debug(\"Creating directory %s to extract archive into\" % dir_)\n        os.makedirs(dir_)\n\n    _decompress_file(archive, dir_)\n\n    if leading_directories == 'strip':\n        _, dirs, files = next(os.walk(dir_))\n        if not len(files) and len(dirs) == 1:\n            # move all the content under dirs[0] up 1 level\n            widow_dir = opj(dir_, dirs[0])\n            lgr.debug(\"Moving content within %s upstairs\" % widow_dir)\n            subdir, subdirs_, files_ = next(os.walk(opj(dir_, dirs[0])))\n            for f in subdirs_ + files_:\n                os.rename(opj(subdir, f), opj(dir_, f))\n            rmdir(widow_dir)\n    elif leading_directories is None:\n        pass   # really do nothing\n    else:\n        raise NotImplementedError(\"Not supported %s\" % leading_directories)\n\n\ndef _get_cached_filename(archive):\n    \"\"\"A helper to generate a filename which has original filename and additional suffix\n    which wouldn't collide across files with the same name from different locations\n    \"\"\"\n    #return \"%s_%s\" % (basename(archive), hashlib.md5(archive).hexdigest()[:5])\n    # per se there is no reason to maintain any long original name here.\n    archive_cached = hashlib.md5(ensure_bytes(str(Path(archive).resolve()))).hexdigest()[:10]\n    lgr.debug(\"Cached directory for archive %s is %s\", archive, archive_cached)\n    return archive_cached\n\n\ndef _get_random_id(size=6, chars=string.ascii_uppercase + string.digits):\n    \"\"\"Return a random ID composed from digits and uppercase letters\n\n    upper-case so we are tolerant to unlikely collisions on dummy FSs\n    \"\"\"\n    return ''.join(random.choice(chars) for _ in range(size))\n\n\nclass ArchivesCache(object):\n    \"\"\"Cache to maintain extracted archives\n\n    Parameters\n    ----------\n    toppath : str\n      Top directory under .git/ of which temp directory would be created.\n      If not provided -- random tempdir is used\n    persistent : bool, optional\n      Passed over into generated ExtractedArchives\n    \"\"\"\n    # TODO: make caching persistent across sessions/runs, with cleanup\n    # IDEA: extract under .git/annex/tmp so later on annex unused could clean it\n    #       all up\n    def __init__(self, toppath=None, persistent=False):\n\n        self._toppath = toppath\n        if toppath:\n            path = opj(toppath, ARCHIVES_TEMP_DIR)\n            if not persistent:\n                tempsuffix = \"-\" + _get_random_id()\n                lgr.debug(\"For non-persistent archives using %s suffix for path %s\",\n                          tempsuffix, path)\n                path += tempsuffix\n        else:\n            if persistent:\n                raise ValueError(\"%s cannot be persistent since no toppath was provided\" % self)\n            path = tempfile.mktemp(**get_tempfile_kwargs())\n        self._path = path\n        self.persistent = persistent\n        # TODO?  ensure that it is absent or we should allow for it to persist a bit?\n        #if exists(path):\n        #    self._clean_cache()\n        self._archives = {}\n\n        # TODO: begging for a race condition\n        if not exists(path):\n            lgr.debug(\"Initiating clean cache for the archives under %s\" % self.path)\n            try:\n                self._made_path = True\n                os.makedirs(path)\n                lgr.debug(\"Cache initialized\")\n            except Exception as e:\n                lgr.error(\"Failed to initialize cached under %s\" % path)\n                raise\n        else:\n            lgr.debug(\"Not initiating existing cache for the archives under %s\" % self.path)\n            self._made_path = False\n\n    @property\n    def path(self):\n        return self._path\n\n    def clean(self, force=False):\n        for aname, a in list(self._archives.items()):\n            a.clean(force=force)\n            del self._archives[aname]\n        # Probably we should not rely on _made_path and not bother if persistent removing it\n        # if ((not self.persistent) or force) and self._made_path:\n        #     lgr.debug(\"Removing the entire archives cache under %s\" % self.path)\n        #     rmtemp(self.path)\n        if (not self.persistent) or force:\n            lgr.debug(\"Removing the entire archives cache under %s\" % self.path)\n            rmtemp(self.path)\n\n    def _get_normalized_archive_path(self, archive):\n        \"\"\"Return full path to archive\n\n        So we have consistent operation from different subdirs,\n        while referencing archives from the topdir\n\n        TODO: why do we need it???\n        \"\"\"\n        if not isabs(archive) and self._toppath:\n            out = normpath(opj(self._toppath, archive))\n            if relpath(out, self._toppath).startswith(pardir):\n                raise RuntimeError(\"%s points outside of the topdir %s\"\n                                   % (archive, self._toppath))\n            if isdir(out):\n                raise RuntimeError(\"got a directory here... bleh\")\n            return out\n        return archive\n\n    def get_archive(self, archive):\n        archive = self._get_normalized_archive_path(archive)\n\n        if archive not in self._archives:\n            self._archives[archive] = \\\n                ExtractedArchive(archive,\n                                 opj(self.path, _get_cached_filename(archive)),\n                                 persistent=self.persistent)\n\n        return self._archives[archive]\n\n    def __getitem__(self, archive):\n        return self.get_archive(archive)\n\n    def __delitem__(self, archive):\n        archive = self._get_normalized_archive_path(archive)\n        self._archives[archive].clean()\n        del self._archives[archive]\n\n    def __del__(self):\n        try:\n            # we can at least try\n            if not self.persistent:\n                self.clean()\n        except:  # MIH: IOError?\n            pass\n\n\nclass ExtractedArchive(object):\n    \"\"\"Container for the extracted archive\n    \"\"\"\n\n    # suffix to use for a stamp so we could guarantee that extracted archive is\n    STAMP_SUFFIX = '.stamp'\n\n    def __init__(self, archive, path=None, persistent=False):\n        self._archive = archive\n        # TODO: bad location for extracted archive -- use tempfile\n        if not path:\n            path = tempfile.mktemp(**get_tempfile_kwargs(prefix=_get_cached_filename(archive)))\n\n        if exists(path) and not persistent:\n            raise RuntimeError(\"Directory %s already exists whenever it should not \"\n                               \"persist\" % path)\n        self._persistent = persistent\n        self._path = path\n\n    def __repr__(self):\n        return \"%s(%r, path=%r)\" % (self.__class__.__name__, self._archive, self.path)\n\n    def clean(self, force=False):\n        # would interfere with tests\n        # if os.environ.get('DATALAD_TESTS_TEMP_KEEP'):\n        #     lgr.info(\"As instructed, not cleaning up the cache under %s\"\n        #              % self._path)\n        #     return\n\n        for path, name in [\n            (self._path, 'cache'),\n            (self.stamp_path, 'stamp file')\n        ]:\n            if exists(path):\n                if (not self._persistent) or force:\n                    lgr.debug(\"Cleaning up the %s for %s under %s\", name, self._archive, path)\n                    # TODO:  we must be careful here -- to not modify permissions of files\n                    #        only of directories\n                    (rmtree if isdir(path) else unlink)(path)\n\n    @property\n    def path(self):\n        \"\"\"Given an archive -- return full path to it within cache (extracted)\n        \"\"\"\n        return self._path\n\n    @property\n    def stamp_path(self):\n        return self._path + self.STAMP_SUFFIX\n\n    @property\n    def is_extracted(self):\n        return exists(self.path) and exists(self.stamp_path) \\\n            and os.stat(self.stamp_path).st_mtime >= os.stat(self.path).st_mtime\n\n    def assure_extracted(self):\n        \"\"\"Return path to the extracted `archive`.  Extract archive if necessary\n        \"\"\"\n        path = self.path\n\n        with lock_if_check_fails(\n            check=(lambda s: s.is_extracted, (self,)),\n            lock_path=path,\n            operation=\"extract\"\n        ) as (check, lock):\n            if lock:\n                assert not check\n                self._extract_archive(path)\n        return path\n\n    def _extract_archive(self, path):\n        # we need to extract the archive\n        # TODO: extract to _tmp and then move in a single command so we\n        # don't end up picking up broken pieces\n        lgr.debug(u\"Extracting {self._archive} under {path}\".format(**locals()))\n        if exists(path):\n            lgr.debug(\n                \"Previous extracted (but probably not fully) cached archive \"\n                \"found. Removing %s\",\n                path)\n            rmtree(path)\n        os.makedirs(path)\n        assert (exists(path))\n        # remove old stamp\n        if exists(self.stamp_path):\n            rmtree(self.stamp_path)\n        decompress_file(self._archive, path, leading_directories=None)\n        # TODO: must optional since we might to use this content, move it\n        # into the tree etc\n        # lgr.debug(\"Adjusting permissions to R/O for the extracted content\")\n        # rotree(path)\n        assert (exists(path))\n        # create a stamp\n        with open(self.stamp_path, 'wb') as f:\n            f.write(ensure_bytes(self._archive))\n        # assert that stamp mtime is not older than archive's directory\n        assert (self.is_extracted)\n\n    # TODO: remove?\n    #def has_file_ready(self, afile):\n    #    lgr.debug(u\"Checking file {afile} from archive {archive}\".format(**locals()))\n    #    return exists(self.get_extracted_filename(afile))\n\n    def get_extracted_filename(self, afile):\n        \"\"\"Return full path to the `afile` within extracted `archive`\n\n        It does not actually extract any archive\n        \"\"\"\n        return opj(self.path, urlunquote(afile))\n\n    def get_extracted_files(self):\n        \"\"\"Generator to provide filenames which are available under extracted archive\n        \"\"\"\n        path = self.assure_extracted()\n        path_len = len(path) + (len(os.sep) if not path.endswith(os.sep) else 0)\n        for root, dirs, files in os.walk(path):  # TEMP\n            for name in files:\n                yield ensure_unicode(opj(root, name)[path_len:])\n\n    def get_leading_directory(self, depth=None, consider=None, exclude=None):\n        \"\"\"Return leading directory of the content within archive\n\n        Parameters\n        ----------\n        depth: int or None, optional\n          Maximal depth of leading directories to consider.  If None - no upper\n          limit\n        consider : list of str, optional\n          Regular expressions for file/directory names to be considered (before\n          exclude). Applied to the entire relative path to the file as in the archive\n        exclude: list of str, optional\n          Regular expressions for file/directory names to be excluded from consideration.\n          Applied to the entire relative path to the file as in the archive\n\n        Returns\n        -------\n        str or None:\n          If there is no single leading directory -- None returned\n        \"\"\"\n        leading = None\n        # returns only files, so no need to check if a dir or not\n        for fpath in self.get_extracted_files():\n            if consider and not any_re_search(consider, fpath):\n                continue\n            if exclude and any_re_search(exclude, fpath):\n                continue\n\n            lpath = fpath.split(opsep)\n            dpath = lpath[:-1]  # directory path components\n            if leading is None:\n                leading = dpath if depth is None else dpath[:depth]\n            else:\n                if dpath[:len(leading)] != leading:\n                    # find smallest common path\n                    leading_ = []\n                    # TODO: there might be more efficient pythonic way\n                    for d1, d2 in zip(leading, dpath):\n                        if d1 != d2:\n                            break\n                        leading_.append(d1)\n                    leading = leading_\n            if not len(leading):\n                # no common leading - ready to exit\n                return None\n        return leading if leading is None else opj(*leading)\n\n    def get_extracted_file(self, afile):\n        lgr.debug(u\"Requested file {afile} from archive {self._archive}\".format(**locals()))\n        # TODO: That could be a good place to provide \"compatibility\" layer if\n        # filenames within archive are too obscure for local file system.\n        # We could somehow adjust them while extracting and here channel back\n        # \"fixed\" up names since they are only to point to the load\n        self.assure_extracted()\n        path = self.get_extracted_filename(afile)\n        # TODO: make robust\n        lgr.log(2, \"Verifying that %s exists\" % abspath(path))\n        assert exists(path), \"%s must exist\" % path\n        return path\n\n    def __del__(self):\n        try:\n            if self._persistent:\n                self.clean()\n        except Exception as e:  # MIH: IOError?\n            pass\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Various handlers/functionality for different types of files (e.g. for archives)\n\n\"\"\"\n\nimport hashlib\nimport os\nimport tempfile\nfrom urllib.parse import unquote as urlunquote\nimport string\nimport random\nimport logging\n\nfrom datalad.support.path import (\n    join as opj,\n    exists,\n    abspath,\n    isabs,\n    normpath,\n    relpath,\n    pardir,\n    isdir,\n    sep as opsep,\n)\n\nfrom datalad.support.locking import lock_if_check_fails\nfrom datalad.support.external_versions import external_versions\nfrom datalad.consts import ARCHIVES_TEMP_DIR\nfrom datalad.utils import (\n    any_re_search,\n    ensure_bytes,\n    ensure_unicode,\n    unlink,\n    rmdir,\n    rmtemp,\n    rmtree,\n    get_tempfile_kwargs,\n    on_windows,\n    Path,\n)\nfrom datalad import cfg\nfrom datalad.config import anything2bool\n\n# fall back on patool, if a functional implementation is available\n# (i.e. not on windows), it is requested, or 7z is not found\nif not on_windows and (\n        cfg.obtain(\n            'datalad.runtime.use-patool', default=False,\n            valtype=anything2bool) or not external_versions['cmd:7z']):\n    from datalad.support.archive_utils_patool import (\n        decompress_file as _decompress_file,\n        # other code expects this to be here\n        compress_files\n    )\nelse:\n    from datalad.support.archive_utils_7z import (\n        decompress_file as _decompress_file,\n        # other code expects this to be here\n        compress_files\n    )\n\nlgr = logging.getLogger('datalad.support.archives')\n\n\ndef decompress_file(archive, dir_, leading_directories='strip'):\n    \"\"\"Decompress `archive` into a directory `dir_`\n\n    Parameters\n    ----------\n    archive: str\n    dir_: str\n    leading_directories: {'strip', None}\n      If `strip`, and archive contains a single leading directory under which\n      all content is stored, all the content will be moved one directory up\n      and that leading directory will be removed.\n    \"\"\"\n    if not exists(dir_):\n        lgr.debug(\"Creating directory %s to extract archive into\" % dir_)\n        os.makedirs(dir_)\n\n    _decompress_file(archive, dir_)\n\n    if leading_directories == 'strip':\n        _, dirs, files = next(os.walk(dir_))\n        if not len(files) and len(dirs) == 1:\n            # move all the content under dirs[0] up 1 level\n            widow_dir = opj(dir_, dirs[0])\n            lgr.debug(\"Moving content within %s upstairs\" % widow_dir)\n            subdir, subdirs_, files_ = next(os.walk(opj(dir_, dirs[0])))\n            for f in subdirs_ + files_:\n                os.rename(opj(subdir, f), opj(dir_, f))\n            rmdir(widow_dir)\n    elif leading_directories is None:\n        pass   # really do nothing\n    else:\n        raise NotImplementedError(\"Not supported %s\" % leading_directories)\n\n\ndef _get_cached_filename(archive):\n    \"\"\"A helper to generate a filename which has original filename and additional suffix\n    which wouldn't collide across files with the same name from different locations\n    \"\"\"\n    #return \"%s_%s\" % (basename(archive), hashlib.md5(archive).hexdigest()[:5])\n    # per se there is no reason to maintain any long original name here.\n    archive_cached = hashlib.md5(ensure_bytes(str(Path(archive).resolve()))).hexdigest()[:10]\n    lgr.debug(\"Cached directory for archive %s is %s\", archive, archive_cached)\n    return archive_cached\n\n\ndef _get_random_id(size=6, chars=string.ascii_uppercase + string.digits):\n    \"\"\"Return a random ID composed from digits and uppercase letters\n\n    upper-case so we are tolerant to unlikely collisions on dummy FSs\n    \"\"\"\n    return ''.join(random.choice(chars) for _ in range(size))\n\n\nclass ArchivesCache(object):\n    \"\"\"Cache to maintain extracted archives\n\n    Parameters\n    ----------\n    toppath : str\n      Top directory under .git/ of which temp directory would be created.\n      If not provided -- random tempdir is used\n    persistent : bool, optional\n      Passed over into generated ExtractedArchives\n    \"\"\"\n    # TODO: make caching persistent across sessions/runs, with cleanup\n    # IDEA: extract under .git/annex/tmp so later on annex unused could clean it\n    #       all up\n    def __init__(self, toppath=None, persistent=False):\n\n        self._toppath = toppath\n        if toppath:\n            path = opj(toppath, ARCHIVES_TEMP_DIR)\n            if not persistent:\n                tempsuffix = \"-\" + _get_random_id()\n                lgr.debug(\"For non-persistent archives using %s suffix for path %s\",\n                          tempsuffix, path)\n                path += tempsuffix\n        else:\n            if persistent:\n                raise ValueError(\"%s cannot be persistent since no toppath was provided\" % self)\n            path = tempfile.mktemp(**get_tempfile_kwargs())\n        self._path = path\n        self.persistent = persistent\n        # TODO?  ensure that it is absent or we should allow for it to persist a bit?\n        #if exists(path):\n        #    self._clean_cache()\n        self._archives = {}\n\n        # TODO: begging for a race condition\n        if not exists(path):\n            lgr.debug(\"Initiating clean cache for the archives under %s\" % self.path)\n            try:\n                self._made_path = True\n                os.makedirs(path)\n                lgr.debug(\"Cache initialized\")\n            except Exception as e:\n                lgr.error(\"Failed to initialize cached under %s\" % path)\n                raise\n        else:\n            lgr.debug(\"Not initiating existing cache for the archives under %s\" % self.path)\n            self._made_path = False\n\n    @property\n    def path(self):\n        return self._path\n\n    def clean(self, force=False):\n        for aname, a in list(self._archives.items()):\n            a.clean(force=force)\n            del self._archives[aname]\n        # Probably we should not rely on _made_path and not bother if persistent removing it\n        # if ((not self.persistent) or force) and self._made_path:\n        #     lgr.debug(\"Removing the entire archives cache under %s\" % self.path)\n        #     rmtemp(self.path)\n        if (not self.persistent) or force:\n            lgr.debug(\"Removing the entire archives cache under %s\" % self.path)\n            rmtemp(self.path)\n\n    def _get_normalized_archive_path(self, archive):\n        \"\"\"Return full path to archive\n\n        So we have consistent operation from different subdirs,\n        while referencing archives from the topdir\n\n        TODO: why do we need it???\n        \"\"\"\n        if not isabs(archive) and self._toppath:\n            out = normpath(opj(self._toppath, archive))\n            if relpath(out, self._toppath).startswith(pardir):\n                raise RuntimeError(\"%s points outside of the topdir %s\"\n                                   % (archive, self._toppath))\n            if isdir(out):\n                raise RuntimeError(\"got a directory here... bleh\")\n            return out\n        return archive\n\n    def get_archive(self, archive):\n        archive = self._get_normalized_archive_path(archive)\n\n        if archive not in self._archives:\n            self._archives[archive] = \\\n                ExtractedArchive(archive,\n                                 opj(self.path, _get_cached_filename(archive)),\n                                 persistent=self.persistent)\n\n        return self._archives[archive]\n\n    def __getitem__(self, archive):\n        return self.get_archive(archive)\n\n    def __delitem__(self, archive):\n        archive = self._get_normalized_archive_path(archive)\n        self._archives[archive].clean()\n        del self._archives[archive]\n\n    def __del__(self):\n        try:\n            # we can at least try\n            if not self.persistent:\n                self.clean()\n        except:  # MIH: IOError?\n            pass\n\n\nclass ExtractedArchive(object):\n    \"\"\"Container for the extracted archive\n    \"\"\"\n\n    # suffix to use for a stamp so we could guarantee that extracted archive is\n    STAMP_SUFFIX = '.stamp'\n\n    def __init__(self, archive, path=None, persistent=False):\n        self._archive = archive\n        # TODO: bad location for extracted archive -- use tempfile\n        if not path:\n            path = tempfile.mktemp(**get_tempfile_kwargs(prefix=_get_cached_filename(archive)))\n\n        if exists(path) and not persistent:\n            raise RuntimeError(\"Directory %s already exists whenever it should not \"\n                               \"persist\" % path)\n        self._persistent = persistent\n        self._path = path\n\n    def __repr__(self):\n        return \"%s(%r, path=%r)\" % (self.__class__.__name__, self._archive, self.path)\n\n    def clean(self, force=False):\n        # would interfere with tests\n        # if os.environ.get('DATALAD_TESTS_TEMP_KEEP'):\n        #     lgr.info(\"As instructed, not cleaning up the cache under %s\"\n        #              % self._path)\n        #     return\n\n        for path, name in [\n            (self._path, 'cache'),\n            (self.stamp_path, 'stamp file')\n        ]:\n            if exists(path):\n                if (not self._persistent) or force:\n                    lgr.debug(\"Cleaning up the %s for %s under %s\", name, self._archive, path)\n                    # TODO:  we must be careful here -- to not modify permissions of files\n                    #        only of directories\n                    (rmtree if isdir(path) else unlink)(path)\n\n    @property\n    def path(self):\n        \"\"\"Given an archive -- return full path to it within cache (extracted)\n        \"\"\"\n        return self._path\n\n    @property\n    def stamp_path(self):\n        return self._path + self.STAMP_SUFFIX\n\n    @property\n    def is_extracted(self):\n        return exists(self.path) and exists(self.stamp_path) \\\n            and os.stat(self.stamp_path).st_mtime >= os.stat(self.path).st_mtime\n\n    def assure_extracted(self):\n        \"\"\"Return path to the extracted `archive`.  Extract archive if necessary\n        \"\"\"\n        path = self.path\n\n        with lock_if_check_fails(\n            check=(lambda s: s.is_extracted, (self,)),\n            lock_path=path,\n            operation=\"extract\"\n        ) as (check, lock):\n            if lock:\n                assert not check\n                self._extract_archive(path)\n        return path\n\n    def _extract_archive(self, path):\n        # we need to extract the archive\n        # TODO: extract to _tmp and then move in a single command so we\n        # don't end up picking up broken pieces\n        lgr.debug(u\"Extracting {self._archive} under {path}\".format(**locals()))\n        if exists(path):\n            lgr.debug(\n                \"Previous extracted (but probably not fully) cached archive \"\n                \"found. Removing %s\",\n                path)\n            rmtree(path)\n        os.makedirs(path)\n        assert (exists(path))\n        # remove old stamp\n        if exists(self.stamp_path):\n            rmtree(self.stamp_path)\n        decompress_file(self._archive, path, leading_directories=None)\n        # TODO: must optional since we might to use this content, move it\n        # into the tree etc\n        # lgr.debug(\"Adjusting permissions to R/O for the extracted content\")\n        # rotree(path)\n        assert (exists(path))\n        # create a stamp\n        with open(self.stamp_path, 'wb') as f:\n            f.write(ensure_bytes(self._archive))\n        # assert that stamp mtime is not older than archive's directory\n        assert (self.is_extracted)\n\n    # TODO: remove?\n    #def has_file_ready(self, afile):\n    #    lgr.debug(u\"Checking file {afile} from archive {archive}\".format(**locals()))\n    #    return exists(self.get_extracted_filename(afile))\n\n    def get_extracted_filename(self, afile):\n        \"\"\"Return full path to the `afile` within extracted `archive`\n\n        It does not actually extract any archive\n        \"\"\"\n        return opj(self.path, afile)\n\n    def get_extracted_files(self):\n        \"\"\"Generator to provide filenames which are available under extracted archive\n        \"\"\"\n        path = self.assure_extracted()\n        path_len = len(path) + (len(os.sep) if not path.endswith(os.sep) else 0)\n        for root, dirs, files in os.walk(path):  # TEMP\n            for name in files:\n                yield ensure_unicode(opj(root, name)[path_len:])\n\n    def get_leading_directory(self, depth=None, consider=None, exclude=None):\n        \"\"\"Return leading directory of the content within archive\n\n        Parameters\n        ----------\n        depth: int or None, optional\n          Maximal depth of leading directories to consider.  If None - no upper\n          limit\n        consider : list of str, optional\n          Regular expressions for file/directory names to be considered (before\n          exclude). Applied to the entire relative path to the file as in the archive\n        exclude: list of str, optional\n          Regular expressions for file/directory names to be excluded from consideration.\n          Applied to the entire relative path to the file as in the archive\n\n        Returns\n        -------\n        str or None:\n          If there is no single leading directory -- None returned\n        \"\"\"\n        leading = None\n        # returns only files, so no need to check if a dir or not\n        for fpath in self.get_extracted_files():\n            if consider and not any_re_search(consider, fpath):\n                continue\n            if exclude and any_re_search(exclude, fpath):\n                continue\n\n            lpath = fpath.split(opsep)\n            dpath = lpath[:-1]  # directory path components\n            if leading is None:\n                leading = dpath if depth is None else dpath[:depth]\n            else:\n                if dpath[:len(leading)] != leading:\n                    # find smallest common path\n                    leading_ = []\n                    # TODO: there might be more efficient pythonic way\n                    for d1, d2 in zip(leading, dpath):\n                        if d1 != d2:\n                            break\n                        leading_.append(d1)\n                    leading = leading_\n            if not len(leading):\n                # no common leading - ready to exit\n                return None\n        return leading if leading is None else opj(*leading)\n\n    def get_extracted_file(self, afile):\n        lgr.debug(u\"Requested file {afile} from archive {self._archive}\".format(**locals()))\n        # TODO: That could be a good place to provide \"compatibility\" layer if\n        # filenames within archive are too obscure for local file system.\n        # We could somehow adjust them while extracting and here channel back\n        # \"fixed\" up names since they are only to point to the load\n        self.assure_extracted()\n        path = self.get_extracted_filename(afile)\n        # TODO: make robust\n        lgr.log(2, \"Verifying that %s exists\" % abspath(path))\n        assert exists(path), \"%s must exist\" % path\n        return path\n\n    def __del__(self):\n        try:\n            if self._persistent:\n                self.clean()\n        except Exception as e:  # MIH: IOError?\n            pass\n",
            "header": {
                "file": {
                    "path": "datalad/support/archives.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "2b8db21fb",
                "new_commit_hash": "a5b076b24",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/archives.py"
            },
            "plus_file": {
                "path": "b/datalad/support/archives.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 341,
                            "length": 7
                        },
                        "new_range": {
                            "start": 341,
                            "length": 7
                        },
                        "section": "class ExtractedArchive(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        It does not actually extract any archive",
                                "type": "context"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "        return opj(self.path, urlunquote(afile))",
                                "type": "deleted"
                            },
                            {
                                "content": "        return opj(self.path, afile)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def get_extracted_files(self):",
                                "type": "context"
                            },
                            {
                                "content": "        \"\"\"Generator to provide filenames which are available under extracted archive",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/archives.py",
                            "type": "method",
                            "name": "ExtractedArchive.get_extracted_filename",
                            "content": "class ExtractedArchive(object):\n    \"\"\"Container for the extracted archive\n    \"\"\"\n\n    # suffix to use for a stamp so we could guarantee that extracted archive is\n    STAMP_SUFFIX = '.stamp'\n\n    def __init__(self, archive, path=None, persistent=False):\n        self._archive = archive\n        # TODO: bad location for extracted archive -- use tempfile\n        if not path:\n            path = tempfile.mktemp(**get_tempfile_kwargs(prefix=_get_cached_filename(archive)))\n\n        if exists(path) and not persistent:\n            raise RuntimeError(\"Directory %s already exists whenever it should not \"\n                               \"persist\" % path)\n        self._persistent = persistent\n        self._path = path\n\n    def __repr__(self):\n        return \"%s(%r, path=%r)\" % (self.__class__.__name__, self._archive, self.path)\n\n    def clean(self, force=False):\n        # would interfere with tests\n        # if os.environ.get('DATALAD_TESTS_TEMP_KEEP'):\n        #     lgr.info(\"As instructed, not cleaning up the cache under %s\"\n        #              % self._path)\n        #     return\n\n        for path, name in [\n            (self._path, 'cache'),\n            (self.stamp_path, 'stamp file')\n        ]:\n            if exists(path):\n                if (not self._persistent) or force:\n                    lgr.debug(\"Cleaning up the %s for %s under %s\", name, self._archive, path)\n                    # TODO:  we must be careful here -- to not modify permissions of files\n                    #        only of directories\n                    (rmtree if isdir(path) else unlink)(path)\n\n    @property\n    def path(self):\n        \"\"\"Given an archive -- return full path to it within cache (extracted)\n        \"\"\"\n        return self._path\n\n    @property\n    def stamp_path(self):\n        return self._path + self.STAMP_SUFFIX\n\n    @property\n    def is_extracted(self):\n        return exists(self.path) and exists(self.stamp_path) \\\n            and os.stat(self.stamp_path).st_mtime >= os.stat(self.path).st_mtime\n\n    def assure_extracted(self):\n        \"\"\"Return path to the extracted `archive`.  Extract archive if necessary\n        \"\"\"\n        path = self.path\n\n        with lock_if_check_fails(\n            check=(lambda s: s.is_extracted, (self,)),\n            lock_path=path,\n            operation=\"extract\"\n        ) as (check, lock):\n            if lock:\n                assert not check\n                self._extract_archive(path)\n        return path\n\n    def _extract_archive(self, path):\n        # we need to extract the archive\n        # TODO: extract to _tmp and then move in a single command so we\n        # don't end up picking up broken pieces\n        lgr.debug(u\"Extracting {self._archive} under {path}\".format(**locals()))\n        if exists(path):\n            lgr.debug(\n                \"Previous extracted (but probably not fully) cached archive \"\n                \"found. Removing %s\",\n                path)\n            rmtree(path)\n        os.makedirs(path)\n        assert (exists(path))\n        # remove old stamp\n        if exists(self.stamp_path):\n            rmtree(self.stamp_path)\n        decompress_file(self._archive, path, leading_directories=None)\n        # TODO: must optional since we might to use this content, move it\n        # into the tree etc\n        # lgr.debug(\"Adjusting permissions to R/O for the extracted content\")\n        # rotree(path)\n        assert (exists(path))\n        # create a stamp\n        with open(self.stamp_path, 'wb') as f:\n            f.write(ensure_bytes(self._archive))\n        # assert that stamp mtime is not older than archive's directory\n        assert (self.is_extracted)\n\n    # TODO: remove?\n    #def has_file_ready(self, afile):\n    #    lgr.debug(u\"Checking file {afile} from archive {archive}\".format(**locals()))\n    #    return exists(self.get_extracted_filename(afile))\n\n    def get_extracted_filename(self, afile):\n        \"\"\"Return full path to the `afile` within extracted `archive`\n\n        It does not actually extract any archive\n        \"\"\"\n        return opj(self.path, urlunquote(afile))\n\n    def get_extracted_files(self):\n        \"\"\"Generator to provide filenames which are available under extracted archive\n        \"\"\"\n        path = self.assure_extracted()\n        path_len = len(path) + (len(os.sep) if not path.endswith(os.sep) else 0)\n        for root, dirs, files in os.walk(path):  # TEMP\n            for name in files:\n                yield ensure_unicode(opj(root, name)[path_len:])\n\n    def get_leading_directory(self, depth=None, consider=None, exclude=None):\n        \"\"\"Return leading directory of the content within archive\n\n        Parameters\n        ----------\n        depth: int or None, optional\n          Maximal depth of leading directories to consider.  If None - no upper\n          limit\n        consider : list of str, optional\n          Regular expressions for file/directory names to be considered (before\n          exclude). Applied to the entire relative path to the file as in the archive\n        exclude: list of str, optional\n          Regular expressions for file/directory names to be excluded from consideration.\n          Applied to the entire relative path to the file as in the archive\n\n        Returns\n        -------\n        str or None:\n          If there is no single leading directory -- None returned\n        \"\"\"\n        leading = None\n        # returns only files, so no need to check if a dir or not\n        for fpath in self.get_extracted_files():\n            if consider and not any_re_search(consider, fpath):\n                continue\n            if exclude and any_re_search(exclude, fpath):\n                continue\n\n            lpath = fpath.split(opsep)\n            dpath = lpath[:-1]  # directory path components\n            if leading is None:\n                leading = dpath if depth is None else dpath[:depth]\n            else:\n                if dpath[:len(leading)] != leading:\n                    # find smallest common path\n                    leading_ = []\n                    # TODO: there might be more efficient pythonic way\n                    for d1, d2 in zip(leading, dpath):\n                        if d1 != d2:\n                            break\n                        leading_.append(d1)\n                    leading = leading_\n            if not len(leading):\n                # no common leading - ready to exit\n                return None\n        return leading if leading is None else opj(*leading)\n\n    def get_extracted_file(self, afile):\n        lgr.debug(u\"Requested file {afile} from archive {self._archive}\".format(**locals()))\n        # TODO: That could be a good place to provide \"compatibility\" layer if\n        # filenames within archive are too obscure for local file system.\n        # We could somehow adjust them while extracting and here channel back\n        # \"fixed\" up names since they are only to point to the load\n        self.assure_extracted()\n        path = self.get_extracted_filename(afile)\n        # TODO: make robust\n        lgr.log(2, \"Verifying that %s exists\" % abspath(path))\n        assert exists(path), \"%s must exist\" % path\n        return path\n\n    def __del__(self):\n        try:\n            if self._persistent:\n                self.clean()\n        except Exception as e:  # MIH: IOError?\n            pass",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 339,
                            "end_lineno": 344,
                            "parent": {
                                "file_name": "datalad/support/archives.py",
                                "type": "class",
                                "name": "ExtractedArchive",
                                "content": "class ExtractedArchive(object):\n    \"\"\"Container for the extracted archive\n    \"\"\"\n\n    # suffix to use for a stamp so we could guarantee that extracted archive is\n    STAMP_SUFFIX = '.stamp'\n\n    def __init__(self, archive, path=None, persistent=False):\n        self._archive = archive\n        # TODO: bad location for extracted archive -- use tempfile\n        if not path:\n            path = tempfile.mktemp(**get_tempfile_kwargs(prefix=_get_cached_filename(archive)))\n\n        if exists(path) and not persistent:\n            raise RuntimeError(\"Directory %s already exists whenever it should not \"\n                               \"persist\" % path)\n        self._persistent = persistent\n        self._path = path\n\n    def __repr__(self):\n        return \"%s(%r, path=%r)\" % (self.__class__.__name__, self._archive, self.path)\n\n    def clean(self, force=False):\n        # would interfere with tests\n        # if os.environ.get('DATALAD_TESTS_TEMP_KEEP'):\n        #     lgr.info(\"As instructed, not cleaning up the cache under %s\"\n        #              % self._path)\n        #     return\n\n        for path, name in [\n            (self._path, 'cache'),\n            (self.stamp_path, 'stamp file')\n        ]:\n            if exists(path):\n                if (not self._persistent) or force:\n                    lgr.debug(\"Cleaning up the %s for %s under %s\", name, self._archive, path)\n                    # TODO:  we must be careful here -- to not modify permissions of files\n                    #        only of directories\n                    (rmtree if isdir(path) else unlink)(path)\n\n    @property\n    def path(self):\n        \"\"\"Given an archive -- return full path to it within cache (extracted)\n        \"\"\"\n        return self._path\n\n    @property\n    def stamp_path(self):\n        return self._path + self.STAMP_SUFFIX\n\n    @property\n    def is_extracted(self):\n        return exists(self.path) and exists(self.stamp_path) \\\n            and os.stat(self.stamp_path).st_mtime >= os.stat(self.path).st_mtime\n\n    def assure_extracted(self):\n        \"\"\"Return path to the extracted `archive`.  Extract archive if necessary\n        \"\"\"\n        path = self.path\n\n        with lock_if_check_fails(\n            check=(lambda s: s.is_extracted, (self,)),\n            lock_path=path,\n            operation=\"extract\"\n        ) as (check, lock):\n            if lock:\n                assert not check\n                self._extract_archive(path)\n        return path\n\n    def _extract_archive(self, path):\n        # we need to extract the archive\n        # TODO: extract to _tmp and then move in a single command so we\n        # don't end up picking up broken pieces\n        lgr.debug(u\"Extracting {self._archive} under {path}\".format(**locals()))\n        if exists(path):\n            lgr.debug(\n                \"Previous extracted (but probably not fully) cached archive \"\n                \"found. Removing %s\",\n                path)\n            rmtree(path)\n        os.makedirs(path)\n        assert (exists(path))\n        # remove old stamp\n        if exists(self.stamp_path):\n            rmtree(self.stamp_path)\n        decompress_file(self._archive, path, leading_directories=None)\n        # TODO: must optional since we might to use this content, move it\n        # into the tree etc\n        # lgr.debug(\"Adjusting permissions to R/O for the extracted content\")\n        # rotree(path)\n        assert (exists(path))\n        # create a stamp\n        with open(self.stamp_path, 'wb') as f:\n            f.write(ensure_bytes(self._archive))\n        # assert that stamp mtime is not older than archive's directory\n        assert (self.is_extracted)\n\n    # TODO: remove?\n    #def has_file_ready(self, afile):\n    #    lgr.debug(u\"Checking file {afile} from archive {archive}\".format(**locals()))\n    #    return exists(self.get_extracted_filename(afile))\n\n    def get_extracted_filename(self, afile):\n        \"\"\"Return full path to the `afile` within extracted `archive`\n\n        It does not actually extract any archive\n        \"\"\"\n        return opj(self.path, urlunquote(afile))\n\n    def get_extracted_files(self):\n        \"\"\"Generator to provide filenames which are available under extracted archive\n        \"\"\"\n        path = self.assure_extracted()\n        path_len = len(path) + (len(os.sep) if not path.endswith(os.sep) else 0)\n        for root, dirs, files in os.walk(path):  # TEMP\n            for name in files:\n                yield ensure_unicode(opj(root, name)[path_len:])\n\n    def get_leading_directory(self, depth=None, consider=None, exclude=None):\n        \"\"\"Return leading directory of the content within archive\n\n        Parameters\n        ----------\n        depth: int or None, optional\n          Maximal depth of leading directories to consider.  If None - no upper\n          limit\n        consider : list of str, optional\n          Regular expressions for file/directory names to be considered (before\n          exclude). Applied to the entire relative path to the file as in the archive\n        exclude: list of str, optional\n          Regular expressions for file/directory names to be excluded from consideration.\n          Applied to the entire relative path to the file as in the archive\n\n        Returns\n        -------\n        str or None:\n          If there is no single leading directory -- None returned\n        \"\"\"\n        leading = None\n        # returns only files, so no need to check if a dir or not\n        for fpath in self.get_extracted_files():\n            if consider and not any_re_search(consider, fpath):\n                continue\n            if exclude and any_re_search(exclude, fpath):\n                continue\n\n            lpath = fpath.split(opsep)\n            dpath = lpath[:-1]  # directory path components\n            if leading is None:\n                leading = dpath if depth is None else dpath[:depth]\n            else:\n                if dpath[:len(leading)] != leading:\n                    # find smallest common path\n                    leading_ = []\n                    # TODO: there might be more efficient pythonic way\n                    for d1, d2 in zip(leading, dpath):\n                        if d1 != d2:\n                            break\n                        leading_.append(d1)\n                    leading = leading_\n            if not len(leading):\n                # no common leading - ready to exit\n                return None\n        return leading if leading is None else opj(*leading)\n\n    def get_extracted_file(self, afile):\n        lgr.debug(u\"Requested file {afile} from archive {self._archive}\".format(**locals()))\n        # TODO: That could be a good place to provide \"compatibility\" layer if\n        # filenames within archive are too obscure for local file system.\n        # We could somehow adjust them while extracting and here channel back\n        # \"fixed\" up names since they are only to point to the load\n        self.assure_extracted()\n        path = self.get_extracted_filename(afile)\n        # TODO: make robust\n        lgr.log(2, \"Verifying that %s exists\" % abspath(path))\n        assert exists(path), \"%s must exist\" % path\n        return path\n\n    def __del__(self):\n        try:\n            if self._persistent:\n                self.clean()\n        except Exception as e:  # MIH: IOError?\n            pass",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 236,
                                "end_lineno": 420,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/support/archives.py",
                            "type": "class",
                            "name": "ExtractedArchive",
                            "content": "class ExtractedArchive(object):\n    \"\"\"Container for the extracted archive\n    \"\"\"\n\n    # suffix to use for a stamp so we could guarantee that extracted archive is\n    STAMP_SUFFIX = '.stamp'\n\n    def __init__(self, archive, path=None, persistent=False):\n        self._archive = archive\n        # TODO: bad location for extracted archive -- use tempfile\n        if not path:\n            path = tempfile.mktemp(**get_tempfile_kwargs(prefix=_get_cached_filename(archive)))\n\n        if exists(path) and not persistent:\n            raise RuntimeError(\"Directory %s already exists whenever it should not \"\n                               \"persist\" % path)\n        self._persistent = persistent\n        self._path = path\n\n    def __repr__(self):\n        return \"%s(%r, path=%r)\" % (self.__class__.__name__, self._archive, self.path)\n\n    def clean(self, force=False):\n        # would interfere with tests\n        # if os.environ.get('DATALAD_TESTS_TEMP_KEEP'):\n        #     lgr.info(\"As instructed, not cleaning up the cache under %s\"\n        #              % self._path)\n        #     return\n\n        for path, name in [\n            (self._path, 'cache'),\n            (self.stamp_path, 'stamp file')\n        ]:\n            if exists(path):\n                if (not self._persistent) or force:\n                    lgr.debug(\"Cleaning up the %s for %s under %s\", name, self._archive, path)\n                    # TODO:  we must be careful here -- to not modify permissions of files\n                    #        only of directories\n                    (rmtree if isdir(path) else unlink)(path)\n\n    @property\n    def path(self):\n        \"\"\"Given an archive -- return full path to it within cache (extracted)\n        \"\"\"\n        return self._path\n\n    @property\n    def stamp_path(self):\n        return self._path + self.STAMP_SUFFIX\n\n    @property\n    def is_extracted(self):\n        return exists(self.path) and exists(self.stamp_path) \\\n            and os.stat(self.stamp_path).st_mtime >= os.stat(self.path).st_mtime\n\n    def assure_extracted(self):\n        \"\"\"Return path to the extracted `archive`.  Extract archive if necessary\n        \"\"\"\n        path = self.path\n\n        with lock_if_check_fails(\n            check=(lambda s: s.is_extracted, (self,)),\n            lock_path=path,\n            operation=\"extract\"\n        ) as (check, lock):\n            if lock:\n                assert not check\n                self._extract_archive(path)\n        return path\n\n    def _extract_archive(self, path):\n        # we need to extract the archive\n        # TODO: extract to _tmp and then move in a single command so we\n        # don't end up picking up broken pieces\n        lgr.debug(u\"Extracting {self._archive} under {path}\".format(**locals()))\n        if exists(path):\n            lgr.debug(\n                \"Previous extracted (but probably not fully) cached archive \"\n                \"found. Removing %s\",\n                path)\n            rmtree(path)\n        os.makedirs(path)\n        assert (exists(path))\n        # remove old stamp\n        if exists(self.stamp_path):\n            rmtree(self.stamp_path)\n        decompress_file(self._archive, path, leading_directories=None)\n        # TODO: must optional since we might to use this content, move it\n        # into the tree etc\n        # lgr.debug(\"Adjusting permissions to R/O for the extracted content\")\n        # rotree(path)\n        assert (exists(path))\n        # create a stamp\n        with open(self.stamp_path, 'wb') as f:\n            f.write(ensure_bytes(self._archive))\n        # assert that stamp mtime is not older than archive's directory\n        assert (self.is_extracted)\n\n    # TODO: remove?\n    #def has_file_ready(self, afile):\n    #    lgr.debug(u\"Checking file {afile} from archive {archive}\".format(**locals()))\n    #    return exists(self.get_extracted_filename(afile))\n\n    def get_extracted_filename(self, afile):\n        \"\"\"Return full path to the `afile` within extracted `archive`\n\n        It does not actually extract any archive\n        \"\"\"\n        return opj(self.path, urlunquote(afile))\n\n    def get_extracted_files(self):\n        \"\"\"Generator to provide filenames which are available under extracted archive\n        \"\"\"\n        path = self.assure_extracted()\n        path_len = len(path) + (len(os.sep) if not path.endswith(os.sep) else 0)\n        for root, dirs, files in os.walk(path):  # TEMP\n            for name in files:\n                yield ensure_unicode(opj(root, name)[path_len:])\n\n    def get_leading_directory(self, depth=None, consider=None, exclude=None):\n        \"\"\"Return leading directory of the content within archive\n\n        Parameters\n        ----------\n        depth: int or None, optional\n          Maximal depth of leading directories to consider.  If None - no upper\n          limit\n        consider : list of str, optional\n          Regular expressions for file/directory names to be considered (before\n          exclude). Applied to the entire relative path to the file as in the archive\n        exclude: list of str, optional\n          Regular expressions for file/directory names to be excluded from consideration.\n          Applied to the entire relative path to the file as in the archive\n\n        Returns\n        -------\n        str or None:\n          If there is no single leading directory -- None returned\n        \"\"\"\n        leading = None\n        # returns only files, so no need to check if a dir or not\n        for fpath in self.get_extracted_files():\n            if consider and not any_re_search(consider, fpath):\n                continue\n            if exclude and any_re_search(exclude, fpath):\n                continue\n\n            lpath = fpath.split(opsep)\n            dpath = lpath[:-1]  # directory path components\n            if leading is None:\n                leading = dpath if depth is None else dpath[:depth]\n            else:\n                if dpath[:len(leading)] != leading:\n                    # find smallest common path\n                    leading_ = []\n                    # TODO: there might be more efficient pythonic way\n                    for d1, d2 in zip(leading, dpath):\n                        if d1 != d2:\n                            break\n                        leading_.append(d1)\n                    leading = leading_\n            if not len(leading):\n                # no common leading - ready to exit\n                return None\n        return leading if leading is None else opj(*leading)\n\n    def get_extracted_file(self, afile):\n        lgr.debug(u\"Requested file {afile} from archive {self._archive}\".format(**locals()))\n        # TODO: That could be a good place to provide \"compatibility\" layer if\n        # filenames within archive are too obscure for local file system.\n        # We could somehow adjust them while extracting and here channel back\n        # \"fixed\" up names since they are only to point to the load\n        self.assure_extracted()\n        path = self.get_extracted_filename(afile)\n        # TODO: make robust\n        lgr.log(2, \"Verifying that %s exists\" % abspath(path))\n        assert exists(path), \"%s must exist\" % path\n        return path\n\n    def __del__(self):\n        try:\n            if self._persistent:\n                self.clean()\n        except Exception as e:  # MIH: IOError?\n            pass",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 236,
                            "end_lineno": 420,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil; coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Miscellaneous utilities to assist with testing\"\"\"\n\nimport glob\nimport gzip\nimport inspect\nimport shutil\nimport stat\nfrom json import dumps\nimport os\nimport re\nimport tempfile\nimport platform\nimport multiprocessing\nimport logging\nimport random\nimport socket\nimport textwrap\nimport warnings\nfrom fnmatch import fnmatch\nimport time\nfrom difflib import unified_diff\nfrom contextlib import contextmanager\nfrom unittest.mock import patch\n\nfrom http.server import (\n    HTTPServer,\n    SimpleHTTPRequestHandler,\n)\n\nfrom functools import wraps\nfrom os.path import (\n    curdir,\n    exists,\n    join as opj,\n    relpath,\n    split as pathsplit,\n)\n\nfrom nose.plugins.attrib import attr\nfrom nose.tools import (\n    assert_equal,\n    assert_false,\n    assert_greater,\n    assert_greater_equal,\n    assert_in as in_,\n    assert_in,\n    assert_is,\n    assert_is_none,\n    assert_is_not,\n    assert_is_not_none,\n    assert_not_equal,\n    assert_not_in,\n    assert_not_is_instance,\n    assert_raises,\n    assert_true,\n    eq_,\n    make_decorator,\n    ok_,\n    raises,\n)\n\nfrom nose.tools import assert_set_equal\nfrom nose.tools import assert_is_instance\nfrom nose import SkipTest\n\nfrom datalad import cfg as dl_cfg\nimport datalad.utils as ut\n# TODO this must go\nfrom ..utils import *\nfrom datalad.utils import (\n    Path,\n    ensure_unicode,\n)\n\nfrom ..cmd import Runner\nfrom .. import utils\nfrom ..support.exceptions import (\n    CommandError,\n    CommandNotAvailableError,\n)\nfrom ..support.external_versions import external_versions\nfrom ..support.vcr_ import *\nfrom ..support.keyring_ import MemoryKeyring\nfrom ..support.network import RI\nfrom ..dochelpers import exc_str, borrowkwargs\nfrom ..cmdline.helpers import get_repo_instance\nfrom ..consts import (\n    ARCHIVES_TEMP_DIR,\n)\nfrom . import _TEMP_PATHS_GENERATED\n\n# temp paths used by clones\n_TEMP_PATHS_CLONES = set()\n\n\n# Additional indicators\non_travis = bool(os.environ.get('TRAVIS', False))\n\nif external_versions[\"cmd:git\"] >= \"2.28\":\n    # The specific value here doesn't matter, but it should not be the default\n    # from any Git version to test that we work with custom values.\n    DEFAULT_BRANCH = \"dl-test-branch\"  # Set by setup_package().\nelse:\n    DEFAULT_BRANCH = \"master\"\n\n# additional shortcuts\nneq_ = assert_not_equal\nnok_ = assert_false\n\nlgr = logging.getLogger(\"datalad.tests.utils\")\n\n\ndef skip_if_no_module(module):\n    try:\n        imp = __import__(module)\n    except Exception as exc:\n        raise SkipTest(\"Module %s fails to load: %s\" % (module, exc_str(exc)))\n\n\ndef skip_if_scrapy_without_selector():\n    \"\"\"A little helper to skip some tests which require recent scrapy\"\"\"\n    try:\n        import scrapy\n        from scrapy.selector import Selector\n    except ImportError:\n        from nose import SkipTest\n        raise SkipTest(\n            \"scrapy misses Selector (too old? version: %s)\"\n            % getattr(scrapy, '__version__'))\n\n\ndef skip_if_url_is_not_available(url, regex=None):\n    # verify that dataset is available\n    from datalad.downloaders.providers import Providers\n    from datalad.downloaders.base import DownloadError\n    providers = Providers.from_config_files()\n    try:\n        content = providers.fetch(url)\n        if regex and re.search(regex, content):\n            raise SkipTest(\"%s matched %r -- skipping the test\" % (url, regex))\n    except DownloadError:\n        raise SkipTest(\"%s failed to download\" % url)\n\n\ndef check_not_generatorfunction(func):\n    \"\"\"Internal helper to verify that we are not decorating generator tests\"\"\"\n    if inspect.isgeneratorfunction(func):\n        raise RuntimeError(\"{}: must not be decorated, is a generator test\"\n                           .format(func.__name__))\n\n\ndef skip_if_no_network(func=None):\n    \"\"\"Skip test completely in NONETWORK settings\n\n    If not used as a decorator, and just a function, could be used at the module level\n    \"\"\"\n    check_not_generatorfunction(func)\n\n    def check_and_raise():\n        if dl_cfg.get('datalad.tests.nonetwork'):\n            raise SkipTest(\"Skipping since no network settings\")\n\n    if func:\n        @wraps(func)\n        @attr('network')\n        @attr('skip_if_no_network')\n        def  _wrap_skip_if_no_network(*args, **kwargs):\n            check_and_raise()\n            return func(*args, **kwargs)\n        return  _wrap_skip_if_no_network\n    else:\n        check_and_raise()\n\n\ndef skip_if_on_windows(func=None):\n    \"\"\"Skip test completely under Windows\n    \"\"\"\n    check_not_generatorfunction(func)\n\n    def check_and_raise():\n        if on_windows:\n            raise SkipTest(\"Skipping on Windows\")\n\n    if func:\n        @wraps(func)\n        @attr('skip_if_on_windows')\n        def  _wrap_skip_if_on_windows(*args, **kwargs):\n            check_and_raise()\n            return func(*args, **kwargs)\n        return  _wrap_skip_if_on_windows\n    else:\n        check_and_raise()\n\n\ndef skip_if_root(func=None):\n    \"\"\"Skip test if uid == 0.\n\n    Note that on Windows (or anywhere else `os.geteuid` is not available) the\n    test is _not_ skipped.\n    \"\"\"\n    check_not_generatorfunction(func)\n\n    def check_and_raise():\n        if hasattr(os, \"geteuid\") and os.geteuid() == 0:\n            raise SkipTest(\"Skipping: test assumptions fail under root\")\n\n    if func:\n        @wraps(func)\n        @attr('skip_if_root')\n        def  _wrap_skip_if_root(*args, **kwargs):\n            check_and_raise()\n            return func(*args, **kwargs)\n        return  _wrap_skip_if_root\n    else:\n        check_and_raise()\n\n\n@optional_args\ndef skip_if(func, cond=True, msg=None, method='raise'):\n    \"\"\"Skip test for specific condition\n\n    Parameters\n    ----------\n    cond: bool\n      condition on which to skip\n    msg: str\n      message to print if skipping\n    method: str\n      either 'raise' or 'pass'. Whether to skip by raising `SkipTest` or by\n      just proceeding and simply not calling the decorated function.\n      This is particularly meant to be used, when decorating single assertions\n      in a test with method='pass' in order to not skip the entire test, but\n      just that assertion.\n    \"\"\"\n\n    check_not_generatorfunction(func)\n\n    @wraps(func)\n    def  _wrap_skip_if(*args, **kwargs):\n        if cond:\n            if method == 'raise':\n                raise SkipTest(msg if msg else \"condition was True\")\n            elif method == 'pass':\n                print(msg if msg else \"condition was True\")\n                return\n        return func(*args, **kwargs)\n    return  _wrap_skip_if\n\n\ndef skip_ssh(func):\n    \"\"\"Skips SSH tests if on windows or if environment variable\n    DATALAD_TESTS_SSH was not set\n    \"\"\"\n\n    check_not_generatorfunction(func)\n\n    @wraps(func)\n    @attr('skip_ssh')\n    def  _wrap_skip_ssh(*args, **kwargs):\n        test_ssh = dl_cfg.get(\"datalad.tests.ssh\", '')\n        if not test_ssh or test_ssh in ('0', 'false', 'no'):\n            raise SkipTest(\"Run this test by setting DATALAD_TESTS_SSH\")\n        return func(*args, **kwargs)\n    return  _wrap_skip_ssh\n\n\n@optional_args\ndef skip_v6_or_later(func, method='raise'):\n    \"\"\"Skip tests if v6 or later will be used as the default repo version.\n\n    The default repository version is controlled by the configured value of\n    DATALAD_REPO_VERSION and whether v5 repositories are supported by the\n    installed git-annex.\n    \"\"\"\n\n    from datalad.support.annexrepo import AnnexRepo\n\n    version = dl_cfg.obtain(\"datalad.repo.version\")\n    info = AnnexRepo.check_repository_versions()\n\n    @skip_if(version >= 6 or 5 not in info[\"supported\"],\n             msg=\"Skip test in v6+ test run\", method=method)\n    @wraps(func)\n    @attr('skip_v6_or_later')\n    @attr('v6_or_later')\n    def  _wrap_skip_v6_or_later(*args, **kwargs):\n        return func(*args, **kwargs)\n    return  _wrap_skip_v6_or_later\n\n\n#\n# Addition \"checkers\"\n#\n\nimport os\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.annexrepo import AnnexRepo, FileNotInAnnexError\nfrom datalad.distribution.dataset import Dataset\nfrom ..utils import chpwd, getpwd\n\n\ndef ok_clean_git(path, annex=None, index_modified=[], untracked=[]):\n    \"\"\"Obsolete test helper. Use assert_repo_status() instead.\n\n    Still maps a few common cases to the new helper, to ease transition\n    in extensions.\n    \"\"\"\n    kwargs = {}\n    if index_modified:\n        kwargs['modified'] = index_modified\n    if untracked:\n        kwargs['untracked'] = untracked\n    assert_repo_status(\n        path,\n        annex=annex,\n        **kwargs,\n    )\n\n\ndef ok_file_under_git(path, filename=None, annexed=False):\n    \"\"\"Test if file is present and under git/annex control\n\n    If relative path provided, then test from current directory\n    \"\"\"\n    annex, file_repo_path, filename, path, repo = _prep_file_under_git(path, filename)\n    assert_in(file_repo_path, repo.get_indexed_files())  # file is known to Git\n\n    if annex:\n        try:\n            # operates on relative to curdir path\n            repo.get_file_key(file_repo_path)\n            in_annex = True\n        except FileNotInAnnexError as e:\n            in_annex = False\n    else:\n        in_annex = False\n\n    assert(annexed == in_annex)\n\n\ndef put_file_under_git(path, filename=None, content=None, annexed=False):\n    \"\"\"Place file under git/annex and return used Repo\n    \"\"\"\n    annex, file_repo_path, filename, path, repo = _prep_file_under_git(path, filename)\n    if content is None:\n        content = \"\"\n    with open(opj(repo.path, file_repo_path), 'w') as f_:\n        f_.write(content)\n\n    if annexed:\n        if not isinstance(repo, AnnexRepo):\n            repo = AnnexRepo(repo.path)\n        repo.add(file_repo_path)\n    else:\n        repo.add(file_repo_path, git=True)\n    repo.commit(_datalad_msg=True)\n    ok_file_under_git(repo.path, file_repo_path, annexed)\n    return repo\n\n\ndef _prep_file_under_git(path, filename):\n    \"\"\"Get instance of the repository for the given filename\n\n    Helper to be used by few functions\n    \"\"\"\n    if filename is None:\n        # path provides the path and the name\n        path, filename = pathsplit(path)\n    try:\n        # if succeeds when must not (not `annexed`) -- fail\n        repo = get_repo_instance(path, class_=AnnexRepo)\n        annex = True\n    except RuntimeError as e:\n        # TODO: make a dedicated Exception\n        if \"No annex repository found in\" in str(e):\n            repo = get_repo_instance(path, class_=GitRepo)\n            annex = False\n        else:\n            raise\n\n    # path to the file within the repository\n    # repo.path is a \"realpath\" so to get relpath working correctly\n    # we need to realpath our path as well\n    # do absolute() in addition to always get an absolute path\n    # even with non-existing paths on windows\n    path = str(Path(path).resolve().absolute())  # intentional realpath to match GitRepo behavior\n    file_repo_dir = op.relpath(path, repo.path)\n    file_repo_path = filename if file_repo_dir == curdir else opj(file_repo_dir, filename)\n    return annex, file_repo_path, filename, path, repo\n\n\n#\n# Helpers to test symlinks\n#\n\ndef ok_symlink(path):\n    \"\"\"Checks whether path is either a working or broken symlink\"\"\"\n    link_path = os.path.islink(path)\n    if not link_path:\n        raise AssertionError(\"Path {} seems not to be a symlink\".format(path))\n\n\ndef ok_good_symlink(path):\n    ok_symlink(path)\n    rpath = Path(path).resolve()\n    ok_(rpath.exists(),\n        msg=\"Path {} seems to be missing.  Symlink {} is broken\".format(\n                rpath, path))\n\n\ndef ok_broken_symlink(path):\n    ok_symlink(path)\n    rpath = Path(path).resolve()\n    assert_false(rpath.exists(),\n            msg=\"Path {} seems to be present.  Symlink {} is not broken\".format(\n                    rpath, path))\n\n\ndef ok_startswith(s, prefix):\n    ok_(s.startswith(prefix),\n        msg=\"String %r doesn't start with %r\" % (s, prefix))\n\n\ndef ok_endswith(s, suffix):\n    ok_(s.endswith(suffix),\n        msg=\"String %r doesn't end with %r\" % (s, suffix))\n\n\ndef nok_startswith(s, prefix):\n    assert_false(s.startswith(prefix),\n        msg=\"String %r starts with %r\" % (s, prefix))\n\n\ndef ok_git_config_not_empty(ar):\n    \"\"\"Helper to verify that nothing rewritten the config file\"\"\"\n    # TODO: we don't support bare -- do we?\n    assert_true(os.stat(opj(ar.path, '.git', 'config')).st_size)\n\n\ndef ok_annex_get(ar, files, network=True):\n    \"\"\"Helper to run .get decorated checking for correct operation\n\n    get passes through stderr from the ar to the user, which pollutes\n    screen while running tests\n\n    Note: Currently not true anymore, since usage of --json disables\n    progressbars\n    \"\"\"\n    ok_git_config_not_empty(ar) # we should be working in already inited repo etc\n    with swallow_outputs() as cmo:\n        ar.get(files)\n    # verify that load was fetched\n    ok_git_config_not_empty(ar) # whatever we do shouldn't destroy the config file\n    has_content = ar.file_has_content(files)\n    if isinstance(has_content, bool):\n        ok_(has_content)\n    else:\n        ok_(all(has_content))\n\n\ndef ok_generator(gen):\n    assert_true(inspect.isgenerator(gen), msg=\"%s is not a generator\" % gen)\n\n\nassert_is_generator = ok_generator  # just an alias\n\n\ndef ok_archives_caches(repopath, n=1, persistent=None):\n    \"\"\"Given a path to repository verify number of archives\n\n    Parameters\n    ----------\n    repopath : str\n      Path to the repository\n    n : int, optional\n      Number of archives directories to expect\n    persistent: bool or None, optional\n      If None -- both persistent and not count.\n    \"\"\"\n    # looking into subdirectories\n    glob_ptn = opj(repopath,\n                   ARCHIVES_TEMP_DIR + {None: '*', True: '', False: '-*'}[persistent],\n                   '*')\n    dirs = glob.glob(glob_ptn)\n    n2 = n * 2  # per each directory we should have a .stamp file\n    assert_equal(len(dirs), n2,\n                 msg=\"Found following dirs when needed %d of them: %s\" % (n2, dirs))\n\n\ndef ok_exists(path):\n    assert Path(path).exists(), 'path %s does not exist (or dangling symlink)' % path\n\n\ndef ok_file_has_content(path, content, strip=False, re_=False,\n                        decompress=False, **kwargs):\n    \"\"\"Verify that file exists and has expected content\"\"\"\n    path = Path(path)\n    ok_exists(path)\n    if decompress:\n        if path.suffix == '.gz':\n            open_func = gzip.open\n        else:\n            raise NotImplementedError(\"Don't know how to decompress %s\" % path)\n    else:\n        open_func = open\n\n    with open_func(str(path), 'rb') as f:\n        file_content = f.read()\n\n    if isinstance(content, str):\n        file_content = ensure_unicode(file_content)\n\n    if os.linesep != '\\n':\n        # for consistent comparisons etc. Apparently when reading in `b` mode\n        # on Windows we would also get \\r\n        # https://github.com/datalad/datalad/pull/3049#issuecomment-444128715\n        file_content = file_content.replace(os.linesep, '\\n')\n\n    if strip:\n        file_content = file_content.strip()\n\n    if re_:\n        assert_re_in(content, file_content, **kwargs)\n    else:\n        assert_equal(content, file_content, **kwargs)\n\n\n#\n# Decorators\n#\n\n\n@optional_args\ndef with_tree(t, tree=None, archives_leading_dir=True, delete=True, **tkwargs):\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n            return t(*(arg + (d,)), **kw)\n        finally:\n            if delete:\n                rmtemp(d)\n    return  _wrap_with_tree\n\n\nlgr = logging.getLogger('datalad.tests')\n\n\nclass SilentHTTPHandler(SimpleHTTPRequestHandler):\n    \"\"\"A little adapter to silence the handler\n    \"\"\"\n    def __init__(self, *args, **kwargs):\n        self._silent = lgr.getEffectiveLevel() > logging.DEBUG\n        SimpleHTTPRequestHandler.__init__(self, *args, **kwargs)\n\n    def log_message(self, format, *args):\n        if self._silent:\n            return\n        lgr.debug(\"HTTP: \" + format % args)\n\n\ndef _multiproc_serve_path_via_http(hostname, path_to_serve_from, queue): # pragma: no cover\n    chpwd(path_to_serve_from)\n    httpd = HTTPServer((hostname, 0), SilentHTTPHandler)\n    queue.put(httpd.server_port)\n    httpd.serve_forever()\n\n\nclass HTTPPath(object):\n    \"\"\"Serve the content of a path via an HTTP URL.\n\n    This class can be used as a context manager, in which case it returns the\n    URL.\n\n    Alternatively, the `start` and `stop` methods can be called directly.\n\n    Parameters\n    ----------\n    path : str\n        Directory with content to serve.\n    \"\"\"\n    def __init__(self, path):\n        self.path = path\n        self.url = None\n        self._env_patch = None\n        self._mproc = None\n\n    def __enter__(self):\n        self.start()\n        return self.url\n\n    def __exit__(self, *args):\n        self.stop()\n\n    def start(self):\n        \"\"\"Start serving `path` via HTTP.\n        \"\"\"\n        # There is a problem with Haskell on wheezy trying to\n        # fetch via IPv6 whenever there is a ::1 localhost entry in\n        # /etc/hosts.  Apparently fixing that docker image reliably\n        # is not that straightforward, although see\n        # http://jasonincode.com/customizing-hosts-file-in-docker/\n        # so we just force to use 127.0.0.1 while on wheezy\n        #hostname = '127.0.0.1' if on_debian_wheezy else 'localhost'\n        hostname = '127.0.0.1'\n\n        queue = multiprocessing.Queue()\n        self._mproc = multiprocessing.Process(\n            target=_multiproc_serve_path_via_http,\n            args=(hostname, self.path, queue))\n        self._mproc.start()\n        port = queue.get(timeout=300)\n        self.url = 'http://{}:{}/'.format(hostname, port)\n        lgr.debug(\"HTTP: serving %s under %s\", self.path, self.url)\n\n        # Such tests don't require real network so if http_proxy settings were\n        # provided, we remove them from the env for the duration of this run\n        env = os.environ.copy()\n        env.pop('http_proxy', None)\n        self._env_patch = patch.dict('os.environ', env, clear=True)\n        self._env_patch.start()\n\n    def stop(self):\n        \"\"\"Stop serving `path`.\n        \"\"\"\n        lgr.debug(\"HTTP: stopping server under %s\", self.path)\n        self._env_patch.stop()\n        self._mproc.terminate()\n\n\n@optional_args\ndef serve_path_via_http(tfunc, *targs):\n    \"\"\"Decorator which serves content of a directory via http url\n    \"\"\"\n\n    @wraps(tfunc)\n    @attr('serve_path_via_http')\n    def  _wrap_serve_path_via_http(*args, **kwargs):\n\n        if targs:\n            # if a path is passed into serve_path_via_http, then it's in targs\n            assert len(targs) == 1\n            path = targs[0]\n\n        elif len(args) > 1:\n            args, path = args[:-1], args[-1]\n        else:\n            args, path = (), args[0]\n\n        with HTTPPath(path) as url:\n            return tfunc(*(args + (path, url)), **kwargs)\n    return  _wrap_serve_path_via_http\n\n\n@optional_args\ndef with_memory_keyring(t):\n    \"\"\"Decorator to use non-persistant MemoryKeyring instance\n    \"\"\"\n    @wraps(t)\n    @attr('with_memory_keyring')\n    def  _wrap_with_memory_keyring(*args, **kwargs):\n        keyring = MemoryKeyring()\n        with patch(\"datalad.downloaders.credentials.keyring_\", keyring):\n            return t(*(args + (keyring,)), **kwargs)\n\n    return  _wrap_with_memory_keyring\n\n\n@optional_args\ndef without_http_proxy(tfunc):\n    \"\"\"Decorator to remove http*_proxy env variables for the duration of the test\n    \"\"\"\n\n    @wraps(tfunc)\n    @attr('without_http_proxy')\n    def  _wrap_without_http_proxy(*args, **kwargs):\n        # Such tests don't require real network so if http_proxy settings were\n        # provided, we remove them from the env for the duration of this run\n        env = os.environ.copy()\n        env.pop('http_proxy', None)\n        env.pop('https_proxy', None)\n        with patch.dict('os.environ', env, clear=True):\n            return tfunc(*args, **kwargs)\n\n    return  _wrap_without_http_proxy\n\n\n@borrowkwargs(methodname=make_tempfile)\n@optional_args\ndef with_tempfile(t, **tkwargs):\n    \"\"\"Decorator function to provide a temporary file name and remove it at the end\n\n    Parameters\n    ----------\n\n    To change the used directory without providing keyword argument 'dir' set\n    DATALAD_TESTS_TEMP_DIR.\n\n    Examples\n    --------\n\n    ::\n\n        @with_tempfile\n        def test_write(tfile):\n            open(tfile, 'w').write('silly test')\n    \"\"\"\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n            return t(*(arg + (filename,)), **kw)\n\n    return  _wrap_with_tempfile\n\n\n# ### ###\n# START known failure decorators\n# ### ###\n\ndef probe_known_failure(func):\n    \"\"\"Test decorator allowing the test to pass when it fails and vice versa\n\n    Setting config datalad.tests.knownfailures.probe to True tests, whether or\n    not the test is still failing. If it's not, an AssertionError is raised in\n    order to indicate that the reason for failure seems to be gone.\n    \"\"\"\n\n    @wraps(func)\n    @attr('probe_known_failure')\n    def  _wrap_probe_known_failure(*args, **kwargs):\n        if dl_cfg.obtain(\"datalad.tests.knownfailures.probe\"):\n            assert_raises(Exception, func, *args, **kwargs)  # marked as known failure\n            # Note: Since assert_raises lacks a `msg` argument, a comment\n            # in the same line is helpful to determine what's going on whenever\n            # this assertion fails and we see a trace back. Otherwise that line\n            # wouldn't be very telling.\n        else:\n            return func(*args, **kwargs)\n    return  _wrap_probe_known_failure\n\n\n@optional_args\ndef skip_known_failure(func, method='raise'):\n    \"\"\"Test decorator allowing to skip a test that is known to fail\n\n    Setting config datalad.tests.knownfailures.skip to a bool enables/disables\n    skipping.\n    \"\"\"\n\n    @skip_if(cond=dl_cfg.obtain(\"datalad.tests.knownfailures.skip\"),\n             msg=\"Skip test known to fail\",\n             method=method)\n    @wraps(func)\n    @attr('skip_known_failure')\n    def  _wrap_skip_known_failure(*args, **kwargs):\n        return func(*args, **kwargs)\n    return  _wrap_skip_known_failure\n\n\ndef known_failure(func):\n    \"\"\"Test decorator marking a test as known to fail\n\n    This combines `probe_known_failure` and `skip_known_failure` giving the\n    skipping precedence over the probing.\n    \"\"\"\n\n    @skip_known_failure\n    @probe_known_failure\n    @wraps(func)\n    @attr('known_failure')\n    def  _wrap_known_failure(*args, **kwargs):\n        return func(*args, **kwargs)\n    return  _wrap_known_failure\n\n\ndef known_failure_v6_or_later(func):\n    \"\"\"Test decorator marking a test as known to fail in a v6+ test run\n\n    If the default repository version is 6 or later behaves like `known_failure`.\n    Otherwise the original (undecorated) function is returned.\n    The default repository version is controlled by the configured value of\n    DATALAD_REPO_VERSION and whether v5 repositories are supported by the\n    installed git-annex.\n    \"\"\"\n\n    from datalad.support.annexrepo import AnnexRepo\n\n    version = dl_cfg.obtain(\"datalad.repo.version\")\n    info = AnnexRepo.check_repository_versions()\n\n    if (version and version >= 6) or 5 not in info[\"supported\"]:\n\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_v6_or_later')\n        @attr('v6_or_later')\n        def v6_func(*args, **kwargs):\n            return func(*args, **kwargs)\n\n        return v6_func\n\n    return func\n\n\n# TODO: Remove once the released version of datalad-crawler no longer uses it.\nknown_failure_v6 = known_failure_v6_or_later\n\n\ndef known_failure_direct_mode(func):\n    \"\"\"DEPRECATED.  Stop using.  Does nothing\n\n    Test decorator marking a test as known to fail in a direct mode test run\n\n    If datalad.repo.direct is set to True behaves like `known_failure`.\n    Otherwise the original (undecorated) function is returned.\n    \"\"\"\n    # TODO: consider adopting   nibabel/deprecated.py  nibabel/deprecator.py\n    # mechanism to consistently deprecate functionality and ensure they are\n    # displayed.\n    # Since 2.7 Deprecation warnings aren't displayed by default\n    # and thus kinda pointless to issue a warning here, so we will just log\n    msg = \"Direct mode support is deprecated, so no point in using \" \\\n          \"@known_failure_direct_mode for %r since glorious future \" \\\n          \"DataLad 0.12\" % func.__name__\n    lgr.warning(msg)\n    return func\n\n\ndef known_failure_windows(func):\n    \"\"\"Test decorator marking a test as known to fail on windows\n\n    On Windows behaves like `known_failure`.\n    Otherwise the original (undecorated) function is returned.\n    \"\"\"\n    if on_windows:\n\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_windows')\n        @attr('windows')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n\n        return dm_func\n    return func\n\n\ndef known_failure_appveyor(func):\n    \"\"\"Test decorator marking a test as known to fail on AppVeyor.\n    \"\"\"\n    if 'APPVEYOR' in os.environ:\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_appveyor')\n        @attr('appveyor')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n        return dm_func\n    return func\n\n\ndef known_failure_githubci_win(func):\n    \"\"\"Test decorator for a known test failure on Github's Windows CI\n    \"\"\"\n    if 'GITHUB_WORKFLOW' in os.environ and on_windows:\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_githubci_win')\n        @attr('githubci_win')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n        return dm_func\n    return func\n\n\ndef known_failure_githubci_osx(func):\n    \"\"\"Test decorator for a known test failure on Github's macOS CI\n    \"\"\"\n    if 'GITHUB_WORKFLOW' in os.environ and on_osx:\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_githubci_osx')\n        @attr('githubci_osx')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n        return dm_func\n    return func\n\n\n# ### ###\n# END known failure decorators\n# ### ###\n\n\ndef _get_resolved_flavors(flavors):\n    #flavors_ = (['local', 'clone'] + (['local-url'] if not on_windows else [])) \\\n    #           if flavors == 'auto' else flavors\n    flavors_ = (['local', 'clone', 'local-url', 'network'] if not on_windows\n                else ['network', 'network-clone']) \\\n               if flavors == 'auto' else flavors\n\n    if not isinstance(flavors_, list):\n        flavors_ = [flavors_]\n\n    if dl_cfg.get('datalad.tests.nonetwork'):\n        flavors_ = [x for x in flavors_ if not x.startswith('network')]\n    return flavors_\n\n\ndef clone_url(url):\n    # delay import of our code until needed for certain\n    from ..cmd import Runner\n    runner = Runner()\n    tdir = tempfile.mkdtemp(**get_tempfile_kwargs(\n        {'dir': dl_cfg.get(\"datalad.tests.temp.dir\")}, prefix='clone_url'))\n    _ = runner([\"git\", \"clone\", url, tdir], expect_stderr=True)\n    if GitRepo(tdir).is_with_annex():\n        AnnexRepo(tdir, init=True)\n    _TEMP_PATHS_CLONES.add(tdir)\n    return tdir\n\n\nif not on_windows:\n    local_testrepo_flavors = ['local'] # 'local-url'\nelse:\n    local_testrepo_flavors = ['network-clone']\n\n_TESTREPOS = None\n\ndef _get_testrepos_uris(regex, flavors):\n    global _TESTREPOS\n    # we should instantiate those whenever test repos actually asked for\n    # TODO: just absorb all this lazy construction within some class\n    if not _TESTREPOS:\n        from .utils_testrepos import BasicAnnexTestRepo, BasicGitTestRepo, \\\n            SubmoduleDataset, NestedDataset, InnerSubmodule\n\n        _basic_annex_test_repo = BasicAnnexTestRepo()\n        _basic_git_test_repo = BasicGitTestRepo()\n        _submodule_annex_test_repo = SubmoduleDataset()\n        _nested_submodule_annex_test_repo = NestedDataset()\n        _inner_submodule_annex_test_repo = InnerSubmodule()\n        _TESTREPOS = {'basic_annex':\n                        {'network': 'git://github.com/datalad/testrepo--basic--r1',\n                         'local': _basic_annex_test_repo.path,\n                         'local-url': _basic_annex_test_repo.url},\n                      'basic_git':\n                        {'local': _basic_git_test_repo.path,\n                         'local-url': _basic_git_test_repo.url},\n                      'submodule_annex':\n                        {'local': _submodule_annex_test_repo.path,\n                         'local-url': _submodule_annex_test_repo.url},\n                      'nested_submodule_annex':\n                        {'local': _nested_submodule_annex_test_repo.path,\n                         'local-url': _nested_submodule_annex_test_repo.url},\n                      # TODO: append 'annex' to the name:\n                      # Currently doesn't work with some annex tests, despite\n                      # working manually. So, figure out how the tests' setup\n                      # messes things up with this one.\n                      'inner_submodule':\n                        {'local': _inner_submodule_annex_test_repo.path,\n                         'local-url': _inner_submodule_annex_test_repo.url}\n                      }\n        # assure that now we do have those test repos created -- delayed\n        # their creation until actually used\n        _basic_annex_test_repo.create()\n        _basic_git_test_repo.create()\n        _submodule_annex_test_repo.create()\n        _nested_submodule_annex_test_repo.create()\n        _inner_submodule_annex_test_repo.create()\n    uris = []\n    for name, spec in _TESTREPOS.items():\n        if not re.match(regex, name):\n            continue\n        uris += [spec[x] for x in set(spec.keys()).intersection(flavors)]\n\n        # additional flavors which might have not been\n        if 'clone' in flavors and 'clone' not in spec:\n            uris.append(clone_url(spec['local']))\n\n        if 'network-clone' in flavors \\\n                and 'network' in spec \\\n                and 'network-clone' not in spec:\n            uris.append(clone_url(spec['network']))\n\n    return uris\n\n\n# addurls with our generated file:// URLs doesn't work on appveyor\n# https://ci.appveyor.com/project/mih/datalad/builds/29841505/job/330rwn2a3cvtrakj\n@known_failure_appveyor\n@optional_args\ndef with_testrepos(t, regex='.*', flavors='auto', skip=False, count=None):\n    \"\"\"Decorator to provide a local/remote test repository\n\n    All tests under datalad/tests/testrepos are stored in two-level hierarchy,\n    where top-level name describes nature/identifier of the test repository,\n    and there could be multiple instances (e.g. generated differently) of the\n    same \"content\"\n\n    Parameters\n    ----------\n    regex : string, optional\n      Regex to select which test repos to use\n    flavors : {'auto', 'local', 'local-url', 'clone', 'network', 'network-clone'} or list of thereof, optional\n      What URIs to provide.  E.g. 'local' would just provide path to the\n      repository, while 'network' would provide url of the remote location\n      available on Internet containing the test repository.  'clone' would\n      clone repository first to a temporary location. 'network-clone' would\n      first clone from the network location. 'auto' would include the list of\n      appropriate ones (e.g., no 'network*' flavors if network tests are\n      \"forbidden\").\n    count: int, optional\n      If specified, only up to that number of repositories to test with\n\n    Examples\n    --------\n\n    >>> from datalad.tests.utils import with_testrepos\n    >>> @with_testrepos('basic_annex')\n    ... def test_write(repo):\n    ...    assert(os.path.exists(os.path.join(repo, '.git', 'annex')))\n\n    \"\"\"\n    @wraps(t)\n    @attr('with_testrepos')\n    def  _wrap_with_testrepos(*arg, **kw):\n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n\n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (dl_cfg.get('datalad.tests.nonetwork') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n\n        fake_dates = dl_cfg.get(\"datalad.fake-dates\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n                t(*(arg + (uri,)), **kw)\n            finally:\n                # The is_explicit_path check is needed because it may be a URL,\n                # but check_dates needs a local path or GitRepo object.\n                if fake_dates and is_explicit_path(uri):\n                    from ..support.repodates import check_dates\n                    assert_false(\n                        check_dates(uri, annex=\"tree\")[\"objects\"])\n                if uri in _TEMP_PATHS_CLONES:\n                    _TEMP_PATHS_CLONES.discard(uri)\n                    rmtemp(uri)\n                pass  # might need to provide additional handling so, handle\n    return  _wrap_with_testrepos\nwith_testrepos.__test__ = False\n\n\n@optional_args\ndef with_sameas_remote(func, autoenabled=False):\n    \"\"\"Provide a repository with a git-annex sameas remote configured.\n\n    The repository will have two special remotes: r_dir (type=directory) and\n    r_rsync (type=rsync). The rsync remote will be configured with\n    --sameas=r_dir, and autoenabled if `autoenabled` is true.\n    \"\"\"\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import CommandError\n\n    @wraps(func)\n    @attr('with_sameas_remotes')\n    @skip_if_on_windows\n    @skip_ssh\n    @with_tempfile(mkdir=True)\n    @with_tempfile(mkdir=True)\n    def  _wrap_with_sameas_remote(*args, **kwargs):\n        # With git-annex's 8.20200522-77-g1f2e2d15e, transferring from an rsync\n        # special remote hangs on Xenial. This is likely due to an interaction\n        # with an older rsync or openssh version. Use openssh as a rough\n        # indicator. See\n        # https://git-annex.branchable.com/bugs/Recent_hang_with_rsync_remote_with_older_systems___40__Xenial__44___Jessie__41__/\n        if external_versions['cmd:system-ssh'] < '7.4' and \\\n           '8.20200522' < external_versions['cmd:annex'] < '8.20200720':\n            raise SkipTest(\"Test known to hang\")\n\n        sr_path, repo_path = args[-2:]\n        fn_args = args[:-2]\n        repo = AnnexRepo(repo_path)\n        repo.init_remote(\"r_dir\",\n                         options=[\"type=directory\",\n                                  \"encryption=none\",\n                                  \"directory=\" + sr_path])\n        options = [\"type=rsync\",\n                   \"rsyncurl=datalad-test:\" + sr_path]\n        if autoenabled:\n            options.append(\"autoenable=true\")\n        options.append(\"--sameas=r_dir\")\n\n        try:\n            repo.init_remote(\"r_rsync\", options=options)\n        except CommandError:\n            if repo.git_annex_version < \"7.20191017\":\n                raise SkipTest(\"git-annex lacks --sameas support\")\n            # This should have --sameas support.\n            raise\n        return func(*(fn_args + (repo,)), **kwargs)\n    return  _wrap_with_sameas_remote\n\n\n@optional_args\ndef with_fake_cookies_db(func, cookies={}):\n    \"\"\"mock original cookies db with a fake one for the duration of the test\n    \"\"\"\n    from ..support.cookies import cookies_db\n\n    @wraps(func)\n    @attr('with_fake_cookies_db')\n    def  _wrap_with_fake_cookies_db(*args, **kwargs):\n        try:\n            orig_cookies_db = cookies_db._cookies_db\n            cookies_db._cookies_db = cookies.copy()\n            return func(*args, **kwargs)\n        finally:\n            cookies_db._cookies_db = orig_cookies_db\n    return  _wrap_with_fake_cookies_db\n\n\n@optional_args\ndef assert_cwd_unchanged(func, ok_to_chdir=False):\n    \"\"\"Decorator to test whether the current working directory remains unchanged\n\n    Parameters\n    ----------\n    ok_to_chdir: bool, optional\n      If True, allow to chdir, so this decorator would not then raise exception\n      if chdir'ed but only return to original directory\n    \"\"\"\n\n    @wraps(func)\n    def  _wrap_assert_cwd_unchanged(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n\n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n\n        if exc_info is not None:\n            raise exc_info[1]\n\n        return ret\n\n    return  _wrap_assert_cwd_unchanged\n\n\n@optional_args\ndef run_under_dir(func, newdir='.'):\n    \"\"\"Decorator to run tests under another directory\n\n    It is somewhat ugly since we can't really chdir\n    back to a directory which had a symlink in its path.\n    So using this decorator has potential to move entire\n    testing run under the dereferenced directory name -- sideeffect.\n\n    The only way would be to instruct testing framework (i.e. nose\n    in our case ATM) to run a test by creating a new process with\n    a new cwd\n    \"\"\"\n\n    @wraps(func)\n    def  _wrap_run_under_dir(*args, **kwargs):\n        pwd_before = getpwd()\n        try:\n            chpwd(newdir)\n            func(*args, **kwargs)\n        finally:\n            chpwd(pwd_before)\n\n\n    return  _wrap_run_under_dir\n\n\ndef assert_re_in(regex, c, flags=0, match=True, msg=None):\n    \"\"\"Assert that container (list, str, etc) contains entry matching the regex\n    \"\"\"\n    if not isinstance(c, (list, tuple)):\n        c = [c]\n    for e in c:\n        if (re.match if match else re.search)(regex, e, flags=flags):\n            return\n    raise AssertionError(\n        msg or \"Not a single entry matched %r in %r\" % (regex, c)\n    )\n\n\ndef assert_dict_equal(d1, d2):\n    msgs = []\n    if set(d1).difference(d2):\n        msgs.append(\" keys in the first dict but not in the second: %s\"\n                    % list(set(d1).difference(d2)))\n    if set(d2).difference(d1):\n        msgs.append(\" keys in the second dict but not in the first: %s\"\n                    % list(set(d2).difference(d1)))\n    for k in set(d1).intersection(d2):\n        same = True\n        try:\n            if isinstance(d1[k], str):\n                # do not compare types for string types to avoid all the hassle\n                # with the distinction of str and unicode in PY3, and simple\n                # test for equality\n                same = bool(d1[k] == d2[k])\n            else:\n                same = type(d1[k]) == type(d2[k]) and bool(d1[k] == d2[k])\n        except:  # if comparison or conversion to bool (e.g. with numpy arrays) fails\n            same = False\n\n        if not same:\n            msgs.append(\" [%r] differs: %r != %r\" % (k, d1[k], d2[k]))\n\n        if len(msgs) > 10:\n            msgs.append(\"and more\")\n            break\n    if msgs:\n        raise AssertionError(\"dicts differ:\\n%s\" % \"\\n\".join(msgs))\n    # do generic comparison just in case we screwed up to detect difference correctly above\n    eq_(d1, d2)\n\n\ndef assert_str_equal(s1, s2):\n    \"\"\"Helper to compare two lines\"\"\"\n    diff = list(unified_diff(s1.splitlines(), s2.splitlines()))\n    assert not diff, '\\n'.join(diff)\n    assert_equal(s1, s2)\n\n\ndef assert_status(label, results):\n    \"\"\"Verify that each status dict in the results has a given status label\n\n    `label` can be a sequence, in which case status must be one of the items\n    in this sequence.\n    \"\"\"\n    label = assure_list(label)\n    results = assure_list(results)\n    for i, r in enumerate(results):\n        try:\n            assert_in('status', r)\n            assert_in(r['status'], label)\n        except AssertionError:\n            raise AssertionError('Test {}/{}: expected status {} not found in:\\n{}'.format(\n                i + 1,\n                len(results),\n                label,\n                dumps(results, indent=1, default=lambda x: str(x))))\n\n\ndef assert_message(message, results):\n    \"\"\"Verify that each status dict in the results has a message\n\n    This only tests the message template string, and not a formatted message\n    with args expanded.\n    \"\"\"\n    for r in assure_list(results):\n        assert_in('message', r)\n        m = r['message'][0] if isinstance(r['message'], tuple) else r['message']\n        assert_equal(m, message)\n\n\ndef _format_res(x):\n    return textwrap.indent(\n        dumps(x, indent=1, default=str, sort_keys=True),\n        prefix=\"  \")\n\n\ndef assert_result_count(results, n, **kwargs):\n    \"\"\"Verify specific number of results (matching criteria, if any)\"\"\"\n    count = 0\n    results = assure_list(results)\n    for r in results:\n        if not len(kwargs):\n            count += 1\n        elif all(k in r and r[k] == v for k, v in kwargs.items()):\n            count += 1\n    if not n == count:\n        raise AssertionError(\n            'Got {} instead of {} expected results matching\\n{}\\nInspected {} record(s):\\n{}'.format(\n                count,\n                n,\n                _format_res(kwargs),\n                len(results),\n                _format_res(results)))\n\n\ndef assert_in_results(results, **kwargs):\n    \"\"\"Verify that the particular combination of keys and values is found in\n    one of the results\"\"\"\n    found = False\n    for r in assure_list(results):\n        if all(k in r and r[k] == v for k, v in kwargs.items()):\n            found = True\n    if not found:\n        raise AssertionError(\n            \"Desired result\\n{}\\nnot found among\\n{}\"\n            .format(_format_res(kwargs), _format_res(results)))\n\n\ndef assert_not_in_results(results, **kwargs):\n    \"\"\"Verify that the particular combination of keys and values is not in any\n    of the results\"\"\"\n    for r in assure_list(results):\n        assert any(k not in r or r[k] != v for k, v in kwargs.items())\n\n\ndef assert_result_values_equal(results, prop, values):\n    \"\"\"Verify that the values of all results for a given key in the status dicts\n    match the given sequence\"\"\"\n    assert_equal(\n        [r[prop] for r in results],\n        values)\n\n\ndef assert_result_values_cond(results, prop, cond):\n    \"\"\"Verify that the values of all results for a given key in the status dicts\n    fullfill condition `cond`.\n\n    Parameters\n    ----------\n    results:\n    prop: str\n    cond: callable\n    \"\"\"\n    for r in assure_list(results):\n        ok_(cond(r[prop]),\n            msg=\"r[{prop}]: {value}\".format(prop=prop, value=r[prop]))\n\n\ndef ignore_nose_capturing_stdout(func):\n    \"\"\"DEPRECATED and will be removed soon.  Does nothing!\n\n    Originally was intended as a decorator workaround for nose's behaviour\n    with redirecting sys.stdout, but now we monkey patch nose now so no test\n    should no longer be skipped.\n\n    See issue reported here:\n    https://code.google.com/p/python-nose/issues/detail?id=243&can=1&sort=-id&colspec=ID%20Type%20Status%20Priority%20Stars%20Milestone%20Owner%20Summary\n\n    \"\"\"\n    lgr.warning(\n        \"@ignore_nose_capturing_stdout no longer does anything - nose should \"\n        \"just be monkey patched in setup_package. {} still has it\"\n        .format(func.__name__)\n    )\n    return func\n\n\ndef skip_httpretty_on_problematic_pythons(func):\n    \"\"\"As discovered some httpretty bug causes a side-effect\n    on other tests on some Pythons.  So we skip the test if such\n    problematic combination detected\n\n    References\n    https://travis-ci.org/datalad/datalad/jobs/94464988\n    http://stackoverflow.com/a/29603206/1265472\n    \"\"\"\n\n    @make_decorator(func)\n    def  _wrap_skip_httpretty_on_problematic_pythons(*args, **kwargs):\n        if sys.version_info[:3] == (3, 4, 2):\n            raise SkipTest(\"Known to cause trouble due to httpretty bug on this Python\")\n        return func(*args, **kwargs)\n    return  _wrap_skip_httpretty_on_problematic_pythons\n\n\n@optional_args\ndef with_parametric_batch(t):\n    \"\"\"Helper to run parametric test with possible combinations of batch and direct\n    \"\"\"\n    @wraps(t)\n    def  _wrap_with_parametric_batch():\n        for batch in (False, True):\n                yield t, batch\n\n    return  _wrap_with_parametric_batch\n\n\n# List of most obscure filenames which might or not be supported by different\n# filesystems across different OSs.  Start with the most obscure\nOBSCURE_PREFIX = os.getenv('DATALAD_TESTS_OBSCURE_PREFIX', '')\n# Those will be tried to be added to the base name if filesystem allows\nOBSCURE_FILENAME_PARTS = ['/', '|', ';', '&',\n                          # TODO: add '%b5',\n                          ' ']\nUNICODE_FILENAME = u\"ΔЙקم๗あ\"\n\n# OSX is exciting -- some I guess FS might be encoding differently from decoding\n# so Й might get recoded\n# (ref: https://github.com/datalad/datalad/pull/1921#issuecomment-385809366)\nif sys.getfilesystemencoding().lower() == 'utf-8':\n    if on_osx:\n        # TODO: figure it really out\n        UNICODE_FILENAME = UNICODE_FILENAME.replace(u\"Й\", u\"\")\n    if on_windows:\n        # TODO: really figure out unicode handling on windows\n        UNICODE_FILENAME = ''\n    if UNICODE_FILENAME:\n        OBSCURE_FILENAME_PARTS.append(UNICODE_FILENAME)\n# simple extension to finish it up\nOBSCURE_FILENAME_PARTS.append('.datc')\n\n\n@with_tempfile(mkdir=True)\ndef get_most_obscure_supported_name(tdir, return_candidates=False):\n    \"\"\"Return the most obscure filename that the filesystem would support under TEMPDIR\n\n    Parameters\n    ----------\n    return_candidates: bool, optional\n      if True, return a tuple of (good, candidates) where candidates are \"partially\"\n      sorted from trickiest considered\n    TODO: we might want to use it as a function where we would provide tdir\n    \"\"\"\n    # we need separate good_base so we do not breed leading/trailing spaces\n    initial = good = 'a'  # everyone should support that!\n    system = platform.system()\n\n    OBSCURE_FILENAMES = []\n    def good_filename(filename):\n        OBSCURE_FILENAMES.append(candidate)\n        try:\n            with open(opj(tdir, filename), 'w') as f:\n                f.write(\"TEST LOAD\")\n            return True\n        except:\n            lgr.debug(\"Filename %r is not supported on %s under %s\",\n                      filename, system, tdir)\n            return False\n\n    # incrementally build up the most obscure filename from parts\n    for part in OBSCURE_FILENAME_PARTS:\n        candidate = good + part\n        if good_filename(OBSCURE_PREFIX + candidate):\n            good = candidate\n\n    # now we will compose some candidates with trailing spaces - trickier ones first\n    # so we break the loop as soon as we get it\n    for candidate in [' ' + good + ' ', ' ' + good, good + ' ', good]:\n        candidate = OBSCURE_PREFIX + candidate\n        if on_windows and candidate.rstrip() != candidate:\n            continue\n        if good_filename(candidate):\n            good = candidate\n            break\n\n    if good == initial:\n        raise RuntimeError(\"Could not create any of the files under %s among %s\"\n                       % (tdir, OBSCURE_FILENAMES))\n    lgr.debug(\"Tested %d obscure filename candidates. The winner: %r\", len(OBSCURE_FILENAMES), good)\n    if return_candidates:\n        return good, OBSCURE_FILENAMES[::-1]\n    else:\n        return good\n\n\nOBSCURE_FILENAME, OBSCURE_FILENAMES = get_most_obscure_supported_name(return_candidates=True)\n\n\n@optional_args\ndef with_testsui(t, responses=None, interactive=True):\n    \"\"\"Switch main UI to be 'tests' UI and possibly provide answers to be used\"\"\"\n\n    @wraps(t)\n    def  _wrap_with_testsui(*args, **kwargs):\n        from datalad.ui import ui\n        old_backend = ui.backend\n        try:\n            ui.set_backend('tests' if interactive else 'tests-noninteractive')\n            if responses:\n                ui.add_responses(responses)\n            ret = t(*args, **kwargs)\n            if responses:\n                responses_left = ui.get_responses()\n                assert not len(responses_left), \"Some responses were left not used: %s\" % str(responses_left)\n            return ret\n        finally:\n            ui.set_backend(old_backend)\n\n    if not interactive and responses is not None:\n        raise ValueError(\"Non-interactive UI cannot provide responses\")\n\n    return  _wrap_with_testsui\n\nwith_testsui.__test__ = False\n\n\ndef assert_no_errors_logged(func, skip_re=None):\n    \"\"\"Decorator around function to assert that no errors logged during its execution\"\"\"\n    @wraps(func)\n    def  _wrap_assert_no_errors_logged(*args, **kwargs):\n        with swallow_logs(new_level=logging.ERROR) as cml:\n            out = func(*args, **kwargs)\n            if cml.out:\n                if not (skip_re and re.search(skip_re, cml.out)):\n                    raise AssertionError(\n                        \"Expected no errors to be logged, but log output is %s\"\n                        % cml.out\n                    )\n        return out\n\n    return  _wrap_assert_no_errors_logged\n\n\ndef get_mtimes_and_digests(target_path):\n    \"\"\"Return digests (md5) and mtimes for all the files under target_path\"\"\"\n    from datalad.utils import find_files\n    from datalad.support.digests import Digester\n    digester = Digester(['md5'])\n\n    # bother only with existing ones for this test, i.e. skip annexed files without content\n    target_files = [\n        f for f in find_files('.*', topdir=target_path, exclude_vcs=False, exclude_datalad=False)\n        if exists(f)\n    ]\n    # let's leave only relative paths for easier analysis\n    target_files_ = [relpath(f, target_path) for f in target_files]\n\n    digests = {frel: digester(f) for f, frel in zip(target_files, target_files_)}\n    mtimes = {frel: os.stat(f).st_mtime for f, frel in zip(target_files, target_files_)}\n    return digests, mtimes\n\n\ndef get_datasets_topdir():\n    \"\"\"Delayed parsing so it could be monkey patched etc\"\"\"\n    from datalad.consts import DATASETS_TOPURL\n    return RI(DATASETS_TOPURL).hostname\n\n\ndef assert_repo_status(path, annex=None, untracked_mode='normal', **kwargs):\n    \"\"\"Compare a repo status against (optional) exceptions.\n\n    Anything file/directory that is not explicitly indicated must have\n    state 'clean', i.e. no modifications and recorded in Git.\n\n    Parameters\n    ----------\n    path: str or Repo\n      in case of a str: path to the repository's base dir;\n      Note, that passing a Repo instance prevents detecting annex. This might\n      be useful in case of a non-initialized annex, a GitRepo is pointing to.\n    annex: bool or None\n      explicitly set to True or False to indicate, that an annex is (not)\n      expected; set to None to autodetect, whether there is an annex.\n      Default: None.\n    untracked_mode: {'no', 'normal', 'all'}\n      If and how untracked content is reported. The specification of untracked\n      files that are OK to be found must match this mode. See `Repo.status()`\n    **kwargs\n      Files/directories that are OK to not be in 'clean' state. Each argument\n      must be one of 'added', 'untracked', 'deleted', 'modified' and each\n      value must be a list of filenames (relative to the root of the\n      repository, in POSIX convention).\n    \"\"\"\n    r = None\n    if isinstance(path, AnnexRepo):\n        if annex is None:\n            annex = True\n        # if `annex` was set to False, but we find an annex => fail\n        assert_is(annex, True)\n        r = path\n    elif isinstance(path, GitRepo):\n        if annex is None:\n            annex = False\n        # explicitly given GitRepo instance doesn't make sense with\n        # 'annex' True\n        assert_is(annex, False)\n        r = path\n    else:\n        # 'path' is an actual path\n        try:\n            r = AnnexRepo(path, init=False, create=False)\n            if annex is None:\n                annex = True\n            # if `annex` was set to False, but we find an annex => fail\n            assert_is(annex, True)\n        except Exception:\n            # Instantiation failed => no annex\n            try:\n                r = GitRepo(path, init=False, create=False)\n            except Exception:\n                raise AssertionError(\"Couldn't find an annex or a git \"\n                                     \"repository at {}.\".format(path))\n            if annex is None:\n                annex = False\n            # explicitly given GitRepo instance doesn't make sense with\n            # 'annex' True\n            assert_is(annex, False)\n\n    status = r.status(untracked=untracked_mode)\n    # for any file state that indicates some kind of change (all but 'clean)\n    for state in ('added', 'untracked', 'deleted', 'modified'):\n        oktobefound = sorted(r.pathobj.joinpath(ut.PurePosixPath(p))\n                             for p in kwargs.get(state, []))\n        state_files = sorted(k for k, v in status.items()\n                             if v.get('state', None) == state)\n        eq_(state_files, oktobefound,\n            'unexpected content of state \"%s\": %r != %r'\n            % (state, state_files, oktobefound))\n\n\ndef get_convoluted_situation(path, repocls=AnnexRepo):\n    from datalad.api import create\n\n    if 'APPVEYOR' in os.environ:\n        # issue only happens on appveyor, Python itself implodes\n        # cannot be reproduced on a real windows box\n        raise SkipTest(\n            'get_convoluted_situation() causes appveyor to crash, '\n            'reason unknown')\n    repo = repocls(path, create=True)\n    # use create(force) to get an ID and config into the empty repo\n    ds = Dataset(path).create(force=True)\n    # base content\n    create_tree(\n        ds.path,\n        {\n            '.gitignore': '*.ignored',\n            'subdir': {\n                'file_clean': 'file_clean',\n                'file_deleted': 'file_deleted',\n                'file_modified': 'file_clean',\n            },\n            'subdir-only-ignored': {\n                '1.ignored': '',\n            },\n            'file_clean': 'file_clean',\n            'file_deleted': 'file_deleted',\n            'file_staged_deleted': 'file_staged_deleted',\n            'file_modified': 'file_clean',\n        }\n    )\n    if isinstance(ds.repo, AnnexRepo):\n        create_tree(\n            ds.path,\n            {\n                'subdir': {\n                    'file_dropped_clean': 'file_dropped_clean',\n                },\n                'file_dropped_clean': 'file_dropped_clean',\n            }\n        )\n    ds.save()\n    if isinstance(ds.repo, AnnexRepo):\n        # some files straight in git\n        create_tree(\n            ds.path,\n            {\n                'subdir': {\n                    'file_ingit_clean': 'file_ingit_clean',\n                    'file_ingit_modified': 'file_ingit_clean',\n                },\n                'file_ingit_clean': 'file_ingit_clean',\n                'file_ingit_modified': 'file_ingit_clean',\n            }\n        )\n        ds.save(to_git=True)\n        ds.drop([\n            'file_dropped_clean',\n            op.join('subdir', 'file_dropped_clean')],\n            check=False)\n    # clean and proper subdatasets\n    ds.create('subds_clean')\n    ds.create(op.join('subdir', 'subds_clean'))\n    ds.create('subds_unavailable_clean')\n    ds.create(op.join('subdir', 'subds_unavailable_clean'))\n    # uninstall some subdatasets (still clean)\n    ds.uninstall([\n        'subds_unavailable_clean',\n        op.join('subdir', 'subds_unavailable_clean')],\n        check=False)\n    assert_repo_status(ds.path)\n    # make a dirty subdataset\n    ds.create('subds_modified')\n    ds.create(op.join('subds_modified', 'someds'))\n    ds.create(op.join('subds_modified', 'someds', 'dirtyds'))\n    # make a subdataset with additional commits\n    ds.create(op.join('subdir', 'subds_modified'))\n    pdspath = op.join(ds.path, 'subdir', 'subds_modified', 'progressedds')\n    ds.create(pdspath)\n    create_tree(\n        pdspath,\n        {'file_clean': 'file_ingit_clean'}\n    )\n    Dataset(pdspath).save()\n    assert_repo_status(pdspath)\n    # staged subds, and files\n    create(op.join(ds.path, 'subds_added'))\n    ds.repo.add_submodule('subds_added')\n    create(op.join(ds.path, 'subdir', 'subds_added'))\n    ds.repo.add_submodule(op.join('subdir', 'subds_added'))\n    # some more untracked files\n    create_tree(\n        ds.path,\n        {\n            'subdir': {\n                'file_untracked': 'file_untracked',\n                'file_added': 'file_added',\n            },\n            'file_untracked': 'file_untracked',\n            'file_added': 'file_added',\n            'dir_untracked': {\n                'file_untracked': 'file_untracked',\n            },\n            'subds_modified': {\n                'someds': {\n                    \"dirtyds\": {\n                        'file_untracked': 'file_untracked',\n                    },\n                },\n            },\n        }\n    )\n    ds.repo.add(['file_added', op.join('subdir', 'file_added')])\n    # untracked subdatasets\n    create(op.join(ds.path, 'subds_untracked'))\n    create(op.join(ds.path, 'subdir', 'subds_untracked'))\n    # deleted files\n    os.remove(op.join(ds.path, 'file_deleted'))\n    os.remove(op.join(ds.path, 'subdir', 'file_deleted'))\n    # staged deletion\n    ds.repo.remove('file_staged_deleted')\n    # modified files\n    if isinstance(ds.repo, AnnexRepo):\n        ds.repo.unlock(['file_modified', op.join('subdir', 'file_modified')])\n        create_tree(\n            ds.path,\n            {\n                'subdir': {\n                    'file_ingit_modified': 'file_ingit_modified',\n                },\n                'file_ingit_modified': 'file_ingit_modified',\n            }\n        )\n    create_tree(\n        ds.path,\n        {\n            'subdir': {\n                'file_modified': 'file_modified',\n            },\n            'file_modified': 'file_modified',\n        }\n    )\n    return ds\n\n\ndef get_deeply_nested_structure(path):\n    \"\"\" Here is what this does (assuming UNIX, locked):\n    |  .\n    |  ├── directory_untracked\n    |  │   └── link2dir -> ../subdir\n    |  ├── OBSCURE_FILENAME_file_modified\n    |  ├── link2dir -> subdir\n    |  ├── link2subdsdir -> subds_modified/subdir\n    |  ├── link2subdsroot -> subds_modified\n    |  ├── subdir\n    |  │   ├── annexed_file.txt -> ../.git/annex/objects/...\n    |  │   ├── file_modified\n    |  │   ├── git_file.txt\n    |  │   └── link2annex_files.txt -> annexed_file.txt\n    |  └── subds_modified\n    |      ├── link2superdsdir -> ../subdir\n    |      ├── subdir\n    |      │   └── annexed_file.txt -> ../.git/annex/objects/...\n    |      └── subds_lvl1_modified\n    |          └── OBSCURE_FILENAME_directory_untracked\n    |              └── untracked_file\n\n    When a system has no symlink support, the link2... components are not\n    included.\n    \"\"\"\n    ds = Dataset(path).create()\n    (ds.pathobj / 'subdir').mkdir()\n    (ds.pathobj / 'subdir' / 'annexed_file.txt').write_text(u'dummy')\n    ds.save()\n    (ds.pathobj / 'subdir' / 'git_file.txt').write_text(u'dummy')\n    ds.save(to_git=True)\n    # a subtree of datasets\n    subds = ds.create('subds_modified')\n    # another dataset, plus an additional dir in it\n    ds.create(op.join('subds_modified', 'subds_lvl1_modified'))\n    create_tree(\n        ds.path,\n        {\n            'subdir': {\n                'file_modified': 'file_modified',\n            },\n            OBSCURE_FILENAME + u'file_modified_': 'file_modified',\n        }\n    )\n    create_tree(\n        str(ds.pathobj / 'subds_modified' / 'subds_lvl1_modified'),\n        {OBSCURE_FILENAME + u'_directory_untracked': {\"untraced_file\": \"\"}}\n    )\n    (ut.Path(subds.path) / 'subdir').mkdir()\n    (ut.Path(subds.path) / 'subdir' / 'annexed_file.txt').write_text(u'dummy')\n    subds.save()\n    (ds.pathobj / 'directory_untracked').mkdir()\n\n    if not has_symlink_capability():\n        return ds\n\n    # symlink farm #1\n    # symlink to annexed file\n    (ds.pathobj / 'subdir' / 'link2annex_files.txt').symlink_to(\n        'annexed_file.txt')\n    # symlink to directory within the dataset\n    (ds.pathobj / 'link2dir').symlink_to('subdir')\n    # upwards pointing symlink to directory within the same dataset\n    (ds.pathobj / 'directory_untracked' / 'link2dir').symlink_to(\n        op.join('..', 'subdir'))\n    # symlink pointing to a subdataset mount in the same dataset\n    (ds.pathobj / 'link2subdsroot').symlink_to('subds_modified')\n    # symlink to a dir in a subdataset (across dataset boundaries)\n    (ds.pathobj / 'link2subdsdir').symlink_to(\n        op.join('subds_modified', 'subdir'))\n    # symlink to a dir in a superdataset (across dataset boundaries)\n    (ut.Path(subds.path) / 'link2superdsdir').symlink_to(\n        op.join('..', 'subdir'))\n    return ds\n\n\n@with_tempfile\n@with_tempfile\ndef has_symlink_capability(p1, p2):\n\n    path = ut.Path(p1)\n    target = ut.Path(p2)\n    return utils.check_symlink_capability(path, target)\n\n\ndef skip_wo_symlink_capability(func):\n    \"\"\"Skip test when environment does not support symlinks\n\n    Perform a behavioral test instead of top-down logic, as on\n    windows this could be on or off on a case-by-case basis.\n    \"\"\"\n    @wraps(func)\n    @attr('skip_wo_symlink_capability')\n    def  _wrap_skip_wo_symlink_capability(*args, **kwargs):\n        if not has_symlink_capability():\n            raise SkipTest(\"no symlink capabilities\")\n        return func(*args, **kwargs)\n    return  _wrap_skip_wo_symlink_capability\n\n\ndef get_ssh_port(host):\n    \"\"\"Get port of `host` in ssh_config.\n\n    Our tests depend on the host being defined in ssh_config, including its\n    port. This method can be used by tests that want to check handling of an\n    explicitly specified\n\n    Note that if `host` does not match a host in ssh_config, the default value\n    of 22 is returned.\n\n    Parameters\n    ----------\n    host : str\n\n    Returns\n    -------\n    port (int)\n\n    Raises\n    ------\n    SkipTest if port cannot be found.\n    \"\"\"\n    out = ''\n    try:\n        out, err = Runner()([\"ssh\", \"-G\", host])\n    except Exception as exc:\n        err = str(exc)\n\n    port = None\n    for line in out.splitlines():\n        if line.startswith(\"port \"):\n            try:\n                port = int(line.split()[1])\n            except Exception as exc:\n                err = str(exc)\n            break\n\n    if port is None:\n        raise SkipTest(\"port for {} could not be determined: {}\"\n                       .format(host, err))\n    return port\n\n\n#\n# Context Managers\n#\n\n\ndef patch_config(vars):\n    \"\"\"Patch our config with custom settings. Returns mock.patch cm\n    \"\"\"\n    return patch.dict(dl_cfg._store, vars)\n\n\n@contextmanager\ndef set_date(timestamp):\n    \"\"\"Temporarily override environment variables for git/git-annex dates.\n\n    Parameters\n    ----------\n    timestamp : int\n        Unix timestamp.\n    \"\"\"\n    git_ts = \"@{} +0000\".format(timestamp)\n    with patch.dict(\"os.environ\",\n                    {\"GIT_COMMITTER_DATE\": git_ts,\n                     \"GIT_AUTHOR_DATE\": git_ts,\n                     \"GIT_ANNEX_VECTOR_CLOCK\": str(timestamp),\n                     \"DATALAD_FAKE__DATES\": \"0\"}):\n        yield\n\n\n@contextmanager\ndef set_annex_version(version):\n    \"\"\"Override the git-annex version.\n\n    This temporarily masks the git-annex version present in external_versions\n    and make AnnexRepo forget its cached version information.\n    \"\"\"\n    from datalad.support.annexrepo import AnnexRepo\n    ar_vers = AnnexRepo.git_annex_version\n    with patch.dict(\n            \"datalad.support.annexrepo.external_versions._versions\",\n            {\"cmd:annex\": version}):\n        try:\n            AnnexRepo.git_annex_version = None\n            yield\n        finally:\n            AnnexRepo.git_annex_version = ar_vers\n\n#\n# Test tags\n#\n# To be explicit, and not \"loose\" some tests due to typos, decided to make\n# explicit decorators for common types\n\nfrom nose.plugins.attrib import attr\n\n\ndef integration(f):\n    \"\"\"Mark test as an \"integration\" test which generally is not needed to be run\n    \n    Generally tend to be slower.\n    Should be used in combination with @slow and @turtle if that is the case.\n    \"\"\"\n    return attr('integration')(f)\n\n\ndef slow(f):\n    \"\"\"Mark test as a slow, although not necessarily integration or usecase test\n\n    Rule of thumb cut-off to mark as slow is 10 sec\n    \"\"\"\n    return attr('slow')(f)\n\n\ndef turtle(f):\n    \"\"\"Mark test as very slow, meaning to not run it on Travis due to its\n    time limit\n\n    Rule of thumb cut-off to mark as turtle is 2 minutes\n    \"\"\"\n    return attr('turtle')(f)\n\n\ndef usecase(f):\n    \"\"\"Mark test as a usecase user ran into and which (typically) caused bug report\n    to be filed/troubleshooted\n\n    Should be used in combination with @slow and @turtle if slow.\n    \"\"\"\n    return attr('usecase')(f)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil; coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Miscellaneous utilities to assist with testing\"\"\"\n\nimport glob\nimport gzip\nimport inspect\nimport shutil\nimport stat\nfrom json import dumps\nimport os\nimport re\nimport tempfile\nimport platform\nimport multiprocessing\nimport logging\nimport random\nimport socket\nimport textwrap\nimport warnings\nfrom fnmatch import fnmatch\nimport time\nfrom difflib import unified_diff\nfrom contextlib import contextmanager\nfrom unittest.mock import patch\n\nfrom http.server import (\n    HTTPServer,\n    SimpleHTTPRequestHandler,\n)\n\nfrom functools import wraps\nfrom os.path import (\n    curdir,\n    exists,\n    join as opj,\n    relpath,\n    split as pathsplit,\n)\n\nfrom nose.plugins.attrib import attr\nfrom nose.tools import (\n    assert_equal,\n    assert_false,\n    assert_greater,\n    assert_greater_equal,\n    assert_in as in_,\n    assert_in,\n    assert_is,\n    assert_is_none,\n    assert_is_not,\n    assert_is_not_none,\n    assert_not_equal,\n    assert_not_in,\n    assert_not_is_instance,\n    assert_raises,\n    assert_true,\n    eq_,\n    make_decorator,\n    ok_,\n    raises,\n)\n\nfrom nose.tools import assert_set_equal\nfrom nose.tools import assert_is_instance\nfrom nose import SkipTest\n\nfrom datalad import cfg as dl_cfg\nimport datalad.utils as ut\n# TODO this must go\nfrom ..utils import *\nfrom datalad.utils import (\n    Path,\n    ensure_unicode,\n)\n\nfrom ..cmd import Runner\nfrom .. import utils\nfrom ..support.exceptions import (\n    CommandError,\n    CommandNotAvailableError,\n)\nfrom ..support.external_versions import external_versions\nfrom ..support.vcr_ import *\nfrom ..support.keyring_ import MemoryKeyring\nfrom ..support.network import RI\nfrom ..dochelpers import exc_str, borrowkwargs\nfrom ..cmdline.helpers import get_repo_instance\nfrom ..consts import (\n    ARCHIVES_TEMP_DIR,\n)\nfrom . import _TEMP_PATHS_GENERATED\n\n# temp paths used by clones\n_TEMP_PATHS_CLONES = set()\n\n\n# Additional indicators\non_travis = bool(os.environ.get('TRAVIS', False))\n\nif external_versions[\"cmd:git\"] >= \"2.28\":\n    # The specific value here doesn't matter, but it should not be the default\n    # from any Git version to test that we work with custom values.\n    DEFAULT_BRANCH = \"dl-test-branch\"  # Set by setup_package().\nelse:\n    DEFAULT_BRANCH = \"master\"\n\n# additional shortcuts\nneq_ = assert_not_equal\nnok_ = assert_false\n\nlgr = logging.getLogger(\"datalad.tests.utils\")\n\n\ndef skip_if_no_module(module):\n    try:\n        imp = __import__(module)\n    except Exception as exc:\n        raise SkipTest(\"Module %s fails to load: %s\" % (module, exc_str(exc)))\n\n\ndef skip_if_scrapy_without_selector():\n    \"\"\"A little helper to skip some tests which require recent scrapy\"\"\"\n    try:\n        import scrapy\n        from scrapy.selector import Selector\n    except ImportError:\n        from nose import SkipTest\n        raise SkipTest(\n            \"scrapy misses Selector (too old? version: %s)\"\n            % getattr(scrapy, '__version__'))\n\n\ndef skip_if_url_is_not_available(url, regex=None):\n    # verify that dataset is available\n    from datalad.downloaders.providers import Providers\n    from datalad.downloaders.base import DownloadError\n    providers = Providers.from_config_files()\n    try:\n        content = providers.fetch(url)\n        if regex and re.search(regex, content):\n            raise SkipTest(\"%s matched %r -- skipping the test\" % (url, regex))\n    except DownloadError:\n        raise SkipTest(\"%s failed to download\" % url)\n\n\ndef check_not_generatorfunction(func):\n    \"\"\"Internal helper to verify that we are not decorating generator tests\"\"\"\n    if inspect.isgeneratorfunction(func):\n        raise RuntimeError(\"{}: must not be decorated, is a generator test\"\n                           .format(func.__name__))\n\n\ndef skip_if_no_network(func=None):\n    \"\"\"Skip test completely in NONETWORK settings\n\n    If not used as a decorator, and just a function, could be used at the module level\n    \"\"\"\n    check_not_generatorfunction(func)\n\n    def check_and_raise():\n        if dl_cfg.get('datalad.tests.nonetwork'):\n            raise SkipTest(\"Skipping since no network settings\")\n\n    if func:\n        @wraps(func)\n        @attr('network')\n        @attr('skip_if_no_network')\n        def  _wrap_skip_if_no_network(*args, **kwargs):\n            check_and_raise()\n            return func(*args, **kwargs)\n        return  _wrap_skip_if_no_network\n    else:\n        check_and_raise()\n\n\ndef skip_if_on_windows(func=None):\n    \"\"\"Skip test completely under Windows\n    \"\"\"\n    check_not_generatorfunction(func)\n\n    def check_and_raise():\n        if on_windows:\n            raise SkipTest(\"Skipping on Windows\")\n\n    if func:\n        @wraps(func)\n        @attr('skip_if_on_windows')\n        def  _wrap_skip_if_on_windows(*args, **kwargs):\n            check_and_raise()\n            return func(*args, **kwargs)\n        return  _wrap_skip_if_on_windows\n    else:\n        check_and_raise()\n\n\ndef skip_if_root(func=None):\n    \"\"\"Skip test if uid == 0.\n\n    Note that on Windows (or anywhere else `os.geteuid` is not available) the\n    test is _not_ skipped.\n    \"\"\"\n    check_not_generatorfunction(func)\n\n    def check_and_raise():\n        if hasattr(os, \"geteuid\") and os.geteuid() == 0:\n            raise SkipTest(\"Skipping: test assumptions fail under root\")\n\n    if func:\n        @wraps(func)\n        @attr('skip_if_root')\n        def  _wrap_skip_if_root(*args, **kwargs):\n            check_and_raise()\n            return func(*args, **kwargs)\n        return  _wrap_skip_if_root\n    else:\n        check_and_raise()\n\n\n@optional_args\ndef skip_if(func, cond=True, msg=None, method='raise'):\n    \"\"\"Skip test for specific condition\n\n    Parameters\n    ----------\n    cond: bool\n      condition on which to skip\n    msg: str\n      message to print if skipping\n    method: str\n      either 'raise' or 'pass'. Whether to skip by raising `SkipTest` or by\n      just proceeding and simply not calling the decorated function.\n      This is particularly meant to be used, when decorating single assertions\n      in a test with method='pass' in order to not skip the entire test, but\n      just that assertion.\n    \"\"\"\n\n    check_not_generatorfunction(func)\n\n    @wraps(func)\n    def  _wrap_skip_if(*args, **kwargs):\n        if cond:\n            if method == 'raise':\n                raise SkipTest(msg if msg else \"condition was True\")\n            elif method == 'pass':\n                print(msg if msg else \"condition was True\")\n                return\n        return func(*args, **kwargs)\n    return  _wrap_skip_if\n\n\ndef skip_ssh(func):\n    \"\"\"Skips SSH tests if on windows or if environment variable\n    DATALAD_TESTS_SSH was not set\n    \"\"\"\n\n    check_not_generatorfunction(func)\n\n    @wraps(func)\n    @attr('skip_ssh')\n    def  _wrap_skip_ssh(*args, **kwargs):\n        test_ssh = dl_cfg.get(\"datalad.tests.ssh\", '')\n        if not test_ssh or test_ssh in ('0', 'false', 'no'):\n            raise SkipTest(\"Run this test by setting DATALAD_TESTS_SSH\")\n        return func(*args, **kwargs)\n    return  _wrap_skip_ssh\n\n\n@optional_args\ndef skip_v6_or_later(func, method='raise'):\n    \"\"\"Skip tests if v6 or later will be used as the default repo version.\n\n    The default repository version is controlled by the configured value of\n    DATALAD_REPO_VERSION and whether v5 repositories are supported by the\n    installed git-annex.\n    \"\"\"\n\n    from datalad.support.annexrepo import AnnexRepo\n\n    version = dl_cfg.obtain(\"datalad.repo.version\")\n    info = AnnexRepo.check_repository_versions()\n\n    @skip_if(version >= 6 or 5 not in info[\"supported\"],\n             msg=\"Skip test in v6+ test run\", method=method)\n    @wraps(func)\n    @attr('skip_v6_or_later')\n    @attr('v6_or_later')\n    def  _wrap_skip_v6_or_later(*args, **kwargs):\n        return func(*args, **kwargs)\n    return  _wrap_skip_v6_or_later\n\n\n#\n# Addition \"checkers\"\n#\n\nimport os\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.annexrepo import AnnexRepo, FileNotInAnnexError\nfrom datalad.distribution.dataset import Dataset\nfrom ..utils import chpwd, getpwd\n\n\ndef ok_clean_git(path, annex=None, index_modified=[], untracked=[]):\n    \"\"\"Obsolete test helper. Use assert_repo_status() instead.\n\n    Still maps a few common cases to the new helper, to ease transition\n    in extensions.\n    \"\"\"\n    kwargs = {}\n    if index_modified:\n        kwargs['modified'] = index_modified\n    if untracked:\n        kwargs['untracked'] = untracked\n    assert_repo_status(\n        path,\n        annex=annex,\n        **kwargs,\n    )\n\n\ndef ok_file_under_git(path, filename=None, annexed=False):\n    \"\"\"Test if file is present and under git/annex control\n\n    If relative path provided, then test from current directory\n    \"\"\"\n    annex, file_repo_path, filename, path, repo = _prep_file_under_git(path, filename)\n    assert_in(file_repo_path, repo.get_indexed_files())  # file is known to Git\n\n    if annex:\n        try:\n            # operates on relative to curdir path\n            repo.get_file_key(file_repo_path)\n            in_annex = True\n        except FileNotInAnnexError as e:\n            in_annex = False\n    else:\n        in_annex = False\n\n    assert(annexed == in_annex)\n\n\ndef put_file_under_git(path, filename=None, content=None, annexed=False):\n    \"\"\"Place file under git/annex and return used Repo\n    \"\"\"\n    annex, file_repo_path, filename, path, repo = _prep_file_under_git(path, filename)\n    if content is None:\n        content = \"\"\n    with open(opj(repo.path, file_repo_path), 'w') as f_:\n        f_.write(content)\n\n    if annexed:\n        if not isinstance(repo, AnnexRepo):\n            repo = AnnexRepo(repo.path)\n        repo.add(file_repo_path)\n    else:\n        repo.add(file_repo_path, git=True)\n    repo.commit(_datalad_msg=True)\n    ok_file_under_git(repo.path, file_repo_path, annexed)\n    return repo\n\n\ndef _prep_file_under_git(path, filename):\n    \"\"\"Get instance of the repository for the given filename\n\n    Helper to be used by few functions\n    \"\"\"\n    if filename is None:\n        # path provides the path and the name\n        path, filename = pathsplit(path)\n    try:\n        # if succeeds when must not (not `annexed`) -- fail\n        repo = get_repo_instance(path, class_=AnnexRepo)\n        annex = True\n    except RuntimeError as e:\n        # TODO: make a dedicated Exception\n        if \"No annex repository found in\" in str(e):\n            repo = get_repo_instance(path, class_=GitRepo)\n            annex = False\n        else:\n            raise\n\n    # path to the file within the repository\n    # repo.path is a \"realpath\" so to get relpath working correctly\n    # we need to realpath our path as well\n    # do absolute() in addition to always get an absolute path\n    # even with non-existing paths on windows\n    path = str(Path(path).resolve().absolute())  # intentional realpath to match GitRepo behavior\n    file_repo_dir = op.relpath(path, repo.path)\n    file_repo_path = filename if file_repo_dir == curdir else opj(file_repo_dir, filename)\n    return annex, file_repo_path, filename, path, repo\n\n\n#\n# Helpers to test symlinks\n#\n\ndef ok_symlink(path):\n    \"\"\"Checks whether path is either a working or broken symlink\"\"\"\n    link_path = os.path.islink(path)\n    if not link_path:\n        raise AssertionError(\"Path {} seems not to be a symlink\".format(path))\n\n\ndef ok_good_symlink(path):\n    ok_symlink(path)\n    rpath = Path(path).resolve()\n    ok_(rpath.exists(),\n        msg=\"Path {} seems to be missing.  Symlink {} is broken\".format(\n                rpath, path))\n\n\ndef ok_broken_symlink(path):\n    ok_symlink(path)\n    rpath = Path(path).resolve()\n    assert_false(rpath.exists(),\n            msg=\"Path {} seems to be present.  Symlink {} is not broken\".format(\n                    rpath, path))\n\n\ndef ok_startswith(s, prefix):\n    ok_(s.startswith(prefix),\n        msg=\"String %r doesn't start with %r\" % (s, prefix))\n\n\ndef ok_endswith(s, suffix):\n    ok_(s.endswith(suffix),\n        msg=\"String %r doesn't end with %r\" % (s, suffix))\n\n\ndef nok_startswith(s, prefix):\n    assert_false(s.startswith(prefix),\n        msg=\"String %r starts with %r\" % (s, prefix))\n\n\ndef ok_git_config_not_empty(ar):\n    \"\"\"Helper to verify that nothing rewritten the config file\"\"\"\n    # TODO: we don't support bare -- do we?\n    assert_true(os.stat(opj(ar.path, '.git', 'config')).st_size)\n\n\ndef ok_annex_get(ar, files, network=True):\n    \"\"\"Helper to run .get decorated checking for correct operation\n\n    get passes through stderr from the ar to the user, which pollutes\n    screen while running tests\n\n    Note: Currently not true anymore, since usage of --json disables\n    progressbars\n    \"\"\"\n    ok_git_config_not_empty(ar) # we should be working in already inited repo etc\n    with swallow_outputs() as cmo:\n        ar.get(files)\n    # verify that load was fetched\n    ok_git_config_not_empty(ar) # whatever we do shouldn't destroy the config file\n    has_content = ar.file_has_content(files)\n    if isinstance(has_content, bool):\n        ok_(has_content)\n    else:\n        ok_(all(has_content))\n\n\ndef ok_generator(gen):\n    assert_true(inspect.isgenerator(gen), msg=\"%s is not a generator\" % gen)\n\n\nassert_is_generator = ok_generator  # just an alias\n\n\ndef ok_archives_caches(repopath, n=1, persistent=None):\n    \"\"\"Given a path to repository verify number of archives\n\n    Parameters\n    ----------\n    repopath : str\n      Path to the repository\n    n : int, optional\n      Number of archives directories to expect\n    persistent: bool or None, optional\n      If None -- both persistent and not count.\n    \"\"\"\n    # looking into subdirectories\n    glob_ptn = opj(repopath,\n                   ARCHIVES_TEMP_DIR + {None: '*', True: '', False: '-*'}[persistent],\n                   '*')\n    dirs = glob.glob(glob_ptn)\n    n2 = n * 2  # per each directory we should have a .stamp file\n    assert_equal(len(dirs), n2,\n                 msg=\"Found following dirs when needed %d of them: %s\" % (n2, dirs))\n\n\ndef ok_exists(path):\n    assert Path(path).exists(), 'path %s does not exist (or dangling symlink)' % path\n\n\ndef ok_file_has_content(path, content, strip=False, re_=False,\n                        decompress=False, **kwargs):\n    \"\"\"Verify that file exists and has expected content\"\"\"\n    path = Path(path)\n    ok_exists(path)\n    if decompress:\n        if path.suffix == '.gz':\n            open_func = gzip.open\n        else:\n            raise NotImplementedError(\"Don't know how to decompress %s\" % path)\n    else:\n        open_func = open\n\n    with open_func(str(path), 'rb') as f:\n        file_content = f.read()\n\n    if isinstance(content, str):\n        file_content = ensure_unicode(file_content)\n\n    if os.linesep != '\\n':\n        # for consistent comparisons etc. Apparently when reading in `b` mode\n        # on Windows we would also get \\r\n        # https://github.com/datalad/datalad/pull/3049#issuecomment-444128715\n        file_content = file_content.replace(os.linesep, '\\n')\n\n    if strip:\n        file_content = file_content.strip()\n\n    if re_:\n        assert_re_in(content, file_content, **kwargs)\n    else:\n        assert_equal(content, file_content, **kwargs)\n\n\n#\n# Decorators\n#\n\n\n@optional_args\ndef with_tree(t, tree=None, archives_leading_dir=True, delete=True, **tkwargs):\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n            return t(*(arg + (d,)), **kw)\n        finally:\n            if delete:\n                rmtemp(d)\n    return  _wrap_with_tree\n\n\nlgr = logging.getLogger('datalad.tests')\n\n\nclass SilentHTTPHandler(SimpleHTTPRequestHandler):\n    \"\"\"A little adapter to silence the handler\n    \"\"\"\n    def __init__(self, *args, **kwargs):\n        self._silent = lgr.getEffectiveLevel() > logging.DEBUG\n        SimpleHTTPRequestHandler.__init__(self, *args, **kwargs)\n\n    def log_message(self, format, *args):\n        if self._silent:\n            return\n        lgr.debug(\"HTTP: \" + format % args)\n\n\ndef _multiproc_serve_path_via_http(hostname, path_to_serve_from, queue): # pragma: no cover\n    chpwd(path_to_serve_from)\n    httpd = HTTPServer((hostname, 0), SilentHTTPHandler)\n    queue.put(httpd.server_port)\n    httpd.serve_forever()\n\n\nclass HTTPPath(object):\n    \"\"\"Serve the content of a path via an HTTP URL.\n\n    This class can be used as a context manager, in which case it returns the\n    URL.\n\n    Alternatively, the `start` and `stop` methods can be called directly.\n\n    Parameters\n    ----------\n    path : str\n        Directory with content to serve.\n    \"\"\"\n    def __init__(self, path):\n        self.path = path\n        self.url = None\n        self._env_patch = None\n        self._mproc = None\n\n    def __enter__(self):\n        self.start()\n        return self.url\n\n    def __exit__(self, *args):\n        self.stop()\n\n    def start(self):\n        \"\"\"Start serving `path` via HTTP.\n        \"\"\"\n        # There is a problem with Haskell on wheezy trying to\n        # fetch via IPv6 whenever there is a ::1 localhost entry in\n        # /etc/hosts.  Apparently fixing that docker image reliably\n        # is not that straightforward, although see\n        # http://jasonincode.com/customizing-hosts-file-in-docker/\n        # so we just force to use 127.0.0.1 while on wheezy\n        #hostname = '127.0.0.1' if on_debian_wheezy else 'localhost'\n        hostname = '127.0.0.1'\n\n        queue = multiprocessing.Queue()\n        self._mproc = multiprocessing.Process(\n            target=_multiproc_serve_path_via_http,\n            args=(hostname, self.path, queue))\n        self._mproc.start()\n        port = queue.get(timeout=300)\n        self.url = 'http://{}:{}/'.format(hostname, port)\n        lgr.debug(\"HTTP: serving %s under %s\", self.path, self.url)\n\n        # Such tests don't require real network so if http_proxy settings were\n        # provided, we remove them from the env for the duration of this run\n        env = os.environ.copy()\n        env.pop('http_proxy', None)\n        self._env_patch = patch.dict('os.environ', env, clear=True)\n        self._env_patch.start()\n\n    def stop(self):\n        \"\"\"Stop serving `path`.\n        \"\"\"\n        lgr.debug(\"HTTP: stopping server under %s\", self.path)\n        self._env_patch.stop()\n        self._mproc.terminate()\n\n\n@optional_args\ndef serve_path_via_http(tfunc, *targs):\n    \"\"\"Decorator which serves content of a directory via http url\n    \"\"\"\n\n    @wraps(tfunc)\n    @attr('serve_path_via_http')\n    def  _wrap_serve_path_via_http(*args, **kwargs):\n\n        if targs:\n            # if a path is passed into serve_path_via_http, then it's in targs\n            assert len(targs) == 1\n            path = targs[0]\n\n        elif len(args) > 1:\n            args, path = args[:-1], args[-1]\n        else:\n            args, path = (), args[0]\n\n        with HTTPPath(path) as url:\n            return tfunc(*(args + (path, url)), **kwargs)\n    return  _wrap_serve_path_via_http\n\n\n@optional_args\ndef with_memory_keyring(t):\n    \"\"\"Decorator to use non-persistant MemoryKeyring instance\n    \"\"\"\n    @wraps(t)\n    @attr('with_memory_keyring')\n    def  _wrap_with_memory_keyring(*args, **kwargs):\n        keyring = MemoryKeyring()\n        with patch(\"datalad.downloaders.credentials.keyring_\", keyring):\n            return t(*(args + (keyring,)), **kwargs)\n\n    return  _wrap_with_memory_keyring\n\n\n@optional_args\ndef without_http_proxy(tfunc):\n    \"\"\"Decorator to remove http*_proxy env variables for the duration of the test\n    \"\"\"\n\n    @wraps(tfunc)\n    @attr('without_http_proxy')\n    def  _wrap_without_http_proxy(*args, **kwargs):\n        # Such tests don't require real network so if http_proxy settings were\n        # provided, we remove them from the env for the duration of this run\n        env = os.environ.copy()\n        env.pop('http_proxy', None)\n        env.pop('https_proxy', None)\n        with patch.dict('os.environ', env, clear=True):\n            return tfunc(*args, **kwargs)\n\n    return  _wrap_without_http_proxy\n\n\n@borrowkwargs(methodname=make_tempfile)\n@optional_args\ndef with_tempfile(t, **tkwargs):\n    \"\"\"Decorator function to provide a temporary file name and remove it at the end\n\n    Parameters\n    ----------\n\n    To change the used directory without providing keyword argument 'dir' set\n    DATALAD_TESTS_TEMP_DIR.\n\n    Examples\n    --------\n\n    ::\n\n        @with_tempfile\n        def test_write(tfile):\n            open(tfile, 'w').write('silly test')\n    \"\"\"\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n            return t(*(arg + (filename,)), **kw)\n\n    return  _wrap_with_tempfile\n\n\n# ### ###\n# START known failure decorators\n# ### ###\n\ndef probe_known_failure(func):\n    \"\"\"Test decorator allowing the test to pass when it fails and vice versa\n\n    Setting config datalad.tests.knownfailures.probe to True tests, whether or\n    not the test is still failing. If it's not, an AssertionError is raised in\n    order to indicate that the reason for failure seems to be gone.\n    \"\"\"\n\n    @wraps(func)\n    @attr('probe_known_failure')\n    def  _wrap_probe_known_failure(*args, **kwargs):\n        if dl_cfg.obtain(\"datalad.tests.knownfailures.probe\"):\n            assert_raises(Exception, func, *args, **kwargs)  # marked as known failure\n            # Note: Since assert_raises lacks a `msg` argument, a comment\n            # in the same line is helpful to determine what's going on whenever\n            # this assertion fails and we see a trace back. Otherwise that line\n            # wouldn't be very telling.\n        else:\n            return func(*args, **kwargs)\n    return  _wrap_probe_known_failure\n\n\n@optional_args\ndef skip_known_failure(func, method='raise'):\n    \"\"\"Test decorator allowing to skip a test that is known to fail\n\n    Setting config datalad.tests.knownfailures.skip to a bool enables/disables\n    skipping.\n    \"\"\"\n\n    @skip_if(cond=dl_cfg.obtain(\"datalad.tests.knownfailures.skip\"),\n             msg=\"Skip test known to fail\",\n             method=method)\n    @wraps(func)\n    @attr('skip_known_failure')\n    def  _wrap_skip_known_failure(*args, **kwargs):\n        return func(*args, **kwargs)\n    return  _wrap_skip_known_failure\n\n\ndef known_failure(func):\n    \"\"\"Test decorator marking a test as known to fail\n\n    This combines `probe_known_failure` and `skip_known_failure` giving the\n    skipping precedence over the probing.\n    \"\"\"\n\n    @skip_known_failure\n    @probe_known_failure\n    @wraps(func)\n    @attr('known_failure')\n    def  _wrap_known_failure(*args, **kwargs):\n        return func(*args, **kwargs)\n    return  _wrap_known_failure\n\n\ndef known_failure_v6_or_later(func):\n    \"\"\"Test decorator marking a test as known to fail in a v6+ test run\n\n    If the default repository version is 6 or later behaves like `known_failure`.\n    Otherwise the original (undecorated) function is returned.\n    The default repository version is controlled by the configured value of\n    DATALAD_REPO_VERSION and whether v5 repositories are supported by the\n    installed git-annex.\n    \"\"\"\n\n    from datalad.support.annexrepo import AnnexRepo\n\n    version = dl_cfg.obtain(\"datalad.repo.version\")\n    info = AnnexRepo.check_repository_versions()\n\n    if (version and version >= 6) or 5 not in info[\"supported\"]:\n\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_v6_or_later')\n        @attr('v6_or_later')\n        def v6_func(*args, **kwargs):\n            return func(*args, **kwargs)\n\n        return v6_func\n\n    return func\n\n\n# TODO: Remove once the released version of datalad-crawler no longer uses it.\nknown_failure_v6 = known_failure_v6_or_later\n\n\ndef known_failure_direct_mode(func):\n    \"\"\"DEPRECATED.  Stop using.  Does nothing\n\n    Test decorator marking a test as known to fail in a direct mode test run\n\n    If datalad.repo.direct is set to True behaves like `known_failure`.\n    Otherwise the original (undecorated) function is returned.\n    \"\"\"\n    # TODO: consider adopting   nibabel/deprecated.py  nibabel/deprecator.py\n    # mechanism to consistently deprecate functionality and ensure they are\n    # displayed.\n    # Since 2.7 Deprecation warnings aren't displayed by default\n    # and thus kinda pointless to issue a warning here, so we will just log\n    msg = \"Direct mode support is deprecated, so no point in using \" \\\n          \"@known_failure_direct_mode for %r since glorious future \" \\\n          \"DataLad 0.12\" % func.__name__\n    lgr.warning(msg)\n    return func\n\n\ndef known_failure_windows(func):\n    \"\"\"Test decorator marking a test as known to fail on windows\n\n    On Windows behaves like `known_failure`.\n    Otherwise the original (undecorated) function is returned.\n    \"\"\"\n    if on_windows:\n\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_windows')\n        @attr('windows')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n\n        return dm_func\n    return func\n\n\ndef known_failure_appveyor(func):\n    \"\"\"Test decorator marking a test as known to fail on AppVeyor.\n    \"\"\"\n    if 'APPVEYOR' in os.environ:\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_appveyor')\n        @attr('appveyor')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n        return dm_func\n    return func\n\n\ndef known_failure_githubci_win(func):\n    \"\"\"Test decorator for a known test failure on Github's Windows CI\n    \"\"\"\n    if 'GITHUB_WORKFLOW' in os.environ and on_windows:\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_githubci_win')\n        @attr('githubci_win')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n        return dm_func\n    return func\n\n\ndef known_failure_githubci_osx(func):\n    \"\"\"Test decorator for a known test failure on Github's macOS CI\n    \"\"\"\n    if 'GITHUB_WORKFLOW' in os.environ and on_osx:\n        @known_failure\n        @wraps(func)\n        @attr('known_failure_githubci_osx')\n        @attr('githubci_osx')\n        def dm_func(*args, **kwargs):\n            return func(*args, **kwargs)\n        return dm_func\n    return func\n\n\n# ### ###\n# END known failure decorators\n# ### ###\n\n\ndef _get_resolved_flavors(flavors):\n    #flavors_ = (['local', 'clone'] + (['local-url'] if not on_windows else [])) \\\n    #           if flavors == 'auto' else flavors\n    flavors_ = (['local', 'clone', 'local-url', 'network'] if not on_windows\n                else ['network', 'network-clone']) \\\n               if flavors == 'auto' else flavors\n\n    if not isinstance(flavors_, list):\n        flavors_ = [flavors_]\n\n    if dl_cfg.get('datalad.tests.nonetwork'):\n        flavors_ = [x for x in flavors_ if not x.startswith('network')]\n    return flavors_\n\n\ndef clone_url(url):\n    # delay import of our code until needed for certain\n    from ..cmd import Runner\n    runner = Runner()\n    tdir = tempfile.mkdtemp(**get_tempfile_kwargs(\n        {'dir': dl_cfg.get(\"datalad.tests.temp.dir\")}, prefix='clone_url'))\n    _ = runner([\"git\", \"clone\", url, tdir], expect_stderr=True)\n    if GitRepo(tdir).is_with_annex():\n        AnnexRepo(tdir, init=True)\n    _TEMP_PATHS_CLONES.add(tdir)\n    return tdir\n\n\nif not on_windows:\n    local_testrepo_flavors = ['local'] # 'local-url'\nelse:\n    local_testrepo_flavors = ['network-clone']\n\n_TESTREPOS = None\n\ndef _get_testrepos_uris(regex, flavors):\n    global _TESTREPOS\n    # we should instantiate those whenever test repos actually asked for\n    # TODO: just absorb all this lazy construction within some class\n    if not _TESTREPOS:\n        from .utils_testrepos import BasicAnnexTestRepo, BasicGitTestRepo, \\\n            SubmoduleDataset, NestedDataset, InnerSubmodule\n\n        _basic_annex_test_repo = BasicAnnexTestRepo()\n        _basic_git_test_repo = BasicGitTestRepo()\n        _submodule_annex_test_repo = SubmoduleDataset()\n        _nested_submodule_annex_test_repo = NestedDataset()\n        _inner_submodule_annex_test_repo = InnerSubmodule()\n        _TESTREPOS = {'basic_annex':\n                        {'network': 'git://github.com/datalad/testrepo--basic--r1',\n                         'local': _basic_annex_test_repo.path,\n                         'local-url': _basic_annex_test_repo.url},\n                      'basic_git':\n                        {'local': _basic_git_test_repo.path,\n                         'local-url': _basic_git_test_repo.url},\n                      'submodule_annex':\n                        {'local': _submodule_annex_test_repo.path,\n                         'local-url': _submodule_annex_test_repo.url},\n                      'nested_submodule_annex':\n                        {'local': _nested_submodule_annex_test_repo.path,\n                         'local-url': _nested_submodule_annex_test_repo.url},\n                      # TODO: append 'annex' to the name:\n                      # Currently doesn't work with some annex tests, despite\n                      # working manually. So, figure out how the tests' setup\n                      # messes things up with this one.\n                      'inner_submodule':\n                        {'local': _inner_submodule_annex_test_repo.path,\n                         'local-url': _inner_submodule_annex_test_repo.url}\n                      }\n        # assure that now we do have those test repos created -- delayed\n        # their creation until actually used\n        _basic_annex_test_repo.create()\n        _basic_git_test_repo.create()\n        _submodule_annex_test_repo.create()\n        _nested_submodule_annex_test_repo.create()\n        _inner_submodule_annex_test_repo.create()\n    uris = []\n    for name, spec in _TESTREPOS.items():\n        if not re.match(regex, name):\n            continue\n        uris += [spec[x] for x in set(spec.keys()).intersection(flavors)]\n\n        # additional flavors which might have not been\n        if 'clone' in flavors and 'clone' not in spec:\n            uris.append(clone_url(spec['local']))\n\n        if 'network-clone' in flavors \\\n                and 'network' in spec \\\n                and 'network-clone' not in spec:\n            uris.append(clone_url(spec['network']))\n\n    return uris\n\n\n# addurls with our generated file:// URLs doesn't work on appveyor\n# https://ci.appveyor.com/project/mih/datalad/builds/29841505/job/330rwn2a3cvtrakj\n@known_failure_appveyor\n@optional_args\ndef with_testrepos(t, regex='.*', flavors='auto', skip=False, count=None):\n    \"\"\"Decorator to provide a local/remote test repository\n\n    All tests under datalad/tests/testrepos are stored in two-level hierarchy,\n    where top-level name describes nature/identifier of the test repository,\n    and there could be multiple instances (e.g. generated differently) of the\n    same \"content\"\n\n    Parameters\n    ----------\n    regex : string, optional\n      Regex to select which test repos to use\n    flavors : {'auto', 'local', 'local-url', 'clone', 'network', 'network-clone'} or list of thereof, optional\n      What URIs to provide.  E.g. 'local' would just provide path to the\n      repository, while 'network' would provide url of the remote location\n      available on Internet containing the test repository.  'clone' would\n      clone repository first to a temporary location. 'network-clone' would\n      first clone from the network location. 'auto' would include the list of\n      appropriate ones (e.g., no 'network*' flavors if network tests are\n      \"forbidden\").\n    count: int, optional\n      If specified, only up to that number of repositories to test with\n\n    Examples\n    --------\n\n    >>> from datalad.tests.utils import with_testrepos\n    >>> @with_testrepos('basic_annex')\n    ... def test_write(repo):\n    ...    assert(os.path.exists(os.path.join(repo, '.git', 'annex')))\n\n    \"\"\"\n    @wraps(t)\n    @attr('with_testrepos')\n    def  _wrap_with_testrepos(*arg, **kw):\n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n\n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (dl_cfg.get('datalad.tests.nonetwork') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n\n        fake_dates = dl_cfg.get(\"datalad.fake-dates\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n                t(*(arg + (uri,)), **kw)\n            finally:\n                # The is_explicit_path check is needed because it may be a URL,\n                # but check_dates needs a local path or GitRepo object.\n                if fake_dates and is_explicit_path(uri):\n                    from ..support.repodates import check_dates\n                    assert_false(\n                        check_dates(uri, annex=\"tree\")[\"objects\"])\n                if uri in _TEMP_PATHS_CLONES:\n                    _TEMP_PATHS_CLONES.discard(uri)\n                    rmtemp(uri)\n                pass  # might need to provide additional handling so, handle\n    return  _wrap_with_testrepos\nwith_testrepos.__test__ = False\n\n\n@optional_args\ndef with_sameas_remote(func, autoenabled=False):\n    \"\"\"Provide a repository with a git-annex sameas remote configured.\n\n    The repository will have two special remotes: r_dir (type=directory) and\n    r_rsync (type=rsync). The rsync remote will be configured with\n    --sameas=r_dir, and autoenabled if `autoenabled` is true.\n    \"\"\"\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import CommandError\n\n    @wraps(func)\n    @attr('with_sameas_remotes')\n    @skip_if_on_windows\n    @skip_ssh\n    @with_tempfile(mkdir=True)\n    @with_tempfile(mkdir=True)\n    def  _wrap_with_sameas_remote(*args, **kwargs):\n        # With git-annex's 8.20200522-77-g1f2e2d15e, transferring from an rsync\n        # special remote hangs on Xenial. This is likely due to an interaction\n        # with an older rsync or openssh version. Use openssh as a rough\n        # indicator. See\n        # https://git-annex.branchable.com/bugs/Recent_hang_with_rsync_remote_with_older_systems___40__Xenial__44___Jessie__41__/\n        if external_versions['cmd:system-ssh'] < '7.4' and \\\n           '8.20200522' < external_versions['cmd:annex'] < '8.20200720':\n            raise SkipTest(\"Test known to hang\")\n\n        sr_path, repo_path = args[-2:]\n        fn_args = args[:-2]\n        repo = AnnexRepo(repo_path)\n        repo.init_remote(\"r_dir\",\n                         options=[\"type=directory\",\n                                  \"encryption=none\",\n                                  \"directory=\" + sr_path])\n        options = [\"type=rsync\",\n                   \"rsyncurl=datalad-test:\" + sr_path]\n        if autoenabled:\n            options.append(\"autoenable=true\")\n        options.append(\"--sameas=r_dir\")\n\n        try:\n            repo.init_remote(\"r_rsync\", options=options)\n        except CommandError:\n            if repo.git_annex_version < \"7.20191017\":\n                raise SkipTest(\"git-annex lacks --sameas support\")\n            # This should have --sameas support.\n            raise\n        return func(*(fn_args + (repo,)), **kwargs)\n    return  _wrap_with_sameas_remote\n\n\n@optional_args\ndef with_fake_cookies_db(func, cookies={}):\n    \"\"\"mock original cookies db with a fake one for the duration of the test\n    \"\"\"\n    from ..support.cookies import cookies_db\n\n    @wraps(func)\n    @attr('with_fake_cookies_db')\n    def  _wrap_with_fake_cookies_db(*args, **kwargs):\n        try:\n            orig_cookies_db = cookies_db._cookies_db\n            cookies_db._cookies_db = cookies.copy()\n            return func(*args, **kwargs)\n        finally:\n            cookies_db._cookies_db = orig_cookies_db\n    return  _wrap_with_fake_cookies_db\n\n\n@optional_args\ndef assert_cwd_unchanged(func, ok_to_chdir=False):\n    \"\"\"Decorator to test whether the current working directory remains unchanged\n\n    Parameters\n    ----------\n    ok_to_chdir: bool, optional\n      If True, allow to chdir, so this decorator would not then raise exception\n      if chdir'ed but only return to original directory\n    \"\"\"\n\n    @wraps(func)\n    def  _wrap_assert_cwd_unchanged(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n\n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n\n        if exc_info is not None:\n            raise exc_info[1]\n\n        return ret\n\n    return  _wrap_assert_cwd_unchanged\n\n\n@optional_args\ndef run_under_dir(func, newdir='.'):\n    \"\"\"Decorator to run tests under another directory\n\n    It is somewhat ugly since we can't really chdir\n    back to a directory which had a symlink in its path.\n    So using this decorator has potential to move entire\n    testing run under the dereferenced directory name -- sideeffect.\n\n    The only way would be to instruct testing framework (i.e. nose\n    in our case ATM) to run a test by creating a new process with\n    a new cwd\n    \"\"\"\n\n    @wraps(func)\n    def  _wrap_run_under_dir(*args, **kwargs):\n        pwd_before = getpwd()\n        try:\n            chpwd(newdir)\n            func(*args, **kwargs)\n        finally:\n            chpwd(pwd_before)\n\n\n    return  _wrap_run_under_dir\n\n\ndef assert_re_in(regex, c, flags=0, match=True, msg=None):\n    \"\"\"Assert that container (list, str, etc) contains entry matching the regex\n    \"\"\"\n    if not isinstance(c, (list, tuple)):\n        c = [c]\n    for e in c:\n        if (re.match if match else re.search)(regex, e, flags=flags):\n            return\n    raise AssertionError(\n        msg or \"Not a single entry matched %r in %r\" % (regex, c)\n    )\n\n\ndef assert_dict_equal(d1, d2):\n    msgs = []\n    if set(d1).difference(d2):\n        msgs.append(\" keys in the first dict but not in the second: %s\"\n                    % list(set(d1).difference(d2)))\n    if set(d2).difference(d1):\n        msgs.append(\" keys in the second dict but not in the first: %s\"\n                    % list(set(d2).difference(d1)))\n    for k in set(d1).intersection(d2):\n        same = True\n        try:\n            if isinstance(d1[k], str):\n                # do not compare types for string types to avoid all the hassle\n                # with the distinction of str and unicode in PY3, and simple\n                # test for equality\n                same = bool(d1[k] == d2[k])\n            else:\n                same = type(d1[k]) == type(d2[k]) and bool(d1[k] == d2[k])\n        except:  # if comparison or conversion to bool (e.g. with numpy arrays) fails\n            same = False\n\n        if not same:\n            msgs.append(\" [%r] differs: %r != %r\" % (k, d1[k], d2[k]))\n\n        if len(msgs) > 10:\n            msgs.append(\"and more\")\n            break\n    if msgs:\n        raise AssertionError(\"dicts differ:\\n%s\" % \"\\n\".join(msgs))\n    # do generic comparison just in case we screwed up to detect difference correctly above\n    eq_(d1, d2)\n\n\ndef assert_str_equal(s1, s2):\n    \"\"\"Helper to compare two lines\"\"\"\n    diff = list(unified_diff(s1.splitlines(), s2.splitlines()))\n    assert not diff, '\\n'.join(diff)\n    assert_equal(s1, s2)\n\n\ndef assert_status(label, results):\n    \"\"\"Verify that each status dict in the results has a given status label\n\n    `label` can be a sequence, in which case status must be one of the items\n    in this sequence.\n    \"\"\"\n    label = assure_list(label)\n    results = assure_list(results)\n    for i, r in enumerate(results):\n        try:\n            assert_in('status', r)\n            assert_in(r['status'], label)\n        except AssertionError:\n            raise AssertionError('Test {}/{}: expected status {} not found in:\\n{}'.format(\n                i + 1,\n                len(results),\n                label,\n                dumps(results, indent=1, default=lambda x: str(x))))\n\n\ndef assert_message(message, results):\n    \"\"\"Verify that each status dict in the results has a message\n\n    This only tests the message template string, and not a formatted message\n    with args expanded.\n    \"\"\"\n    for r in assure_list(results):\n        assert_in('message', r)\n        m = r['message'][0] if isinstance(r['message'], tuple) else r['message']\n        assert_equal(m, message)\n\n\ndef _format_res(x):\n    return textwrap.indent(\n        dumps(x, indent=1, default=str, sort_keys=True),\n        prefix=\"  \")\n\n\ndef assert_result_count(results, n, **kwargs):\n    \"\"\"Verify specific number of results (matching criteria, if any)\"\"\"\n    count = 0\n    results = assure_list(results)\n    for r in results:\n        if not len(kwargs):\n            count += 1\n        elif all(k in r and r[k] == v for k, v in kwargs.items()):\n            count += 1\n    if not n == count:\n        raise AssertionError(\n            'Got {} instead of {} expected results matching\\n{}\\nInspected {} record(s):\\n{}'.format(\n                count,\n                n,\n                _format_res(kwargs),\n                len(results),\n                _format_res(results)))\n\n\ndef assert_in_results(results, **kwargs):\n    \"\"\"Verify that the particular combination of keys and values is found in\n    one of the results\"\"\"\n    found = False\n    for r in assure_list(results):\n        if all(k in r and r[k] == v for k, v in kwargs.items()):\n            found = True\n    if not found:\n        raise AssertionError(\n            \"Desired result\\n{}\\nnot found among\\n{}\"\n            .format(_format_res(kwargs), _format_res(results)))\n\n\ndef assert_not_in_results(results, **kwargs):\n    \"\"\"Verify that the particular combination of keys and values is not in any\n    of the results\"\"\"\n    for r in assure_list(results):\n        assert any(k not in r or r[k] != v for k, v in kwargs.items())\n\n\ndef assert_result_values_equal(results, prop, values):\n    \"\"\"Verify that the values of all results for a given key in the status dicts\n    match the given sequence\"\"\"\n    assert_equal(\n        [r[prop] for r in results],\n        values)\n\n\ndef assert_result_values_cond(results, prop, cond):\n    \"\"\"Verify that the values of all results for a given key in the status dicts\n    fullfill condition `cond`.\n\n    Parameters\n    ----------\n    results:\n    prop: str\n    cond: callable\n    \"\"\"\n    for r in assure_list(results):\n        ok_(cond(r[prop]),\n            msg=\"r[{prop}]: {value}\".format(prop=prop, value=r[prop]))\n\n\ndef ignore_nose_capturing_stdout(func):\n    \"\"\"DEPRECATED and will be removed soon.  Does nothing!\n\n    Originally was intended as a decorator workaround for nose's behaviour\n    with redirecting sys.stdout, but now we monkey patch nose now so no test\n    should no longer be skipped.\n\n    See issue reported here:\n    https://code.google.com/p/python-nose/issues/detail?id=243&can=1&sort=-id&colspec=ID%20Type%20Status%20Priority%20Stars%20Milestone%20Owner%20Summary\n\n    \"\"\"\n    lgr.warning(\n        \"@ignore_nose_capturing_stdout no longer does anything - nose should \"\n        \"just be monkey patched in setup_package. {} still has it\"\n        .format(func.__name__)\n    )\n    return func\n\n\ndef skip_httpretty_on_problematic_pythons(func):\n    \"\"\"As discovered some httpretty bug causes a side-effect\n    on other tests on some Pythons.  So we skip the test if such\n    problematic combination detected\n\n    References\n    https://travis-ci.org/datalad/datalad/jobs/94464988\n    http://stackoverflow.com/a/29603206/1265472\n    \"\"\"\n\n    @make_decorator(func)\n    def  _wrap_skip_httpretty_on_problematic_pythons(*args, **kwargs):\n        if sys.version_info[:3] == (3, 4, 2):\n            raise SkipTest(\"Known to cause trouble due to httpretty bug on this Python\")\n        return func(*args, **kwargs)\n    return  _wrap_skip_httpretty_on_problematic_pythons\n\n\n@optional_args\ndef with_parametric_batch(t):\n    \"\"\"Helper to run parametric test with possible combinations of batch and direct\n    \"\"\"\n    @wraps(t)\n    def  _wrap_with_parametric_batch():\n        for batch in (False, True):\n                yield t, batch\n\n    return  _wrap_with_parametric_batch\n\n\n# List of most obscure filenames which might or not be supported by different\n# filesystems across different OSs.  Start with the most obscure\nOBSCURE_PREFIX = os.getenv('DATALAD_TESTS_OBSCURE_PREFIX', '')\n# Those will be tried to be added to the base name if filesystem allows\nOBSCURE_FILENAME_PARTS = ['/', '|', ';', '&',\n                          '%b5',\n                          ' ']\nUNICODE_FILENAME = u\"ΔЙקم๗あ\"\n\n# OSX is exciting -- some I guess FS might be encoding differently from decoding\n# so Й might get recoded\n# (ref: https://github.com/datalad/datalad/pull/1921#issuecomment-385809366)\nif sys.getfilesystemencoding().lower() == 'utf-8':\n    if on_osx:\n        # TODO: figure it really out\n        UNICODE_FILENAME = UNICODE_FILENAME.replace(u\"Й\", u\"\")\n    if on_windows:\n        # TODO: really figure out unicode handling on windows\n        UNICODE_FILENAME = ''\n    if UNICODE_FILENAME:\n        OBSCURE_FILENAME_PARTS.append(UNICODE_FILENAME)\n# simple extension to finish it up\nOBSCURE_FILENAME_PARTS.append('.datc')\n\n\n@with_tempfile(mkdir=True)\ndef get_most_obscure_supported_name(tdir, return_candidates=False):\n    \"\"\"Return the most obscure filename that the filesystem would support under TEMPDIR\n\n    Parameters\n    ----------\n    return_candidates: bool, optional\n      if True, return a tuple of (good, candidates) where candidates are \"partially\"\n      sorted from trickiest considered\n    TODO: we might want to use it as a function where we would provide tdir\n    \"\"\"\n    # we need separate good_base so we do not breed leading/trailing spaces\n    initial = good = 'a'  # everyone should support that!\n    system = platform.system()\n\n    OBSCURE_FILENAMES = []\n    def good_filename(filename):\n        OBSCURE_FILENAMES.append(candidate)\n        try:\n            with open(opj(tdir, filename), 'w') as f:\n                f.write(\"TEST LOAD\")\n            return True\n        except:\n            lgr.debug(\"Filename %r is not supported on %s under %s\",\n                      filename, system, tdir)\n            return False\n\n    # incrementally build up the most obscure filename from parts\n    for part in OBSCURE_FILENAME_PARTS:\n        candidate = good + part\n        if good_filename(OBSCURE_PREFIX + candidate):\n            good = candidate\n\n    # now we will compose some candidates with trailing spaces - trickier ones first\n    # so we break the loop as soon as we get it\n    for candidate in [' ' + good + ' ', ' ' + good, good + ' ', good]:\n        candidate = OBSCURE_PREFIX + candidate\n        if on_windows and candidate.rstrip() != candidate:\n            continue\n        if good_filename(candidate):\n            good = candidate\n            break\n\n    if good == initial:\n        raise RuntimeError(\"Could not create any of the files under %s among %s\"\n                       % (tdir, OBSCURE_FILENAMES))\n    lgr.debug(\"Tested %d obscure filename candidates. The winner: %r\", len(OBSCURE_FILENAMES), good)\n    if return_candidates:\n        return good, OBSCURE_FILENAMES[::-1]\n    else:\n        return good\n\n\nOBSCURE_FILENAME, OBSCURE_FILENAMES = get_most_obscure_supported_name(return_candidates=True)\n\n\n@optional_args\ndef with_testsui(t, responses=None, interactive=True):\n    \"\"\"Switch main UI to be 'tests' UI and possibly provide answers to be used\"\"\"\n\n    @wraps(t)\n    def  _wrap_with_testsui(*args, **kwargs):\n        from datalad.ui import ui\n        old_backend = ui.backend\n        try:\n            ui.set_backend('tests' if interactive else 'tests-noninteractive')\n            if responses:\n                ui.add_responses(responses)\n            ret = t(*args, **kwargs)\n            if responses:\n                responses_left = ui.get_responses()\n                assert not len(responses_left), \"Some responses were left not used: %s\" % str(responses_left)\n            return ret\n        finally:\n            ui.set_backend(old_backend)\n\n    if not interactive and responses is not None:\n        raise ValueError(\"Non-interactive UI cannot provide responses\")\n\n    return  _wrap_with_testsui\n\nwith_testsui.__test__ = False\n\n\ndef assert_no_errors_logged(func, skip_re=None):\n    \"\"\"Decorator around function to assert that no errors logged during its execution\"\"\"\n    @wraps(func)\n    def  _wrap_assert_no_errors_logged(*args, **kwargs):\n        with swallow_logs(new_level=logging.ERROR) as cml:\n            out = func(*args, **kwargs)\n            if cml.out:\n                if not (skip_re and re.search(skip_re, cml.out)):\n                    raise AssertionError(\n                        \"Expected no errors to be logged, but log output is %s\"\n                        % cml.out\n                    )\n        return out\n\n    return  _wrap_assert_no_errors_logged\n\n\ndef get_mtimes_and_digests(target_path):\n    \"\"\"Return digests (md5) and mtimes for all the files under target_path\"\"\"\n    from datalad.utils import find_files\n    from datalad.support.digests import Digester\n    digester = Digester(['md5'])\n\n    # bother only with existing ones for this test, i.e. skip annexed files without content\n    target_files = [\n        f for f in find_files('.*', topdir=target_path, exclude_vcs=False, exclude_datalad=False)\n        if exists(f)\n    ]\n    # let's leave only relative paths for easier analysis\n    target_files_ = [relpath(f, target_path) for f in target_files]\n\n    digests = {frel: digester(f) for f, frel in zip(target_files, target_files_)}\n    mtimes = {frel: os.stat(f).st_mtime for f, frel in zip(target_files, target_files_)}\n    return digests, mtimes\n\n\ndef get_datasets_topdir():\n    \"\"\"Delayed parsing so it could be monkey patched etc\"\"\"\n    from datalad.consts import DATASETS_TOPURL\n    return RI(DATASETS_TOPURL).hostname\n\n\ndef assert_repo_status(path, annex=None, untracked_mode='normal', **kwargs):\n    \"\"\"Compare a repo status against (optional) exceptions.\n\n    Anything file/directory that is not explicitly indicated must have\n    state 'clean', i.e. no modifications and recorded in Git.\n\n    Parameters\n    ----------\n    path: str or Repo\n      in case of a str: path to the repository's base dir;\n      Note, that passing a Repo instance prevents detecting annex. This might\n      be useful in case of a non-initialized annex, a GitRepo is pointing to.\n    annex: bool or None\n      explicitly set to True or False to indicate, that an annex is (not)\n      expected; set to None to autodetect, whether there is an annex.\n      Default: None.\n    untracked_mode: {'no', 'normal', 'all'}\n      If and how untracked content is reported. The specification of untracked\n      files that are OK to be found must match this mode. See `Repo.status()`\n    **kwargs\n      Files/directories that are OK to not be in 'clean' state. Each argument\n      must be one of 'added', 'untracked', 'deleted', 'modified' and each\n      value must be a list of filenames (relative to the root of the\n      repository, in POSIX convention).\n    \"\"\"\n    r = None\n    if isinstance(path, AnnexRepo):\n        if annex is None:\n            annex = True\n        # if `annex` was set to False, but we find an annex => fail\n        assert_is(annex, True)\n        r = path\n    elif isinstance(path, GitRepo):\n        if annex is None:\n            annex = False\n        # explicitly given GitRepo instance doesn't make sense with\n        # 'annex' True\n        assert_is(annex, False)\n        r = path\n    else:\n        # 'path' is an actual path\n        try:\n            r = AnnexRepo(path, init=False, create=False)\n            if annex is None:\n                annex = True\n            # if `annex` was set to False, but we find an annex => fail\n            assert_is(annex, True)\n        except Exception:\n            # Instantiation failed => no annex\n            try:\n                r = GitRepo(path, init=False, create=False)\n            except Exception:\n                raise AssertionError(\"Couldn't find an annex or a git \"\n                                     \"repository at {}.\".format(path))\n            if annex is None:\n                annex = False\n            # explicitly given GitRepo instance doesn't make sense with\n            # 'annex' True\n            assert_is(annex, False)\n\n    status = r.status(untracked=untracked_mode)\n    # for any file state that indicates some kind of change (all but 'clean)\n    for state in ('added', 'untracked', 'deleted', 'modified'):\n        oktobefound = sorted(r.pathobj.joinpath(ut.PurePosixPath(p))\n                             for p in kwargs.get(state, []))\n        state_files = sorted(k for k, v in status.items()\n                             if v.get('state', None) == state)\n        eq_(state_files, oktobefound,\n            'unexpected content of state \"%s\": %r != %r'\n            % (state, state_files, oktobefound))\n\n\ndef get_convoluted_situation(path, repocls=AnnexRepo):\n    from datalad.api import create\n\n    if 'APPVEYOR' in os.environ:\n        # issue only happens on appveyor, Python itself implodes\n        # cannot be reproduced on a real windows box\n        raise SkipTest(\n            'get_convoluted_situation() causes appveyor to crash, '\n            'reason unknown')\n    repo = repocls(path, create=True)\n    # use create(force) to get an ID and config into the empty repo\n    ds = Dataset(path).create(force=True)\n    # base content\n    create_tree(\n        ds.path,\n        {\n            '.gitignore': '*.ignored',\n            'subdir': {\n                'file_clean': 'file_clean',\n                'file_deleted': 'file_deleted',\n                'file_modified': 'file_clean',\n            },\n            'subdir-only-ignored': {\n                '1.ignored': '',\n            },\n            'file_clean': 'file_clean',\n            'file_deleted': 'file_deleted',\n            'file_staged_deleted': 'file_staged_deleted',\n            'file_modified': 'file_clean',\n        }\n    )\n    if isinstance(ds.repo, AnnexRepo):\n        create_tree(\n            ds.path,\n            {\n                'subdir': {\n                    'file_dropped_clean': 'file_dropped_clean',\n                },\n                'file_dropped_clean': 'file_dropped_clean',\n            }\n        )\n    ds.save()\n    if isinstance(ds.repo, AnnexRepo):\n        # some files straight in git\n        create_tree(\n            ds.path,\n            {\n                'subdir': {\n                    'file_ingit_clean': 'file_ingit_clean',\n                    'file_ingit_modified': 'file_ingit_clean',\n                },\n                'file_ingit_clean': 'file_ingit_clean',\n                'file_ingit_modified': 'file_ingit_clean',\n            }\n        )\n        ds.save(to_git=True)\n        ds.drop([\n            'file_dropped_clean',\n            op.join('subdir', 'file_dropped_clean')],\n            check=False)\n    # clean and proper subdatasets\n    ds.create('subds_clean')\n    ds.create(op.join('subdir', 'subds_clean'))\n    ds.create('subds_unavailable_clean')\n    ds.create(op.join('subdir', 'subds_unavailable_clean'))\n    # uninstall some subdatasets (still clean)\n    ds.uninstall([\n        'subds_unavailable_clean',\n        op.join('subdir', 'subds_unavailable_clean')],\n        check=False)\n    assert_repo_status(ds.path)\n    # make a dirty subdataset\n    ds.create('subds_modified')\n    ds.create(op.join('subds_modified', 'someds'))\n    ds.create(op.join('subds_modified', 'someds', 'dirtyds'))\n    # make a subdataset with additional commits\n    ds.create(op.join('subdir', 'subds_modified'))\n    pdspath = op.join(ds.path, 'subdir', 'subds_modified', 'progressedds')\n    ds.create(pdspath)\n    create_tree(\n        pdspath,\n        {'file_clean': 'file_ingit_clean'}\n    )\n    Dataset(pdspath).save()\n    assert_repo_status(pdspath)\n    # staged subds, and files\n    create(op.join(ds.path, 'subds_added'))\n    ds.repo.add_submodule('subds_added')\n    create(op.join(ds.path, 'subdir', 'subds_added'))\n    ds.repo.add_submodule(op.join('subdir', 'subds_added'))\n    # some more untracked files\n    create_tree(\n        ds.path,\n        {\n            'subdir': {\n                'file_untracked': 'file_untracked',\n                'file_added': 'file_added',\n            },\n            'file_untracked': 'file_untracked',\n            'file_added': 'file_added',\n            'dir_untracked': {\n                'file_untracked': 'file_untracked',\n            },\n            'subds_modified': {\n                'someds': {\n                    \"dirtyds\": {\n                        'file_untracked': 'file_untracked',\n                    },\n                },\n            },\n        }\n    )\n    ds.repo.add(['file_added', op.join('subdir', 'file_added')])\n    # untracked subdatasets\n    create(op.join(ds.path, 'subds_untracked'))\n    create(op.join(ds.path, 'subdir', 'subds_untracked'))\n    # deleted files\n    os.remove(op.join(ds.path, 'file_deleted'))\n    os.remove(op.join(ds.path, 'subdir', 'file_deleted'))\n    # staged deletion\n    ds.repo.remove('file_staged_deleted')\n    # modified files\n    if isinstance(ds.repo, AnnexRepo):\n        ds.repo.unlock(['file_modified', op.join('subdir', 'file_modified')])\n        create_tree(\n            ds.path,\n            {\n                'subdir': {\n                    'file_ingit_modified': 'file_ingit_modified',\n                },\n                'file_ingit_modified': 'file_ingit_modified',\n            }\n        )\n    create_tree(\n        ds.path,\n        {\n            'subdir': {\n                'file_modified': 'file_modified',\n            },\n            'file_modified': 'file_modified',\n        }\n    )\n    return ds\n\n\ndef get_deeply_nested_structure(path):\n    \"\"\" Here is what this does (assuming UNIX, locked):\n    |  .\n    |  ├── directory_untracked\n    |  │   └── link2dir -> ../subdir\n    |  ├── OBSCURE_FILENAME_file_modified\n    |  ├── link2dir -> subdir\n    |  ├── link2subdsdir -> subds_modified/subdir\n    |  ├── link2subdsroot -> subds_modified\n    |  ├── subdir\n    |  │   ├── annexed_file.txt -> ../.git/annex/objects/...\n    |  │   ├── file_modified\n    |  │   ├── git_file.txt\n    |  │   └── link2annex_files.txt -> annexed_file.txt\n    |  └── subds_modified\n    |      ├── link2superdsdir -> ../subdir\n    |      ├── subdir\n    |      │   └── annexed_file.txt -> ../.git/annex/objects/...\n    |      └── subds_lvl1_modified\n    |          └── OBSCURE_FILENAME_directory_untracked\n    |              └── untracked_file\n\n    When a system has no symlink support, the link2... components are not\n    included.\n    \"\"\"\n    ds = Dataset(path).create()\n    (ds.pathobj / 'subdir').mkdir()\n    (ds.pathobj / 'subdir' / 'annexed_file.txt').write_text(u'dummy')\n    ds.save()\n    (ds.pathobj / 'subdir' / 'git_file.txt').write_text(u'dummy')\n    ds.save(to_git=True)\n    # a subtree of datasets\n    subds = ds.create('subds_modified')\n    # another dataset, plus an additional dir in it\n    ds.create(op.join('subds_modified', 'subds_lvl1_modified'))\n    create_tree(\n        ds.path,\n        {\n            'subdir': {\n                'file_modified': 'file_modified',\n            },\n            OBSCURE_FILENAME + u'file_modified_': 'file_modified',\n        }\n    )\n    create_tree(\n        str(ds.pathobj / 'subds_modified' / 'subds_lvl1_modified'),\n        {OBSCURE_FILENAME + u'_directory_untracked': {\"untraced_file\": \"\"}}\n    )\n    (ut.Path(subds.path) / 'subdir').mkdir()\n    (ut.Path(subds.path) / 'subdir' / 'annexed_file.txt').write_text(u'dummy')\n    subds.save()\n    (ds.pathobj / 'directory_untracked').mkdir()\n\n    if not has_symlink_capability():\n        return ds\n\n    # symlink farm #1\n    # symlink to annexed file\n    (ds.pathobj / 'subdir' / 'link2annex_files.txt').symlink_to(\n        'annexed_file.txt')\n    # symlink to directory within the dataset\n    (ds.pathobj / 'link2dir').symlink_to('subdir')\n    # upwards pointing symlink to directory within the same dataset\n    (ds.pathobj / 'directory_untracked' / 'link2dir').symlink_to(\n        op.join('..', 'subdir'))\n    # symlink pointing to a subdataset mount in the same dataset\n    (ds.pathobj / 'link2subdsroot').symlink_to('subds_modified')\n    # symlink to a dir in a subdataset (across dataset boundaries)\n    (ds.pathobj / 'link2subdsdir').symlink_to(\n        op.join('subds_modified', 'subdir'))\n    # symlink to a dir in a superdataset (across dataset boundaries)\n    (ut.Path(subds.path) / 'link2superdsdir').symlink_to(\n        op.join('..', 'subdir'))\n    return ds\n\n\n@with_tempfile\n@with_tempfile\ndef has_symlink_capability(p1, p2):\n\n    path = ut.Path(p1)\n    target = ut.Path(p2)\n    return utils.check_symlink_capability(path, target)\n\n\ndef skip_wo_symlink_capability(func):\n    \"\"\"Skip test when environment does not support symlinks\n\n    Perform a behavioral test instead of top-down logic, as on\n    windows this could be on or off on a case-by-case basis.\n    \"\"\"\n    @wraps(func)\n    @attr('skip_wo_symlink_capability')\n    def  _wrap_skip_wo_symlink_capability(*args, **kwargs):\n        if not has_symlink_capability():\n            raise SkipTest(\"no symlink capabilities\")\n        return func(*args, **kwargs)\n    return  _wrap_skip_wo_symlink_capability\n\n\ndef get_ssh_port(host):\n    \"\"\"Get port of `host` in ssh_config.\n\n    Our tests depend on the host being defined in ssh_config, including its\n    port. This method can be used by tests that want to check handling of an\n    explicitly specified\n\n    Note that if `host` does not match a host in ssh_config, the default value\n    of 22 is returned.\n\n    Parameters\n    ----------\n    host : str\n\n    Returns\n    -------\n    port (int)\n\n    Raises\n    ------\n    SkipTest if port cannot be found.\n    \"\"\"\n    out = ''\n    try:\n        out, err = Runner()([\"ssh\", \"-G\", host])\n    except Exception as exc:\n        err = str(exc)\n\n    port = None\n    for line in out.splitlines():\n        if line.startswith(\"port \"):\n            try:\n                port = int(line.split()[1])\n            except Exception as exc:\n                err = str(exc)\n            break\n\n    if port is None:\n        raise SkipTest(\"port for {} could not be determined: {}\"\n                       .format(host, err))\n    return port\n\n\n#\n# Context Managers\n#\n\n\ndef patch_config(vars):\n    \"\"\"Patch our config with custom settings. Returns mock.patch cm\n    \"\"\"\n    return patch.dict(dl_cfg._store, vars)\n\n\n@contextmanager\ndef set_date(timestamp):\n    \"\"\"Temporarily override environment variables for git/git-annex dates.\n\n    Parameters\n    ----------\n    timestamp : int\n        Unix timestamp.\n    \"\"\"\n    git_ts = \"@{} +0000\".format(timestamp)\n    with patch.dict(\"os.environ\",\n                    {\"GIT_COMMITTER_DATE\": git_ts,\n                     \"GIT_AUTHOR_DATE\": git_ts,\n                     \"GIT_ANNEX_VECTOR_CLOCK\": str(timestamp),\n                     \"DATALAD_FAKE__DATES\": \"0\"}):\n        yield\n\n\n@contextmanager\ndef set_annex_version(version):\n    \"\"\"Override the git-annex version.\n\n    This temporarily masks the git-annex version present in external_versions\n    and make AnnexRepo forget its cached version information.\n    \"\"\"\n    from datalad.support.annexrepo import AnnexRepo\n    ar_vers = AnnexRepo.git_annex_version\n    with patch.dict(\n            \"datalad.support.annexrepo.external_versions._versions\",\n            {\"cmd:annex\": version}):\n        try:\n            AnnexRepo.git_annex_version = None\n            yield\n        finally:\n            AnnexRepo.git_annex_version = ar_vers\n\n#\n# Test tags\n#\n# To be explicit, and not \"loose\" some tests due to typos, decided to make\n# explicit decorators for common types\n\nfrom nose.plugins.attrib import attr\n\n\ndef integration(f):\n    \"\"\"Mark test as an \"integration\" test which generally is not needed to be run\n    \n    Generally tend to be slower.\n    Should be used in combination with @slow and @turtle if that is the case.\n    \"\"\"\n    return attr('integration')(f)\n\n\ndef slow(f):\n    \"\"\"Mark test as a slow, although not necessarily integration or usecase test\n\n    Rule of thumb cut-off to mark as slow is 10 sec\n    \"\"\"\n    return attr('slow')(f)\n\n\ndef turtle(f):\n    \"\"\"Mark test as very slow, meaning to not run it on Travis due to its\n    time limit\n\n    Rule of thumb cut-off to mark as turtle is 2 minutes\n    \"\"\"\n    return attr('turtle')(f)\n\n\ndef usecase(f):\n    \"\"\"Mark test as a usecase user ran into and which (typically) caused bug report\n    to be filed/troubleshooted\n\n    Should be used in combination with @slow and @turtle if slow.\n    \"\"\"\n    return attr('usecase')(f)\n",
            "header": {
                "file": {
                    "path": "datalad/tests/utils.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "d22e8d435",
                "new_commit_hash": "ec5cc2e94",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/tests/utils.py"
            },
            "plus_file": {
                "path": "b/datalad/tests/utils.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1440,
                            "length": 7
                        },
                        "new_range": {
                            "start": 1440,
                            "length": 7
                        },
                        "section": "def with_parametric_batch(t):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "OBSCURE_PREFIX = os.getenv('DATALAD_TESTS_OBSCURE_PREFIX', '')",
                                "type": "context"
                            },
                            {
                                "content": "# Those will be tried to be added to the base name if filesystem allows",
                                "type": "context"
                            },
                            {
                                "content": "OBSCURE_FILENAME_PARTS = ['/', '|', ';', '&',",
                                "type": "context"
                            },
                            {
                                "content": "                          # TODO: add '%b5',",
                                "type": "deleted"
                            },
                            {
                                "content": "                          '%b5',",
                                "type": "added"
                            },
                            {
                                "content": "                          ' ']",
                                "type": "context"
                            },
                            {
                                "content": "UNICODE_FILENAME = u\"ΔЙקم๗あ\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/tests/utils.py",
                            "type": "statement",
                            "name": "",
                            "content": "OBSCURE_FILENAME_PARTS = ['/', '|', ';', '&',\n                          '%b5',\n                          ' ']",
                            "ast_type_str": "Assign",
                            "start_lineno": 1442,
                            "end_lineno": 1444,
                            "parent": null
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "datalad/tests/utils.py",
                            "type": "statement",
                            "name": "",
                            "content": "OBSCURE_FILENAME_PARTS = ['/', '|', ';', '&',\n                          # TODO: add '%b5',\n                          ' ']",
                            "ast_type_str": "Assign",
                            "start_lineno": 1442,
                            "end_lineno": 1444,
                            "parent": null
                        }
                    ]
                }
            ]
        }
    ],
    "old_commit_hash": "09879794b4cdfaab37772270d3e5f843733bc2c5^",
    "new_commit_hash": "09879794b4cdfaab37772270d3e5f843733bc2c5",
    "commit_message": "Merge pull request #4953 from yarikoptic/bf-obscure%",
    "commit_date": "2020-09-24T09:54:41-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "09879794b4cdfaab37772270d3e5f843733bc2c5",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test interface.utils\n\n\"\"\"\n\nfrom contextlib import contextmanager\nimport logging\nfrom os.path import (\n    exists,\n    join as opj,\n)\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_equal,\n    assert_in,\n    assert_not_equal,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_true,\n    ok_,\n    slow,\n    with_tempfile,\n    with_tree,\n)\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import swallow_outputs\nfrom datalad.distribution.dataset import (\n    Dataset,\n    datasetmethod,\n    EnsureDataset,\n)\nfrom datalad.support.param import Parameter\nfrom datalad.support.constraints import (\n    EnsureKeyChoice,\n    EnsureNone,\n    EnsureStr,\n)\nfrom datalad.interface.base import Interface\nfrom datalad.interface.utils import (\n    discover_dataset_trace_to_targets,\n    eval_results,\n    handle_dirty_dataset,\n)\nfrom datalad.interface.results import get_status_dict\nfrom datalad.interface.base import build_doc\n\n\n__docformat__ = 'restructuredtext'\nlgr = logging.getLogger('datalad.interface.tests.test_utils')\n_dirty_modes = ('fail', 'ignore', 'save-before')\n\n\ndef _check_all_clean(ds, state):\n    assert state is not None\n    for mode in _dirty_modes:\n        # nothing wrong, nothing saved\n        handle_dirty_dataset(ds, mode)\n        assert_equal(state, ds.repo.get_hexsha())\n\n\ndef _check_auto_save(ds, orig_state):\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'save-before')\n    state = ds.repo.get_hexsha()\n    assert_not_equal(orig_state, state)\n    _check_all_clean(ds, state)\n    return state\n\n\n@with_tempfile(mkdir=True)\ndef test_dirty(path):\n    for mode in _dirty_modes:\n        # does nothing without a dataset\n        handle_dirty_dataset(None, mode)\n    # placeholder, but not yet created\n    ds = Dataset(path)\n    # unknown mode\n    assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')\n    # not yet created is very dirty\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')\n    handle_dirty_dataset(ds, 'ignore')\n    assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')\n    # should yield a clean repo\n    ds.create()\n    orig_state = ds.repo.get_hexsha()\n    _check_all_clean(ds, orig_state)\n    # tainted: untracked\n    with open(opj(ds.path, 'something'), 'w') as f:\n        f.write('some')\n    # we don't want to auto-add untracked files by saving (anymore)\n    assert_raises(AssertionError, _check_auto_save, ds, orig_state)\n    # tainted: staged\n    ds.repo.add('something', git=True)\n    orig_state = _check_auto_save(ds, orig_state)\n    # tainted: submodule\n    # not added to super on purpose!\n    subds = ds.create('subds')\n    _check_all_clean(subds, subds.repo.get_hexsha())\n    assert_repo_status(ds.path)\n    # subdataset must be added as a submodule!\n    assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])\n\n\ndemo_hierarchy = {\n    'a': {\n        'aa': {\n            'file_aa': 'file_aa'}},\n    'b': {\n        'ba': {\n            'file_ba': 'file_ba'},\n        'bb': {\n            'bba': {\n                'bbaa': {\n                    'file_bbaa': 'file_bbaa'}},\n            'file_bb': 'file_bb'}},\n    'c': {\n        'ca': {\n            'file_ca': 'file_ca'},\n        'file_c': 'file_c'},\n    'd': {\n        'da': {\n            'file_da': 'file_da'},\n        'db': {\n            'file_db': 'file_db'},\n        'file_d': 'file_d'},\n}\n\n\ndef make_demo_hierarchy_datasets(path, tree, parent=None):\n    if parent is None:\n        parent = Dataset(path).create(force=True)\n    for node, items in tree.items():\n        if isinstance(items, dict):\n            node_path = opj(path, node)\n            nodeds = parent.create(node_path, force=True)\n            make_demo_hierarchy_datasets(node_path, items, parent=nodeds)\n    return parent\n\n\n@slow  # 74.4509s\n@with_tree(demo_hierarchy)\ndef test_save_hierarchy(path):\n    # this test doesn't use API`remove` to avoid circularities\n    ds = make_demo_hierarchy_datasets(path, demo_hierarchy)\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    ds_bb = Dataset(opj(ds.path, 'b', 'bb'))\n    ds_bba = Dataset(opj(ds_bb.path, 'bba'))\n    ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa'))\n    # introduce a change at the lowest level\n    ds_bbaa.repo.remove('file_bbaa')\n    for d in (ds, ds_bb, ds_bba, ds_bbaa):\n        ok_(d.repo.dirty)\n    # need to give file specifically, otherwise it will simply just preserve\n    # staged changes\n    ds_bb.save(path=opj(ds_bbaa.path, 'file_bbaa'))\n    # it has saved all changes in the subtrees spanned\n    # by the given datasets, but nothing else\n    for d in (ds_bb, ds_bba, ds_bbaa):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # now with two modified repos\n    d = Dataset(opj(ds.path, 'd'))\n    da = Dataset(opj(d.path, 'da'))\n    da.repo.remove('file_da')\n    db = Dataset(opj(d.path, 'db'))\n    db.repo.remove('file_db')\n    # generator\n    d.save(recursive=True)\n    for d in (d, da, db):\n        assert_repo_status(d.path)\n    ok_(ds.repo.dirty)\n    # and now with files all over the place and saving\n    # all the way to the root\n    aa = Dataset(opj(ds.path, 'a', 'aa'))\n    aa.repo.remove('file_aa')\n    ba = Dataset(opj(ds.path, 'b', 'ba'))\n    ba.repo.remove('file_ba')\n    bb = Dataset(opj(ds.path, 'b', 'bb'))\n    bb.repo.remove('file_bb')\n    c = Dataset(opj(ds.path, 'c'))\n    c.repo.remove('file_c')\n    ca = Dataset(opj(ds.path, 'c', 'ca'))\n    ca.repo.remove('file_ca')\n    d = Dataset(opj(ds.path, 'd'))\n    d.repo.remove('file_d')\n    ds.save(\n        # append trailing slashes to the path to indicate that we want to\n        # have the staged content in the dataset saved, rather than only the\n        # subdataset state in the respective superds.\n        path=[opj(p, '')\n               for p in (aa.path, ba.path, bb.path, c.path, ca.path, d.path)])\n\n\n# Note: class name needs to match module's name\n@build_doc\nclass TestUtils(Interface):\n    \"\"\"TestUtil's fake command\"\"\"\n\n    result_renderer = 'tailored' # overrides None default\n    return_type = 'item-or-list' # overrides 'list'\n\n    _params_ = dict(\n        number=Parameter(\n            args=(\"-n\", \"--number\",),\n            doc=\"\"\"It's a number\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"\"specify the dataset to update.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the input and/or the current working directory\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        result_fn=Parameter(\n            args=tuple(),   # Hide this from the cmdline parser.\n            doc=\"\"\"Generate the result records with this function\n            rather than using the default logic. `number` will be\n            passed as an argument.\"\"\"),)\n\n    @staticmethod\n    @datasetmethod(name='fake_command')\n    @eval_results\n    def __call__(number, dataset=None, result_fn=None):\n        if result_fn:\n            yield from result_fn(number)\n        else:\n            for i in range(number):\n                # this dict will need to have the minimum info\n                # required by eval_results\n                yield {'path': 'some', 'status': 'ok', 'somekey': i,\n                       'action': 'off'}\n\n\ndef test_eval_results_plus_build_doc():\n\n    # test docs\n\n    # docstring was build already:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        TestUtils().__call__(1)\n        assert_not_in(\"Building doc for\", cml.out)\n    # docstring accessible both ways:\n    doc1 = Dataset.fake_command.__doc__\n    doc2 = TestUtils().__call__.__doc__\n\n    # docstring was built from Test_Util's definition:\n    assert_equal(doc1, doc2)\n    assert_in(\"TestUtil's fake command\", doc1)\n    assert_in(\"Parameters\", doc1)\n    assert_in(\"It's a number\", doc1)\n\n    # docstring shows correct override values of defaults in eval_params\n    assert_in(\"Default: 'tailored'\", doc1)\n    assert_in(\"Default: 'item-or-list'\", doc1)\n\n    # docstring also contains eval_result's parameters:\n    assert_in(\"result_filter\", doc1)\n    assert_in(\"return_type\", doc1)\n    assert_in(\"list\", doc1)\n    assert_in(\"None\", doc1)\n    assert_in(\"return value behavior\", doc1)\n    assert_in(\"dictionary is passed\", doc1)\n\n    # test eval_results is able to determine the call, a method of which it is\n    # decorating:\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        Dataset('/does/not/matter').fake_command(3)\n        assert_in(\"Determined class of decorated function: {}\"\n                  \"\".format(TestUtils().__class__), cml.out)\n\n    # test results:\n    result = TestUtils().__call__(2)\n    assert_equal(len(list(result)), 2)\n    result = Dataset('/does/not/matter').fake_command(3)\n    assert_equal(len(list(result)), 3)\n\n    # test absent side-effect of popping eval_defaults\n    kwargs = dict(return_type='list')\n    TestUtils().__call__(2, **kwargs)\n    assert_equal(list(kwargs), ['return_type'])\n\n    # test signature:\n    from datalad.utils import getargspec\n    assert_equal(getargspec(Dataset.fake_command)[0],\n                 ['number', 'dataset', 'result_fn'])\n    assert_equal(getargspec(TestUtils.__call__)[0],\n                 ['number', 'dataset', 'result_fn'])\n\n\ndef test_result_filter():\n    # ensure baseline without filtering\n    assert_equal(\n        [r['somekey'] for r in TestUtils().__call__(4)],\n        [0, 1, 2, 3])\n    # test two functionally equivalent ways to filter results\n    # 1. Constraint-based -- filter by exception\n    #    we have a full set of AND and OR operators for this\n    # 2. custom filer function -- filter by boolean return value\n    for filt in (\n            EnsureKeyChoice('somekey', (0, 2)),\n            lambda x: x['somekey'] in (0, 2)):\n        assert_equal(\n            [r['somekey'] for r in TestUtils().__call__(\n                4,\n                result_filter=filt)],\n            [0, 2])\n        # constraint returns full dict\n        assert_dict_equal(\n            TestUtils().__call__(\n                4,\n                result_filter=filt)[-1],\n            {'action': 'off', 'path': 'some', 'status': 'ok', 'somekey': 2})\n\n    # test more sophisticated filters that actually get to see the\n    # API call's kwargs\n    def greatfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), 'awesome')\n        return True\n    TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)\n\n    def sadfilter(res, **kwargs):\n        assert_equal(kwargs.get('dataset', 'bob'), None)\n        return True\n    TestUtils().__call__(4, result_filter=sadfilter)\n\n\n@with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n@with_tempfile(mkdir=True)\ndef test_discover_ds_trace(path, otherdir):\n    ds = make_demo_hierarchy_datasets(\n        path,\n        {k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n    a = opj(ds.path, 'a')\n    aa = opj(a, 'aa')\n    d = opj(ds.path, 'd')\n    db = opj(d, 'db')\n    # we have to check whether we get the correct hierarchy, as the test\n    # subject is also involved in this\n    assert_true(exists(opj(db, 'file_db')))\n    ds.save(recursive=True)\n    assert_repo_status(ds.path)\n    # now two datasets which are not available locally, but we\n    # know about them (e.g. from metadata)\n    dba = opj(db, 'sub', 'dba')\n    dbaa = opj(dba, 'subsub', 'dbaa')\n    for input, eds, goal in (\n            ([], None, {}),\n            ([ds.path], None, {}),\n            ([otherdir], None, {}),\n            ([opj(ds.path, 'nothere')], None, {}),\n            ([opj(d, 'nothere')], None, {}),\n            ([opj(db, 'nothere')], None, {}),\n            ([a], None,\n             {ds.path: set([a])}),\n            ([aa, a], None,\n             {ds.path: set([a]), a: set([aa])}),\n            ([db], None,\n             {ds.path: set([d]), d: set([db])}),\n            ([opj(db, 'file_db')], None,\n             {ds.path: set([d]), d: set([db])}),\n            # just a regular non-existing path\n            ([dba], None, {}),\n            # but if we inject this knowledge it must come back out\n            # as the child of the closest existing dataset\n            ([dba], [dba],\n             {ds.path: set([d]), d: set([db]), db: set([dba])}),\n            # regardless of the depth\n            ([dbaa], [dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dbaa])}),\n            ([dba, dbaa], [dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n            # we can simply add existing and non-existing datasets to the\n            # include list get the desired result\n            ([d, dba, dbaa], [d, dba, dbaa],\n             {ds.path: set([d]), d: set([db]), db: set([dba, dbaa])}),\n    ):\n        spec = {}\n        discover_dataset_trace_to_targets(ds.path, input, [], spec, includeds=eds)\n        assert_dict_equal(spec, goal)\n\n\n@contextmanager\ndef _swallow_outputs(isatty=True):\n    with swallow_outputs() as cmo:\n        stdout = cmo.handles[0]\n        stdout.isatty = lambda: isatty\n        yield cmo\n\n\ndef test_utils_suppress_similar():\n    tu = TestUtils()\n\n    # Check suppression boundary for straight chain of similar\n    # messages.\n\n    def n_foo(number):\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n\n    with _swallow_outputs() as cmo:\n        cmo.isatty = lambda: True\n        list(tu(9, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path8\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(10, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path9\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        assert_re_in(r\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs() as cmo:\n        list(tu(13, result_fn=n_foo, result_renderer=\"default\"))\n        assert_not_in(\"path10\", cmo.out)\n        # We see an update for each result.\n        assert_re_in(r\"1 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"2 .* suppressed\", cmo.out, match=False)\n        assert_re_in(r\"3 .* suppressed\", cmo.out, match=False)\n\n    with _swallow_outputs(isatty=False) as cmo:\n        list(tu(11, result_fn=n_foo, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n\n    # Check a chain of similar messages, split in half by a distinct one.\n\n    def n_foo_split_by_a_bar(number):\n        half = number // 2 - 1\n        for i in range(number):\n            yield dict(action=\"foo\",\n                       status=\"ok\",\n                       path=\"path{}\".format(i))\n            if i == half:\n                yield dict(action=\"bar\",\n                           status=\"ok\",\n                           path=\"path\")\n\n    with _swallow_outputs() as cmo:\n        list(tu(20, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_in(\"path19\", cmo.out)\n        assert_not_in(\"suppressed\", cmo.out)\n\n    with _swallow_outputs() as cmo:\n        list(tu(21, result_fn=n_foo_split_by_a_bar, result_renderer=\"default\"))\n        assert_in(\"path10\", cmo.out)\n        assert_not_in(\"path20\", cmo.out)\n        assert_re_in(\"[^-0-9]1 .* suppressed\", cmo.out, match=False)\n\n\nclass TestUtils2(Interface):\n    # result_renderer = custom_renderer\n    _params_ = dict(\n        number=Parameter(\n            args=(\"--path\",),\n            constraints=EnsureStr() | EnsureNone()),\n    )\n    @staticmethod\n    @eval_results\n    def __call__(path=None):\n        def logger(msg, *args):\n            return msg % args\n        if path:\n            # we will be testing for path %s\n            message = (\"all good %s\", \"my friend\")\n        else:\n            message = (\"kaboom %s %s\", \"greedy\")\n        yield get_status_dict(\n            action=\"test\",\n            status=\"ok\",\n            message=message,\n            logger=logger,\n            path=path or ''\n        )\n\n\ndef test_incorrect_msg_interpolation():\n    with assert_raises(TypeError) as cme:\n        TestUtils2().__call__()\n    # this must be our custom exception\n    assert_re_in(\"Failed to render.*kaboom.*not enough arguments\", str(cme.exception))\n\n    # there should be no exception if reported in the record path contains %\n    TestUtils2().__call__(\"%eatthis\")",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.13.3.dev109\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 155ms\nInstalled 25 packages in 3.85s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 63 packages in 4.45s\nPrepared 1 package in 3.27s\nInstalled 57 packages in 412ms\n + annexremote==1.6.6\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.13.3 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.0.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 97 packages in 311ms\nInstalled 32 packages in 1.09s\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5\nplugins: cov-6.0.0\ncollected 7 items\n\nr2e_tests/test_1.py FF..E..                                              [100%]\n\n==================================== ERRORS ====================================\n___________________ ERROR at setup of test_discover_ds_trace ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5/r2e_tests/test_1.py, line 337\n  @with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n  @with_tempfile(mkdir=True)\n  def test_discover_ds_trace(path, otherdir):\nE       fixture 'otherdir' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5/r2e_tests/test_1.py:337\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexnark_4rp'}\nfilename = '/tmp/datalad_temp_test_dirty4iwb29r7'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:731: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexa09slv9e'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchyhsawaeck'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:554: TypeError\n==================================== PASSES ====================================\n_______________________ test_eval_results_plus_build_doc _______________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.utils:utils.py:2505 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\nDEBUG    datalad.utils:utils.py:2505 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\n_________________________ test_utils_suppress_similar __________________________\n------------------------------ Captured log call -------------------------------\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_eval_results_plus_build_doc\nPASSED r2e_tests/test_1.py::test_result_filter\nPASSED r2e_tests/test_1.py::test_utils_suppress_similar\nPASSED r2e_tests/test_1.py::test_incorrect_msg_interpolation\nERROR r2e_tests/test_1.py::test_discover_ds_trace\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\n===================== 2 failed, 4 passed, 1 error in 3.40s =====================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5\nplugins: cov-6.0.0\ncollected 7 items\n\nr2e_tests/test_1.py FF..E.F                                              [100%]\n\n==================================== ERRORS ====================================\n___________________ ERROR at setup of test_discover_ds_trace ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5/r2e_tests/test_1.py, line 337\n  @with_tree({k: v for k, v in demo_hierarchy.items() if k in ['a', 'd']})\n  @with_tempfile(mkdir=True)\n  def test_discover_ds_trace(path, otherdir):\nE       fixture 'otherdir' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_09879794b4cdfaab37772270d3e5f843733bc2c5/r2e_tests/test_1.py:337\n=================================== FAILURES ===================================\n__________________________________ test_dirty __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexl0_92d4e'}\nfilename = '/tmp/datalad_temp_test_dirtyg02qa257'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:731: TypeError\n_____________________________ test_save_hierarchy ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex16_of1ix'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_save_hierarchy'}\nd = '/tmp/datalad_temp_tree_test_save_hierarchytp78vepm'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_save_hierarchy() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:554: TypeError\n_______________________ test_incorrect_msg_interpolation _______________________\n\n    def test_incorrect_msg_interpolation():\n        with assert_raises(TypeError) as cme:\n            TestUtils2().__call__()\n        # this must be our custom exception\n>       assert_re_in(\"Failed to render.*kaboom.*not enough arguments\", str(cme.exception))\n\nr2e_tests/test_1.py:496: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nregex = 'Failed to render.*kaboom.*not enough arguments'\nc = ['not enough arguments for format string'], flags = 0, match = True\nmsg = None\n\n    def assert_re_in(regex, c, flags=0, match=True, msg=None):\n        \"\"\"Assert that container (list, str, etc) contains entry matching the regex\n        \"\"\"\n        if not isinstance(c, (list, tuple)):\n            c = [c]\n        for e in c:\n            if (re.match if match else re.search)(regex, e, flags=flags):\n                return\n>       raise AssertionError(\n            msg or \"Not a single entry matched %r in %r\" % (regex, c)\n        )\nE       AssertionError: Not a single entry matched 'Failed to render.*kaboom.*not enough arguments' in ['not enough arguments for format string']\n\ndatalad/tests/utils.py:1244: AssertionError\n==================================== PASSES ====================================\n_______________________ test_eval_results_plus_build_doc _______________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.utils:utils.py:2505 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\nDEBUG    datalad.utils:utils.py:2505 Determined class of decorated function: <class 'r2e_tests.test_1.TestUtils'>\n_________________________ test_utils_suppress_similar __________________________\n------------------------------ Captured log call -------------------------------\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\nINFO     datalad.ui.dialog:log.py:308 Clear progress bars\nINFO     datalad.ui.dialog:log.py:308 Refresh progress bars\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_eval_results_plus_build_doc\nPASSED r2e_tests/test_1.py::test_result_filter\nPASSED r2e_tests/test_1.py::test_utils_suppress_similar\nERROR r2e_tests/test_1.py::test_discover_ds_trace\nFAILED r2e_tests/test_1.py::test_dirty - TypeError: test_dirty() got multiple...\nFAILED r2e_tests/test_1.py::test_save_hierarchy - TypeError: test_save_hierar...\nFAILED r2e_tests/test_1.py::test_incorrect_msg_interpolation - AssertionError...\n===================== 3 failed, 3 passed, 1 error in 0.90s =====================\n",
    "old_commit_res_stderr": ""
}
datalad/interface/base.py datalad/interface/tests/test_utils.py datalad/interface/utils.py datalad/support/archives.py datalad/tests/utils.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Uninformative TypeError Raised During Message Rendering with Insufficient Arguments

**Description:**
When logging messages that include format placeholders, providing fewer arguments than required results in a generic `TypeError`. This makes it difficult to identify and debug issues related to message formatting.

**Example Code:**
```python
def logger(msg, *args):
    return msg % args

# This will raise a TypeError due to insufficient arguments for the format string
logger("Operation failed at %s with error code %s", "ModuleA")
```

**Expected Behavior:**
A clear and descriptive `TypeError` indicating that there are not enough arguments provided for the format string, helping developers quickly identify the mismatch.

**Actual Behavior:**
A generic `TypeError` without specific details about the formatting issue, making it harder to trace and fix the problem.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/utils.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface utility functions

"""

__docformat__ = 'restructuredtext'

import inspect
import logging
import wrapt
import sys
import re
from os import curdir
from os import pardir
from os import listdir
import os.path as op
from os.path import join as opj
from os.path import isdir
from os.path import relpath
from os.path import sep
from os.path import split as psplit
from itertools import chain

import json

# avoid import from API to not get into circular imports
from datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us
from datalad.utils import (
    path_startswith,
    path_is_subpath,
    assure_unicode,
    getargspec,
    get_wrapped_class,
)
from datalad.support.gitrepo import GitRepo
from datalad.support.exceptions import IncompleteResultsError
from datalad import cfg as dlcfg
from datalad.dochelpers import (
    exc_str,
    single_or_plural,
)

from datalad.ui import ui
import datalad.support.ansi_colors as ac

from datalad.interface.base import default_logchannels
from datalad.interface.base import get_allargs_as_kwargs
from datalad.interface.common_opts import eval_params
from datalad.interface.common_opts import eval_defaults
from .results import known_result_xfms
from datalad.config import ConfigManager
from datalad.core.local.resulthooks import (
    get_jsonhooks_from_config,
    match_jsonhook2result,
    run_jsonhook,
)


lgr = logging.getLogger('datalad.interface.utils')


def cls2cmdlinename(cls):
    "Return the cmdline command name from an Interface class"
    r = re.compile(r'([a-z0-9])([A-Z])')
    return r.sub('\\1-\\2', cls.__name__).lower()


# TODO remove
# only `drop` and `uninstall` are still using this
def handle_dirty_dataset(ds, mode, msg=None):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    ds : Dataset or None
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    msg : str or None
      Custom message to use for a potential commit.

    Returns
    -------
    None
    """
    if ds is None:
        # nothing to be handled
        return
    if msg is None:
        msg = '[DATALAD] auto-saved changes'

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    if ds.repo:
        ds.repo.precommit()

    if mode == 'ignore':
        return
    elif mode == 'fail':
        if not ds.repo or ds.repo.dirty:
            raise RuntimeError('dataset {} has unsaved changes'.format(ds))
    elif mode == 'save-before':
        if not ds.is_installed():
            raise RuntimeError('dataset {} is not yet installed'.format(ds))
        from datalad.core.local.save import Save
        Save.__call__(dataset=ds, message=msg, updated=True)
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def get_tree_roots(paths):
    """Return common root paths for a set of paths

    This function determines the smallest set of common root
    paths and sorts all given paths under the respective
    root.

    Returns
    -------
    dict
      paths by root
    """
    paths_ws = [_with_sep(p) for p in paths]
    # sort all paths under their potential roots
    roots = {}
    # start from the top to get all paths down the line
    # and collate them into as few roots as possible
    for s in sorted(paths_ws):
        if any([s.startswith(r) for r in roots]):
            # this path is already covered by a known root
            continue
        # find all sub paths
        subs = [p for p in paths if p.startswith(s)]
        roots[s.rstrip(sep)] = subs
    return roots


# TODO remove
# only `remove` and `uninstall` use this, the uses path `path_is_subpath`
def path_is_under(values, path=None):
    """Whether a given path is a subdirectory of any of the given test values

    Parameters
    ----------
    values : sequence or dict
      Paths to be tested against. This can be a dictionary in which case
      all values from all keys will be tested against.
    path : path or None
      Test path. If None is given, the process' working directory is
      used.

    Returns
    -------
    bool
    """
    if path is None:
        from datalad.utils import getpwd
        path = getpwd()
    if isinstance(values, dict):
        values = chain(*values.values())
    path_drive, _ = op.splitdrive(path)
    for p in values:
        p_drive, _ = op.splitdrive(p)
        # need to protect against unsupported use of relpath() with
        # abspaths on windows from different drives (gh-3724)
        if path_drive != p_drive:
        # different drives, enough evidence for "not under"
            continue
        rpath = relpath(p, start=path)
        if rpath == curdir \
                or rpath == pardir \
                or set(psplit(rpath)) == {pardir}:
            # first match is enough
            return True
    return False


def discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,
                                      spec, includeds=None):
    """Discover the edges and nodes in a dataset tree to given target paths

    Parameters
    ----------
    basepath : path
      Path to a start or top-level dataset. Really has to be a path to a
      dataset!
    targetpaths : list(path)
      Any non-zero number of paths that are termination points for the
      search algorithm. Can be paths to datasets, directories, or files
      (and any combination thereof).
    current_trace : list
      For a top-level call this should probably always be `[]`
    spec : dict
      `content_by_ds`-style dictionary that will receive information about the
      discovered datasets. Specifically, for each discovered dataset there
      will be in item with its path under the key (path) of the respective
      superdataset.
    includeds : sequence, optional
      Any paths given are treated as existing subdatasets, regardless of
      whether they can be found in the filesystem. Such subdatasets will appear
      under the key of the closest existing dataset in the `spec`.

    Returns
    -------
    None
      Function calls itself recursively and populates `spec` dict in-place.
      Keys are dataset paths, values are sets of subdataset paths
    """
    # convert to set for faster lookup
    includeds = includeds if isinstance(includeds, set) else \
        set() if includeds is None else set(includeds)
    # this beast walks the directory tree from a given `basepath` until
    # it discovers any of the given `targetpaths`
    # if it finds one, it commits any accummulated trace of visited
    # datasets on this edge to the spec
    valid_repo = GitRepo.is_valid_repo(basepath)
    if valid_repo:
        # we are passing into a new dataset, extend the dataset trace
        current_trace = current_trace + [basepath]
    # this edge is not done, we need to try to reach any downstream
    # dataset
    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)
    # whether anything in this directory matched a targetpath
    filematch = False
    if isdir(basepath):
        for p in listdir(basepath):
            p = assure_unicode(opj(basepath, p))
            if not isdir(p):
                if p in targetpaths:
                    filematch = True
                # we cannot have anything below this one
                continue
            # OPT listdir might be large and we could have only few items
            # in `targetpaths` -- so traverse only those in spec which have
            # leading dir basepath
            # filter targets matching this downward path
            downward_targets = set(
                t for t in targetpaths if path_startswith(t, p))
            if not downward_targets:
                continue
            # remove the matching ones from the "todo" list
            undiscovered_ds.difference_update(downward_targets)
            # go one deeper
            discover_dataset_trace_to_targets(
                p, downward_targets, current_trace, spec,
                includeds=includeds if not includeds else includeds.intersection(
                    downward_targets))
    undiscovered_ds = [t for t in undiscovered_ds
                       if includeds and
                          path_is_subpath(t, current_trace[-1]) and
                          t in includeds]
    if filematch or basepath in targetpaths or undiscovered_ds:
        for i, p in enumerate(current_trace[:-1]):
            # TODO RF prepare proper annotated path dicts
            subds = spec.get(p, set())
            subds.add(current_trace[i + 1])
            spec[p] = subds
        if undiscovered_ds:
            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(
                undiscovered_ds)


def get_result_filter(fx):
    """Wrap a filter into a helper to be able to accept additional
    arguments, if the filter doesn't support it already"""
    _fx = fx
    if fx and not getargspec(fx).keywords:
        def _fx(res, **kwargs):
            return fx(res)
    return _fx


def eval_results(func):
    """Decorator for return value evaluation of datalad commands.

    Note, this decorator is only compatible with commands that return
    status dict sequences!

    Two basic modes of operation are supported: 1) "generator mode" that
    `yields` individual results, and 2) "list mode" that returns a sequence of
    results. The behavior can be selected via the kwarg `return_type`.
    Default is "list mode".

    This decorator implements common functionality for result rendering/output,
    error detection/handling, and logging.

    Result rendering/output can be triggered via the
    `datalad.api.result-renderer` configuration variable, or the
    `result_renderer` keyword argument of each decorated command. Supported
    modes are: 'default' (one line per result with action, status, path,
    and an optional message); 'json' (one object per result, like git-annex),
    'json_pp' (like 'json', but pretty-printed spanning multiple lines),
    'tailored' custom output formatting provided by each command
    class (if any).

    Error detection works by inspecting the `status` item of all result
    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'
    will cause an IncompleteResultsError exception to be raised that carries
    the failed actions' status dictionaries in its `failed` attribute.

    Status messages will be logged automatically, by default the following
    association of result status and log channel will be used: 'ok' (debug),
    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger
    instances included in the results are used to capture the origin of a
    status report.

    Parameters
    ----------
    func: function
      __call__ method of a subclass of Interface,
      i.e. a datalad command definition
    """

    @wrapt.decorator
    def eval_func(wrapped, instance, args, kwargs):
        lgr.log(2, "Entered eval_func for %s", func)
        # for result filters
        # we need to produce a dict with argname/argvalue pairs for all args
        # incl. defaults and args given as positionals
        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)

        # determine the command class associated with `wrapped`
        wrapped_class = get_wrapped_class(wrapped)

        # retrieve common options from kwargs, and fall back on the command
        # class attributes, or general defaults if needed
        kwargs = kwargs.copy()  # we will pop, which might cause side-effect
        common_params = {
            p_name: kwargs.pop(
                # go with any explicitly given default
                p_name,
                # otherwise determine the command class and pull any
                # default set in that class
                getattr(
                    wrapped_class,
                    p_name,
                    # or the common default
                    eval_defaults[p_name]))
            for p_name in eval_params}

        # short cuts and configured setup for common options
        return_type = common_params['return_type']
        result_filter = get_result_filter(common_params['result_filter'])
        # resolve string labels for transformers too
        result_xfm = known_result_xfms.get(
            common_params['result_xfm'],
            # use verbatim, if not a known label
            common_params['result_xfm'])
        result_renderer = common_params['result_renderer']
        # TODO remove this conditional branch entirely, done outside
        if not result_renderer:
            result_renderer = dlcfg.get('datalad.api.result-renderer', None)
        # look for potential override of logging behavior
        result_log_level = dlcfg.get('datalad.log.result-level', None)

        # query cfg for defaults
        # .is_installed and .config can be costly, so ensure we do
        # it only once. See https://github.com/datalad/datalad/issues/3575
        dataset_arg = allkwargs.get('dataset', None)
        from datalad.distribution.dataset import Dataset
        ds = dataset_arg if isinstance(dataset_arg, Dataset) \
            else Dataset(dataset_arg) if dataset_arg else None
        # do not reuse a dataset's existing config manager here
        # they are configured to read the committed dataset configuration
        # too. That means a datalad update can silently bring in new
        # procedure definitions from the outside, and in some sense enable
        # remote code execution by a 3rd-party
        # To avoid that, create a new config manager that only reads local
        # config (system and .git/config), plus any overrides given to this
        # datalad session
        proc_cfg = ConfigManager(
            ds, source='local', overrides=dlcfg.overrides
        ) if ds and ds.is_installed() else dlcfg

        # look for hooks
        hooks = get_jsonhooks_from_config(proc_cfg)

        # this internal helper function actually drives the command
        # generator-style, it may generate an exception if desired,
        # on incomplete results
        def generator_func(*_args, **_kwargs):
            # flag whether to raise an exception
            incomplete_results = []
            # track what actions were performed how many times
            action_summary = {}

            # if a custom summary is to be provided, collect the results
            # of the command execution
            results = []
            do_custom_result_summary = result_renderer in ('tailored', 'default') \
                and hasattr(wrapped_class, 'custom_result_summary_renderer')

            # process main results
            for r in _process_results(
                    # execution
                    wrapped(*_args, **_kwargs),
                    wrapped_class,
                    common_params['on_failure'],
                    # bookkeeping
                    action_summary,
                    incomplete_results,
                    # communication
                    result_renderer,
                    result_log_level,
                    # let renderers get to see how a command was called
                    allkwargs):
                for hook, spec in hooks.items():
                    # run the hooks before we yield the result
                    # this ensures that they are executed before
                    # a potentially wrapper command gets to act
                    # on them
                    if match_jsonhook2result(hook, r, spec['match']):
                        lgr.debug('Result %s matches hook %s', r, hook)
                        # a hook is also a command that yields results
                        # so yield them outside too
                        # users need to pay attention to void infinite
                        # loops, i.e. when a hook yields a result that
                        # triggers that same hook again
                        for hr in run_jsonhook(hook, spec, r, dataset_arg):
                            # apply same logic as for main results, otherwise
                            # any filters would only tackle the primary results
                            # and a mixture of return values could happen
                            if not keep_result(hr, result_filter, **allkwargs):
                                continue
                            hr = xfm_result(hr, result_xfm)
                            # rationale for conditional is a few lines down
                            if hr:
                                yield hr
                if not keep_result(r, result_filter, **allkwargs):
                    continue
                r = xfm_result(r, result_xfm)
                # in case the result_xfm decided to not give us anything
                # exclude it from the results. There is no particular reason
                # to do so other than that it was established behavior when
                # this comment was written. This will not affect any real
                # result record
                if r:
                    yield r

                # collect if summary is desired
                if do_custom_result_summary:
                    results.append(r)

            # result summary before a potential exception
            # custom first
            if do_custom_result_summary:
                wrapped_class.custom_result_summary_renderer(results)
            elif result_renderer == 'default' and action_summary and \
                    sum(sum(s.values()) for s in action_summary.values()) > 1:
                # give a summary in default mode, when there was more than one
                # action performed
                ui.message("action summary:\n  {}".format(
                    '\n  '.join('{} ({})'.format(
                        act,
                        ', '.join('{}: {}'.format(status, action_summary[act][status])
                                  for status in sorted(action_summary[act])))
                                for act in sorted(action_summary))))

            if incomplete_results:
                raise IncompleteResultsError(
                    failed=incomplete_results,
                    msg="Command did not complete successfully")

        if return_type == 'generator':
            # hand over the generator
            lgr.log(2, "Returning generator_func from eval_func for %s", wrapped_class)
            return generator_func(*args, **kwargs)
        else:
            @wrapt.decorator
            def return_func(wrapped_, instance_, args_, kwargs_):
                results = wrapped_(*args_, **kwargs_)
                if inspect.isgenerator(results):
                    # unwind generator if there is one, this actually runs
                    # any processing
                    results = list(results)
                # render summaries
                if not result_xfm and result_renderer in ('tailored', 'default'):
                    # cannot render transformed results
                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):
                        wrapped_class.custom_result_summary_renderer(results)
                if return_type == 'item-or-list' and \
                        len(results) < 2:
                    return results[0] if results else None
                else:
                    return results
            lgr.log(2, "Returning return_func from eval_func for %s", wrapped_class)
            return return_func(generator_func)(*args, **kwargs)

    return eval_func(func)


def default_result_renderer(res):
    if res.get('status', None) != 'notneeded':
        path = res.get('path', None)
        path = ' {}'.format(path) if path else ''
        ui.message('{action}({status}):{path}{type}{msg}'.format(
                action=ac.color_word(res['action'], ac.BOLD),
                status=ac.color_status(res['status']),
                path=relpath(path, res['refds']) if path and res.get('refds') else path,
                type=' ({})'.format(
                        ac.color_word(res['type'], ac.MAGENTA)
                ) if 'type' in res else '',
                msg=' [{}]'.format(
                        res['message'][0] % res['message'][1:]
                        if isinstance(res['message'], tuple) else res[
                            'message'])
                if res.get('message', None) else ''))


def _display_suppressed_message(nsimilar, ndisplayed, final=False):
    # +1 because there was the original result + nsimilar displayed.
    n_suppressed = nsimilar - ndisplayed + 1
    if n_suppressed > 0:
        ui.message('  [{} similar {} been suppressed]'
                   .format(n_suppressed,
                           single_or_plural("message has",
                                            "messages have",
                                            n_suppressed, False)),
                   cr="\n" if final else "\r")


def _process_results(
        results,
        cmd_class,
        on_failure,
        action_summary,
        incomplete_results,
        result_renderer,
        result_log_level,
        allkwargs):
    # private helper pf @eval_results
    # loop over results generated from some source and handle each
    # of them according to the requested behavior (logging, rendering, ...)

    # used to track repeated messages in the default renderer
    last_result = None
    # which result dict keys to inspect for changes to discover repetions
    # of similar messages
    repetition_keys = set(('action', 'status', 'type', 'refds'))
    # counter for detected repetitions
    result_repetitions = 0
    # how many repetitions to show, before suppression kicks in
    render_n_repetitions = 10 if sys.stdout.isatty() else float("inf")

    for res in results:
        if not res or 'action' not in res:
            # XXX Yarik has to no clue on how to track the origin of the
            # record to figure out WTF, so he just skips it
            continue

        actsum = action_summary.get(res['action'], {})
        if res['status']:
            actsum[res['status']] = actsum.get(res['status'], 0) + 1
            action_summary[res['action']] = actsum
        ## log message, if there is one and a logger was given
        msg = res.get('message', None)
        # remove logger instance from results, as it is no longer useful
        # after logging was done, it isn't serializable, and generally
        # pollutes the output
        res_lgr = res.pop('logger', None)
        if msg and res_lgr:
            if isinstance(res_lgr, logging.Logger):
                # didn't get a particular log function, go with default
                res_lgr = getattr(
                    res_lgr,
                    default_logchannels[res['status']]
                    if result_log_level is None
                    else result_log_level)
            msg = res['message']
            msgargs = None
            if isinstance(msg, tuple):
                msgargs = msg[1:]
                msg = msg[0]
            if 'path' in res:
                msg = '{} [{}({})]'.format(
                    msg, res['action'], res['path'])
            if msgargs:
                # support string expansion of logging to avoid runtime cost
                res_lgr(msg, *msgargs)
            else:
                res_lgr(msg)

        ## output rendering
        # TODO RF this in a simple callable that gets passed into this function
        if result_renderer is None or result_renderer == 'disabled':
            pass
        elif result_renderer == 'default':
            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}
            if res.get('status', None) != 'notneeded' \
                    and trimmed_result == last_result:
                # this is a similar report, suppress if too many, but count it
                result_repetitions += 1
                if result_repetitions < render_n_repetitions:
                    default_result_renderer(res)
                else:
                    _display_suppressed_message(
                        result_repetitions, render_n_repetitions)
            else:
                # this one is new, first report on any prev. suppressed results
                # by number, and then render this fresh one
                _display_suppressed_message(
                    result_repetitions, render_n_repetitions,
                    final=True)
                default_result_renderer(res)
                result_repetitions = 0
            last_result = trimmed_result
        elif result_renderer in ('json', 'json_pp'):
            ui.message(json.dumps(
                {k: v for k, v in res.items()
                 if k not in ('message', 'logger')},
                sort_keys=True,
                indent=2 if result_renderer.endswith('_pp') else None,
                default=lambda x: str(x)))
        elif result_renderer in ('tailored', 'default'):
            if hasattr(cmd_class, 'custom_result_renderer'):
                cmd_class.custom_result_renderer(res, **allkwargs)
        elif hasattr(result_renderer, '__call__'):
            try:
                result_renderer(res, **allkwargs)
            except Exception as e:
                lgr.warning('Result rendering failed for: %s [%s]',
                            res, exc_str(e))
        else:
            raise ValueError('unknown result renderer "{}"'.format(result_renderer))

        ## error handling
        # looks for error status, and report at the end via
        # an exception
        if on_failure in ('continue', 'stop') \
                and res['status'] in ('impossible', 'error'):
            incomplete_results.append(res)
            if on_failure == 'stop':
                # first fail -> that's it
                # raise will happen after the loop
                break
        yield res
    # make sure to report on any issues that we had suppressed
    _display_suppressed_message(
        result_repetitions, render_n_repetitions, final=True)


def keep_result(res, rfilter, **kwargs):
    if not rfilter:
        return True
    try:
        if not rfilter(res, **kwargs):
            # give the slightest indication which filter was employed
            raise ValueError(
                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))
    except ValueError as e:
        # make sure to report the excluded result to massively improve
        # debugging experience
        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)
        return False
    return True


def xfm_result(res, xfm):
    if not xfm:
        return res

    return xfm(res)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Uninformative TypeError Raised During Message Rendering with Insufficient Arguments

**Description:**
When logging messages that include format placeholders, providing fewer arguments than required results in a generic `TypeError`. This makes it difficult to identify and debug issues related to message formatting.

**Example Code:**
```python
def logger(msg, *args):
    return msg % args

# This will raise a TypeError due to insufficient arguments for the format string
logger("Operation failed at %s with error code %s", "ModuleA")
```

**Expected Behavior:**
A clear and descriptive `TypeError` indicating that there are not enough arguments provided for the format string, helping developers quickly identify the mismatch.

**Actual Behavior:**
A generic `TypeError` without specific details about the formatting issue, making it harder to trace and fix the problem.
[/ISSUE]
{
    "test_eval_results_plus_build_doc": "PASSED",
    "test_result_filter": "PASSED",
    "test_utils_suppress_similar": "PASSED",
    "test_incorrect_msg_interpolation": "PASSED",
    "test_discover_ds_trace": "ERROR",
    "test_dirty": "FAILED",
    "test_save_hierarchy": "FAILED"
}
diff --git a/datalad/interface/base.py b/datalad/interface/base.py
index 62abc6e72..fde19b4b0 100644
--- a/datalad/interface/base.py
+++ b/datalad/interface/base.py
@@ -837,6 +837,7 @@ def get_allargs_as_kwargs(call, args, kwargs):
     argspec = getargspec(call)
     defaults = argspec.defaults
     nargs = len(argspec.args)
+    defaults = defaults or []  # ensure it is a list and not None
     assert (nargs >= len(defaults))
     # map any args to their name
     argmap = list(zip(argspec.args[:len(args)], args))
diff --git a/datalad/interface/utils.py b/datalad/interface/utils.py
index 696e600b6..42f51e0de 100644
--- a/datalad/interface/utils.py
+++ b/datalad/interface/utils.py
@@ -579,11 +579,21 @@ def _process_results(
                 msgargs = msg[1:]
                 msg = msg[0]
             if 'path' in res:
+                path = res['path']
+                if msgargs:
+                    # we will pass the msg for %-polation, so % should be doubled
+                    path = path.replace('%', '%%')
                 msg = '{} [{}({})]'.format(
-                    msg, res['action'], res['path'])
+                    msg, res['action'], path)
             if msgargs:
                 # support string expansion of logging to avoid runtime cost
-                res_lgr(msg, *msgargs)
+                try:
+                    res_lgr(msg, *msgargs)
+                except TypeError as exc:
+                    raise TypeError(
+                        "Failed to render %r with %r from %r: %s"
+                        % (msg, msgargs, res, exc_str(exc))
+                    )
             else:
                 res_lgr(msg)
 
diff --git a/datalad/support/archives.py b/datalad/support/archives.py
index 2b8db21fb..a5b076b24 100644
--- a/datalad/support/archives.py
+++ b/datalad/support/archives.py
@@ -341,7 +341,7 @@ class ExtractedArchive(object):
 
         It does not actually extract any archive
         """
-        return opj(self.path, urlunquote(afile))
+        return opj(self.path, afile)
 
     def get_extracted_files(self):
         """Generator to provide filenames which are available under extracted archive

{"datalad/interface/utils.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface utility functions\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport inspect\nimport logging\nimport wrapt\nimport sys\nimport re\nfrom os import curdir\nfrom os import pardir\nfrom os import listdir\nimport os.path as op\nfrom os.path import join as opj\nfrom os.path import isdir\nfrom os.path import relpath\nfrom os.path import sep\nfrom os.path import split as psplit\nfrom itertools import chain\n\nimport json\n\n# avoid import from API to not get into circular imports\nfrom datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us\nfrom datalad.utils import (\n    path_startswith,\n    path_is_subpath,\n    assure_unicode,\n    getargspec,\n    get_wrapped_class,\n)\nfrom datalad.support.gitrepo import GitRepo\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad import cfg as dlcfg\nfrom datalad.dochelpers import (\n    exc_str,\n    single_or_plural,\n)\n\nfrom datalad.ui import ui\nimport datalad.support.ansi_colors as ac\n\nfrom datalad.interface.base import default_logchannels\nfrom datalad.interface.base import get_allargs_as_kwargs\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom .results import known_result_xfms\nfrom datalad.config import ConfigManager\nfrom datalad.core.local.resulthooks import (\n    get_jsonhooks_from_config,\n    match_jsonhook2result,\n    run_jsonhook,\n)\n\n\nlgr = logging.getLogger('datalad.interface.utils')\n\n\ndef cls2cmdlinename(cls):\n    \"Return the cmdline command name from an Interface class\"\n    r = re.compile(r'([a-z0-9])([A-Z])')\n    return r.sub('\\\\1-\\\\2', cls.__name__).lower()\n\n\n# TODO remove\n# only `drop` and `uninstall` are still using this\ndef handle_dirty_dataset(ds, mode, msg=None):\n    \"\"\"Detect and treat unsaved changes as instructed by `mode`\n\n    Parameters\n    ----------\n    ds : Dataset or None\n      Dataset to be inspected. Does nothing if `None`.\n    mode : {'fail', 'ignore', 'save-before'}\n      How to act upon discovering unsaved changes.\n    msg : str or None\n      Custom message to use for a potential commit.\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if ds is None:\n        # nothing to be handled\n        return\n    if msg is None:\n        msg = '[DATALAD] auto-saved changes'\n\n    # make sure that all pending changes (batched annex operations, etc.)\n    # are actually reflected in Git\n    if ds.repo:\n        ds.repo.precommit()\n\n    if mode == 'ignore':\n        return\n    elif mode == 'fail':\n        if not ds.repo or ds.repo.dirty:\n            raise RuntimeError('dataset {} has unsaved changes'.format(ds))\n    elif mode == 'save-before':\n        if not ds.is_installed():\n            raise RuntimeError('dataset {} is not yet installed'.format(ds))\n        from datalad.core.local.save import Save\n        Save.__call__(dataset=ds, message=msg, updated=True)\n    else:\n        raise ValueError(\"unknown if-dirty mode '{}'\".format(mode))\n\n\ndef get_tree_roots(paths):\n    \"\"\"Return common root paths for a set of paths\n\n    This function determines the smallest set of common root\n    paths and sorts all given paths under the respective\n    root.\n\n    Returns\n    -------\n    dict\n      paths by root\n    \"\"\"\n    paths_ws = [_with_sep(p) for p in paths]\n    # sort all paths under their potential roots\n    roots = {}\n    # start from the top to get all paths down the line\n    # and collate them into as few roots as possible\n    for s in sorted(paths_ws):\n        if any([s.startswith(r) for r in roots]):\n            # this path is already covered by a known root\n            continue\n        # find all sub paths\n        subs = [p for p in paths if p.startswith(s)]\n        roots[s.rstrip(sep)] = subs\n    return roots\n\n\n# TODO remove\n# only `remove` and `uninstall` use this, the uses path `path_is_subpath`\ndef path_is_under(values, path=None):\n    \"\"\"Whether a given path is a subdirectory of any of the given test values\n\n    Parameters\n    ----------\n    values : sequence or dict\n      Paths to be tested against. This can be a dictionary in which case\n      all values from all keys will be tested against.\n    path : path or None\n      Test path. If None is given, the process' working directory is\n      used.\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    if path is None:\n        from datalad.utils import getpwd\n        path = getpwd()\n    if isinstance(values, dict):\n        values = chain(*values.values())\n    path_drive, _ = op.splitdrive(path)\n    for p in values:\n        p_drive, _ = op.splitdrive(p)\n        # need to protect against unsupported use of relpath() with\n        # abspaths on windows from different drives (gh-3724)\n        if path_drive != p_drive:\n        # different drives, enough evidence for \"not under\"\n            continue\n        rpath = relpath(p, start=path)\n        if rpath == curdir \\\n                or rpath == pardir \\\n                or set(psplit(rpath)) == {pardir}:\n            # first match is enough\n            return True\n    return False\n\n\ndef discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,\n                                      spec, includeds=None):\n    \"\"\"Discover the edges and nodes in a dataset tree to given target paths\n\n    Parameters\n    ----------\n    basepath : path\n      Path to a start or top-level dataset. Really has to be a path to a\n      dataset!\n    targetpaths : list(path)\n      Any non-zero number of paths that are termination points for the\n      search algorithm. Can be paths to datasets, directories, or files\n      (and any combination thereof).\n    current_trace : list\n      For a top-level call this should probably always be `[]`\n    spec : dict\n      `content_by_ds`-style dictionary that will receive information about the\n      discovered datasets. Specifically, for each discovered dataset there\n      will be in item with its path under the key (path) of the respective\n      superdataset.\n    includeds : sequence, optional\n      Any paths given are treated as existing subdatasets, regardless of\n      whether they can be found in the filesystem. Such subdatasets will appear\n      under the key of the closest existing dataset in the `spec`.\n\n    Returns\n    -------\n    None\n      Function calls itself recursively and populates `spec` dict in-place.\n      Keys are dataset paths, values are sets of subdataset paths\n    \"\"\"\n    # convert to set for faster lookup\n    includeds = includeds if isinstance(includeds, set) else \\\n        set() if includeds is None else set(includeds)\n    # this beast walks the directory tree from a given `basepath` until\n    # it discovers any of the given `targetpaths`\n    # if it finds one, it commits any accummulated trace of visited\n    # datasets on this edge to the spec\n    valid_repo = GitRepo.is_valid_repo(basepath)\n    if valid_repo:\n        # we are passing into a new dataset, extend the dataset trace\n        current_trace = current_trace + [basepath]\n    # this edge is not done, we need to try to reach any downstream\n    # dataset\n    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)\n    # whether anything in this directory matched a targetpath\n    filematch = False\n    if isdir(basepath):\n        for p in listdir(basepath):\n            p = assure_unicode(opj(basepath, p))\n            if not isdir(p):\n                if p in targetpaths:\n                    filematch = True\n                # we cannot have anything below this one\n                continue\n            # OPT listdir might be large and we could have only few items\n            # in `targetpaths` -- so traverse only those in spec which have\n            # leading dir basepath\n            # filter targets matching this downward path\n            downward_targets = set(\n                t for t in targetpaths if path_startswith(t, p))\n            if not downward_targets:\n                continue\n            # remove the matching ones from the \"todo\" list\n            undiscovered_ds.difference_update(downward_targets)\n            # go one deeper\n            discover_dataset_trace_to_targets(\n                p, downward_targets, current_trace, spec,\n                includeds=includeds if not includeds else includeds.intersection(\n                    downward_targets))\n    undiscovered_ds = [t for t in undiscovered_ds\n                       if includeds and\n                          path_is_subpath(t, current_trace[-1]) and\n                          t in includeds]\n    if filematch or basepath in targetpaths or undiscovered_ds:\n        for i, p in enumerate(current_trace[:-1]):\n            # TODO RF prepare proper annotated path dicts\n            subds = spec.get(p, set())\n            subds.add(current_trace[i + 1])\n            spec[p] = subds\n        if undiscovered_ds:\n            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(\n                undiscovered_ds)\n\n\ndef get_result_filter(fx):\n    \"\"\"Wrap a filter into a helper to be able to accept additional\n    arguments, if the filter doesn't support it already\"\"\"\n    _fx = fx\n    if fx and not getargspec(fx).keywords:\n        def _fx(res, **kwargs):\n            return fx(res)\n    return _fx\n\n\ndef eval_results(func):\n    \"\"\"Decorator for return value evaluation of datalad commands.\n\n    Note, this decorator is only compatible with commands that return\n    status dict sequences!\n\n    Two basic modes of operation are supported: 1) \"generator mode\" that\n    `yields` individual results, and 2) \"list mode\" that returns a sequence of\n    results. The behavior can be selected via the kwarg `return_type`.\n    Default is \"list mode\".\n\n    This decorator implements common functionality for result rendering/output,\n    error detection/handling, and logging.\n\n    Result rendering/output can be triggered via the\n    `datalad.api.result-renderer` configuration variable, or the\n    `result_renderer` keyword argument of each decorated command. Supported\n    modes are: 'default' (one line per result with action, status, path,\n    and an optional message); 'json' (one object per result, like git-annex),\n    'json_pp' (like 'json', but pretty-printed spanning multiple lines),\n    'tailored' custom output formatting provided by each command\n    class (if any).\n\n    Error detection works by inspecting the `status` item of all result\n    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'\n    will cause an IncompleteResultsError exception to be raised that carries\n    the failed actions' status dictionaries in its `failed` attribute.\n\n    Status messages will be logged automatically, by default the following\n    association of result status and log channel will be used: 'ok' (debug),\n    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger\n    instances included in the results are used to capture the origin of a\n    status report.\n\n    Parameters\n    ----------\n    func: function\n      __call__ method of a subclass of Interface,\n      i.e. a datalad command definition\n    \"\"\"\n\n    @wrapt.decorator\n    def eval_func(wrapped, instance, args, kwargs):\n        lgr.log(2, \"Entered eval_func for %s\", func)\n        # for result filters\n        # we need to produce a dict with argname/argvalue pairs for all args\n        # incl. defaults and args given as positionals\n        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)\n\n        # determine the command class associated with `wrapped`\n        wrapped_class = get_wrapped_class(wrapped)\n\n        # retrieve common options from kwargs, and fall back on the command\n        # class attributes, or general defaults if needed\n        kwargs = kwargs.copy()  # we will pop, which might cause side-effect\n        common_params = {\n            p_name: kwargs.pop(\n                # go with any explicitly given default\n                p_name,\n                # otherwise determine the command class and pull any\n                # default set in that class\n                getattr(\n                    wrapped_class,\n                    p_name,\n                    # or the common default\n                    eval_defaults[p_name]))\n            for p_name in eval_params}\n\n        # short cuts and configured setup for common options\n        return_type = common_params['return_type']\n        result_filter = get_result_filter(common_params['result_filter'])\n        # resolve string labels for transformers too\n        result_xfm = known_result_xfms.get(\n            common_params['result_xfm'],\n            # use verbatim, if not a known label\n            common_params['result_xfm'])\n        result_renderer = common_params['result_renderer']\n        # TODO remove this conditional branch entirely, done outside\n        if not result_renderer:\n            result_renderer = dlcfg.get('datalad.api.result-renderer', None)\n        # look for potential override of logging behavior\n        result_log_level = dlcfg.get('datalad.log.result-level', None)\n\n        # query cfg for defaults\n        # .is_installed and .config can be costly, so ensure we do\n        # it only once. See https://github.com/datalad/datalad/issues/3575\n        dataset_arg = allkwargs.get('dataset', None)\n        from datalad.distribution.dataset import Dataset\n        ds = dataset_arg if isinstance(dataset_arg, Dataset) \\\n            else Dataset(dataset_arg) if dataset_arg else None\n        # do not reuse a dataset's existing config manager here\n        # they are configured to read the committed dataset configuration\n        # too. That means a datalad update can silently bring in new\n        # procedure definitions from the outside, and in some sense enable\n        # remote code execution by a 3rd-party\n        # To avoid that, create a new config manager that only reads local\n        # config (system and .git/config), plus any overrides given to this\n        # datalad session\n        proc_cfg = ConfigManager(\n            ds, source='local', overrides=dlcfg.overrides\n        ) if ds and ds.is_installed() else dlcfg\n\n        # look for hooks\n        hooks = get_jsonhooks_from_config(proc_cfg)\n\n        # this internal helper function actually drives the command\n        # generator-style, it may generate an exception if desired,\n        # on incomplete results\n        def generator_func(*_args, **_kwargs):\n            # flag whether to raise an exception\n            incomplete_results = []\n            # track what actions were performed how many times\n            action_summary = {}\n\n            # if a custom summary is to be provided, collect the results\n            # of the command execution\n            results = []\n            do_custom_result_summary = result_renderer in ('tailored', 'default') \\\n                and hasattr(wrapped_class, 'custom_result_summary_renderer')\n\n            # process main results\n            for r in _process_results(\n                    # execution\n                    wrapped(*_args, **_kwargs),\n                    wrapped_class,\n                    common_params['on_failure'],\n                    # bookkeeping\n                    action_summary,\n                    incomplete_results,\n                    # communication\n                    result_renderer,\n                    result_log_level,\n                    # let renderers get to see how a command was called\n                    allkwargs):\n                for hook, spec in hooks.items():\n                    # run the hooks before we yield the result\n                    # this ensures that they are executed before\n                    # a potentially wrapper command gets to act\n                    # on them\n                    if match_jsonhook2result(hook, r, spec['match']):\n                        lgr.debug('Result %s matches hook %s', r, hook)\n                        # a hook is also a command that yields results\n                        # so yield them outside too\n                        # users need to pay attention to void infinite\n                        # loops, i.e. when a hook yields a result that\n                        # triggers that same hook again\n                        for hr in run_jsonhook(hook, spec, r, dataset_arg):\n                            # apply same logic as for main results, otherwise\n                            # any filters would only tackle the primary results\n                            # and a mixture of return values could happen\n                            if not keep_result(hr, result_filter, **allkwargs):\n                                continue\n                            hr = xfm_result(hr, result_xfm)\n                            # rationale for conditional is a few lines down\n                            if hr:\n                                yield hr\n                if not keep_result(r, result_filter, **allkwargs):\n                    continue\n                r = xfm_result(r, result_xfm)\n                # in case the result_xfm decided to not give us anything\n                # exclude it from the results. There is no particular reason\n                # to do so other than that it was established behavior when\n                # this comment was written. This will not affect any real\n                # result record\n                if r:\n                    yield r\n\n                # collect if summary is desired\n                if do_custom_result_summary:\n                    results.append(r)\n\n            # result summary before a potential exception\n            # custom first\n            if do_custom_result_summary:\n                wrapped_class.custom_result_summary_renderer(results)\n            elif result_renderer == 'default' and action_summary and \\\n                    sum(sum(s.values()) for s in action_summary.values()) > 1:\n                # give a summary in default mode, when there was more than one\n                # action performed\n                ui.message(\"action summary:\\n  {}\".format(\n                    '\\n  '.join('{} ({})'.format(\n                        act,\n                        ', '.join('{}: {}'.format(status, action_summary[act][status])\n                                  for status in sorted(action_summary[act])))\n                                for act in sorted(action_summary))))\n\n            if incomplete_results:\n                raise IncompleteResultsError(\n                    failed=incomplete_results,\n                    msg=\"Command did not complete successfully\")\n\n        if return_type == 'generator':\n            # hand over the generator\n            lgr.log(2, \"Returning generator_func from eval_func for %s\", wrapped_class)\n            return generator_func(*args, **kwargs)\n        else:\n            @wrapt.decorator\n            def return_func(wrapped_, instance_, args_, kwargs_):\n                results = wrapped_(*args_, **kwargs_)\n                if inspect.isgenerator(results):\n                    # unwind generator if there is one, this actually runs\n                    # any processing\n                    results = list(results)\n                # render summaries\n                if not result_xfm and result_renderer in ('tailored', 'default'):\n                    # cannot render transformed results\n                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):\n                        wrapped_class.custom_result_summary_renderer(results)\n                if return_type == 'item-or-list' and \\\n                        len(results) < 2:\n                    return results[0] if results else None\n                else:\n                    return results\n            lgr.log(2, \"Returning return_func from eval_func for %s\", wrapped_class)\n            return return_func(generator_func)(*args, **kwargs)\n\n    return eval_func(func)\n\n\ndef default_result_renderer(res):\n    if res.get('status', None) != 'notneeded':\n        path = res.get('path', None)\n        path = ' {}'.format(path) if path else ''\n        ui.message('{action}({status}):{path}{type}{msg}'.format(\n                action=ac.color_word(res['action'], ac.BOLD),\n                status=ac.color_status(res['status']),\n                path=relpath(path, res['refds']) if path and res.get('refds') else path,\n                type=' ({})'.format(\n                        ac.color_word(res['type'], ac.MAGENTA)\n                ) if 'type' in res else '',\n                msg=' [{}]'.format(\n                        res['message'][0] % res['message'][1:]\n                        if isinstance(res['message'], tuple) else res[\n                            'message'])\n                if res.get('message', None) else ''))\n\n\ndef _display_suppressed_message(nsimilar, ndisplayed, final=False):\n    # +1 because there was the original result + nsimilar displayed.\n    n_suppressed = nsimilar - ndisplayed + 1\n    if n_suppressed > 0:\n        ui.message('  [{} similar {} been suppressed]'\n                   .format(n_suppressed,\n                           single_or_plural(\"message has\",\n                                            \"messages have\",\n                                            n_suppressed, False)),\n                   cr=\"\\n\" if final else \"\\r\")\n\n\ndef _process_results(\n        results,\n        cmd_class,\n        on_failure,\n        action_summary,\n        incomplete_results,\n        result_renderer,\n        result_log_level,\n        allkwargs):\n    # private helper pf @eval_results\n    # loop over results generated from some source and handle each\n    # of them according to the requested behavior (logging, rendering, ...)\n\n    # used to track repeated messages in the default renderer\n    last_result = None\n    # which result dict keys to inspect for changes to discover repetions\n    # of similar messages\n    repetition_keys = set(('action', 'status', 'type', 'refds'))\n    # counter for detected repetitions\n    result_repetitions = 0\n    # how many repetitions to show, before suppression kicks in\n    render_n_repetitions = 10 if sys.stdout.isatty() else float(\"inf\")\n\n    for res in results:\n        if not res or 'action' not in res:\n            # XXX Yarik has to no clue on how to track the origin of the\n            # record to figure out WTF, so he just skips it\n            continue\n\n        actsum = action_summary.get(res['action'], {})\n        if res['status']:\n            actsum[res['status']] = actsum.get(res['status'], 0) + 1\n            action_summary[res['action']] = actsum\n        ## log message, if there is one and a logger was given\n        msg = res.get('message', None)\n        # remove logger instance from results, as it is no longer useful\n        # after logging was done, it isn't serializable, and generally\n        # pollutes the output\n        res_lgr = res.pop('logger', None)\n        if msg and res_lgr:\n            if isinstance(res_lgr, logging.Logger):\n                # didn't get a particular log function, go with default\n                res_lgr = getattr(\n                    res_lgr,\n                    default_logchannels[res['status']]\n                    if result_log_level is None\n                    else result_log_level)\n            msg = res['message']\n            msgargs = None\n            if isinstance(msg, tuple):\n                msgargs = msg[1:]\n                msg = msg[0]\n            if 'path' in res:\n                msg = '{} [{}({})]'.format(\n                    msg, res['action'], res['path'])\n            if msgargs:\n                # support string expansion of logging to avoid runtime cost\n                res_lgr(msg, *msgargs)\n            else:\n                res_lgr(msg)\n\n        ## output rendering\n        # TODO RF this in a simple callable that gets passed into this function\n        if result_renderer is None or result_renderer == 'disabled':\n            pass\n        elif result_renderer == 'default':\n            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}\n            if res.get('status', None) != 'notneeded' \\\n                    and trimmed_result == last_result:\n                # this is a similar report, suppress if too many, but count it\n                result_repetitions += 1\n                if result_repetitions < render_n_repetitions:\n                    default_result_renderer(res)\n                else:\n                    _display_suppressed_message(\n                        result_repetitions, render_n_repetitions)\n            else:\n                # this one is new, first report on any prev. suppressed results\n                # by number, and then render this fresh one\n                _display_suppressed_message(\n                    result_repetitions, render_n_repetitions,\n                    final=True)\n                default_result_renderer(res)\n                result_repetitions = 0\n            last_result = trimmed_result\n        elif result_renderer in ('json', 'json_pp'):\n            ui.message(json.dumps(\n                {k: v for k, v in res.items()\n                 if k not in ('message', 'logger')},\n                sort_keys=True,\n                indent=2 if result_renderer.endswith('_pp') else None,\n                default=lambda x: str(x)))\n        elif result_renderer in ('tailored', 'default'):\n            if hasattr(cmd_class, 'custom_result_renderer'):\n                cmd_class.custom_result_renderer(res, **allkwargs)\n        elif hasattr(result_renderer, '__call__'):\n            try:\n                result_renderer(res, **allkwargs)\n            except Exception as e:\n                lgr.warning('Result rendering failed for: %s [%s]',\n                            res, exc_str(e))\n        else:\n            raise ValueError('unknown result renderer \"{}\"'.format(result_renderer))\n\n        ## error handling\n        # looks for error status, and report at the end via\n        # an exception\n        if on_failure in ('continue', 'stop') \\\n                and res['status'] in ('impossible', 'error'):\n            incomplete_results.append(res)\n            if on_failure == 'stop':\n                # first fail -> that's it\n                # raise will happen after the loop\n                break\n        yield res\n    # make sure to report on any issues that we had suppressed\n    _display_suppressed_message(\n        result_repetitions, render_n_repetitions, final=True)\n\n\ndef keep_result(res, rfilter, **kwargs):\n    if not rfilter:\n        return True\n    try:\n        if not rfilter(res, **kwargs):\n            # give the slightest indication which filter was employed\n            raise ValueError(\n                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))\n    except ValueError as e:\n        # make sure to report the excluded result to massively improve\n        # debugging experience\n        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)\n        return False\n    return True\n\n\ndef xfm_result(res, xfm):\n    if not xfm:\n        return res\n\n    return xfm(res)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Uninformative TypeError Raised During Message Rendering with Insufficient Arguments

**Description:**
When logging messages that include format placeholders, providing fewer arguments than required results in a generic `TypeError`. This makes it difficult to identify and debug issues related to message formatting.

**Example Code:**
```python
def logger(msg, *args):
    return msg % args

# This will raise a TypeError due to insufficient arguments for the format string
logger("Operation failed at %s with error code %s", "ModuleA")
```

**Expected Behavior:**
A clear and descriptive `TypeError` indicating that there are not enough arguments provided for the format string, helping developers quickly identify the mismatch.

**Actual Behavior:**
A generic `TypeError` without specific details about the formatting issue, making it harder to trace and fix the problem.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/utils.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface utility functions

"""

__docformat__ = 'restructuredtext'

import inspect
import logging
import wrapt
import sys
import re
from os import curdir
from os import pardir
from os import listdir
import os.path as op
from os.path import join as opj
from os.path import isdir
from os.path import relpath
from os.path import sep
from os.path import split as psplit
from itertools import chain

import json

# avoid import from API to not get into circular imports
from datalad.utils import with_pathsep as _with_sep  # TODO: RF whenever merge conflict is not upon us
from datalad.utils import (
    path_startswith,
    path_is_subpath,
    assure_unicode,
    getargspec,
    get_wrapped_class,
)
from datalad.support.gitrepo import GitRepo
from datalad.support.exceptions import IncompleteResultsError
from datalad import cfg as dlcfg
from datalad.dochelpers import (
    exc_str,
    single_or_plural,
)

from datalad.ui import ui
import datalad.support.ansi_colors as ac

from datalad.interface.base import default_logchannels
from datalad.interface.base import get_allargs_as_kwargs
from datalad.interface.common_opts import eval_params
from datalad.interface.common_opts import eval_defaults
from .results import known_result_xfms
from datalad.config import ConfigManager
from datalad.core.local.resulthooks import (
    get_jsonhooks_from_config,
    match_jsonhook2result,
    run_jsonhook,
)


lgr = logging.getLogger('datalad.interface.utils')


def cls2cmdlinename(cls):
    "Return the cmdline command name from an Interface class"
    r = re.compile(r'([a-z0-9])([A-Z])')
    return r.sub('\\1-\\2', cls.__name__).lower()


# TODO remove
# only `drop` and `uninstall` are still using this
def handle_dirty_dataset(ds, mode, msg=None):
    """Detect and treat unsaved changes as instructed by `mode`

    Parameters
    ----------
    ds : Dataset or None
      Dataset to be inspected. Does nothing if `None`.
    mode : {'fail', 'ignore', 'save-before'}
      How to act upon discovering unsaved changes.
    msg : str or None
      Custom message to use for a potential commit.

    Returns
    -------
    None
    """
    if ds is None:
        # nothing to be handled
        return
    if msg is None:
        msg = '[DATALAD] auto-saved changes'

    # make sure that all pending changes (batched annex operations, etc.)
    # are actually reflected in Git
    if ds.repo:
        ds.repo.precommit()

    if mode == 'ignore':
        return
    elif mode == 'fail':
        if not ds.repo or ds.repo.dirty:
            raise RuntimeError('dataset {} has unsaved changes'.format(ds))
    elif mode == 'save-before':
        if not ds.is_installed():
            raise RuntimeError('dataset {} is not yet installed'.format(ds))
        from datalad.core.local.save import Save
        Save.__call__(dataset=ds, message=msg, updated=True)
    else:
        raise ValueError("unknown if-dirty mode '{}'".format(mode))


def get_tree_roots(paths):
    """Return common root paths for a set of paths

    This function determines the smallest set of common root
    paths and sorts all given paths under the respective
    root.

    Returns
    -------
    dict
      paths by root
    """
    paths_ws = [_with_sep(p) for p in paths]
    # sort all paths under their potential roots
    roots = {}
    # start from the top to get all paths down the line
    # and collate them into as few roots as possible
    for s in sorted(paths_ws):
        if any([s.startswith(r) for r in roots]):
            # this path is already covered by a known root
            continue
        # find all sub paths
        subs = [p for p in paths if p.startswith(s)]
        roots[s.rstrip(sep)] = subs
    return roots


# TODO remove
# only `remove` and `uninstall` use this, the uses path `path_is_subpath`
def path_is_under(values, path=None):
    """Whether a given path is a subdirectory of any of the given test values

    Parameters
    ----------
    values : sequence or dict
      Paths to be tested against. This can be a dictionary in which case
      all values from all keys will be tested against.
    path : path or None
      Test path. If None is given, the process' working directory is
      used.

    Returns
    -------
    bool
    """
    if path is None:
        from datalad.utils import getpwd
        path = getpwd()
    if isinstance(values, dict):
        values = chain(*values.values())
    path_drive, _ = op.splitdrive(path)
    for p in values:
        p_drive, _ = op.splitdrive(p)
        # need to protect against unsupported use of relpath() with
        # abspaths on windows from different drives (gh-3724)
        if path_drive != p_drive:
        # different drives, enough evidence for "not under"
            continue
        rpath = relpath(p, start=path)
        if rpath == curdir \
                or rpath == pardir \
                or set(psplit(rpath)) == {pardir}:
            # first match is enough
            return True
    return False


def discover_dataset_trace_to_targets(basepath, targetpaths, current_trace,
                                      spec, includeds=None):
    """Discover the edges and nodes in a dataset tree to given target paths

    Parameters
    ----------
    basepath : path
      Path to a start or top-level dataset. Really has to be a path to a
      dataset!
    targetpaths : list(path)
      Any non-zero number of paths that are termination points for the
      search algorithm. Can be paths to datasets, directories, or files
      (and any combination thereof).
    current_trace : list
      For a top-level call this should probably always be `[]`
    spec : dict
      `content_by_ds`-style dictionary that will receive information about the
      discovered datasets. Specifically, for each discovered dataset there
      will be in item with its path under the key (path) of the respective
      superdataset.
    includeds : sequence, optional
      Any paths given are treated as existing subdatasets, regardless of
      whether they can be found in the filesystem. Such subdatasets will appear
      under the key of the closest existing dataset in the `spec`.

    Returns
    -------
    None
      Function calls itself recursively and populates `spec` dict in-place.
      Keys are dataset paths, values are sets of subdataset paths
    """
    # convert to set for faster lookup
    includeds = includeds if isinstance(includeds, set) else \
        set() if includeds is None else set(includeds)
    # this beast walks the directory tree from a given `basepath` until
    # it discovers any of the given `targetpaths`
    # if it finds one, it commits any accummulated trace of visited
    # datasets on this edge to the spec
    valid_repo = GitRepo.is_valid_repo(basepath)
    if valid_repo:
        # we are passing into a new dataset, extend the dataset trace
        current_trace = current_trace + [basepath]
    # this edge is not done, we need to try to reach any downstream
    # dataset
    undiscovered_ds = set(t for t in targetpaths) # if t != basepath)
    # whether anything in this directory matched a targetpath
    filematch = False
    if isdir(basepath):
        for p in listdir(basepath):
            p = assure_unicode(opj(basepath, p))
            if not isdir(p):
                if p in targetpaths:
                    filematch = True
                # we cannot have anything below this one
                continue
            # OPT listdir might be large and we could have only few items
            # in `targetpaths` -- so traverse only those in spec which have
            # leading dir basepath
            # filter targets matching this downward path
            downward_targets = set(
                t for t in targetpaths if path_startswith(t, p))
            if not downward_targets:
                continue
            # remove the matching ones from the "todo" list
            undiscovered_ds.difference_update(downward_targets)
            # go one deeper
            discover_dataset_trace_to_targets(
                p, downward_targets, current_trace, spec,
                includeds=includeds if not includeds else includeds.intersection(
                    downward_targets))
    undiscovered_ds = [t for t in undiscovered_ds
                       if includeds and
                          path_is_subpath(t, current_trace[-1]) and
                          t in includeds]
    if filematch or basepath in targetpaths or undiscovered_ds:
        for i, p in enumerate(current_trace[:-1]):
            # TODO RF prepare proper annotated path dicts
            subds = spec.get(p, set())
            subds.add(current_trace[i + 1])
            spec[p] = subds
        if undiscovered_ds:
            spec[current_trace[-1]] = spec.get(current_trace[-1], set()).union(
                undiscovered_ds)


def get_result_filter(fx):
    """Wrap a filter into a helper to be able to accept additional
    arguments, if the filter doesn't support it already"""
    _fx = fx
    if fx and not getargspec(fx).keywords:
        def _fx(res, **kwargs):
            return fx(res)
    return _fx


def eval_results(func):
    """Decorator for return value evaluation of datalad commands.

    Note, this decorator is only compatible with commands that return
    status dict sequences!

    Two basic modes of operation are supported: 1) "generator mode" that
    `yields` individual results, and 2) "list mode" that returns a sequence of
    results. The behavior can be selected via the kwarg `return_type`.
    Default is "list mode".

    This decorator implements common functionality for result rendering/output,
    error detection/handling, and logging.

    Result rendering/output can be triggered via the
    `datalad.api.result-renderer` configuration variable, or the
    `result_renderer` keyword argument of each decorated command. Supported
    modes are: 'default' (one line per result with action, status, path,
    and an optional message); 'json' (one object per result, like git-annex),
    'json_pp' (like 'json', but pretty-printed spanning multiple lines),
    'tailored' custom output formatting provided by each command
    class (if any).

    Error detection works by inspecting the `status` item of all result
    dictionaries. Any occurrence of a status other than 'ok' or 'notneeded'
    will cause an IncompleteResultsError exception to be raised that carries
    the failed actions' status dictionaries in its `failed` attribute.

    Status messages will be logged automatically, by default the following
    association of result status and log channel will be used: 'ok' (debug),
    'notneeded' (debug), 'impossible' (warning), 'error' (error).  Logger
    instances included in the results are used to capture the origin of a
    status report.

    Parameters
    ----------
    func: function
      __call__ method of a subclass of Interface,
      i.e. a datalad command definition
    """

    @wrapt.decorator
    def eval_func(wrapped, instance, args, kwargs):
        lgr.log(2, "Entered eval_func for %s", func)
        # for result filters
        # we need to produce a dict with argname/argvalue pairs for all args
        # incl. defaults and args given as positionals
        allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)

        # determine the command class associated with `wrapped`
        wrapped_class = get_wrapped_class(wrapped)

        # retrieve common options from kwargs, and fall back on the command
        # class attributes, or general defaults if needed
        kwargs = kwargs.copy()  # we will pop, which might cause side-effect
        common_params = {
            p_name: kwargs.pop(
                # go with any explicitly given default
                p_name,
                # otherwise determine the command class and pull any
                # default set in that class
                getattr(
                    wrapped_class,
                    p_name,
                    # or the common default
                    eval_defaults[p_name]))
            for p_name in eval_params}

        # short cuts and configured setup for common options
        return_type = common_params['return_type']
        result_filter = get_result_filter(common_params['result_filter'])
        # resolve string labels for transformers too
        result_xfm = known_result_xfms.get(
            common_params['result_xfm'],
            # use verbatim, if not a known label
            common_params['result_xfm'])
        result_renderer = common_params['result_renderer']
        # TODO remove this conditional branch entirely, done outside
        if not result_renderer:
            result_renderer = dlcfg.get('datalad.api.result-renderer', None)
        # look for potential override of logging behavior
        result_log_level = dlcfg.get('datalad.log.result-level', None)

        # query cfg for defaults
        # .is_installed and .config can be costly, so ensure we do
        # it only once. See https://github.com/datalad/datalad/issues/3575
        dataset_arg = allkwargs.get('dataset', None)
        from datalad.distribution.dataset import Dataset
        ds = dataset_arg if isinstance(dataset_arg, Dataset) \
            else Dataset(dataset_arg) if dataset_arg else None
        # do not reuse a dataset's existing config manager here
        # they are configured to read the committed dataset configuration
        # too. That means a datalad update can silently bring in new
        # procedure definitions from the outside, and in some sense enable
        # remote code execution by a 3rd-party
        # To avoid that, create a new config manager that only reads local
        # config (system and .git/config), plus any overrides given to this
        # datalad session
        proc_cfg = ConfigManager(
            ds, source='local', overrides=dlcfg.overrides
        ) if ds and ds.is_installed() else dlcfg

        # look for hooks
        hooks = get_jsonhooks_from_config(proc_cfg)

        # this internal helper function actually drives the command
        # generator-style, it may generate an exception if desired,
        # on incomplete results
        def generator_func(*_args, **_kwargs):
            # flag whether to raise an exception
            incomplete_results = []
            # track what actions were performed how many times
            action_summary = {}

            # if a custom summary is to be provided, collect the results
            # of the command execution
            results = []
            do_custom_result_summary = result_renderer in ('tailored', 'default') \
                and hasattr(wrapped_class, 'custom_result_summary_renderer')

            # process main results
            for r in _process_results(
                    # execution
                    wrapped(*_args, **_kwargs),
                    wrapped_class,
                    common_params['on_failure'],
                    # bookkeeping
                    action_summary,
                    incomplete_results,
                    # communication
                    result_renderer,
                    result_log_level,
                    # let renderers get to see how a command was called
                    allkwargs):
                for hook, spec in hooks.items():
                    # run the hooks before we yield the result
                    # this ensures that they are executed before
                    # a potentially wrapper command gets to act
                    # on them
                    if match_jsonhook2result(hook, r, spec['match']):
                        lgr.debug('Result %s matches hook %s', r, hook)
                        # a hook is also a command that yields results
                        # so yield them outside too
                        # users need to pay attention to void infinite
                        # loops, i.e. when a hook yields a result that
                        # triggers that same hook again
                        for hr in run_jsonhook(hook, spec, r, dataset_arg):
                            # apply same logic as for main results, otherwise
                            # any filters would only tackle the primary results
                            # and a mixture of return values could happen
                            if not keep_result(hr, result_filter, **allkwargs):
                                continue
                            hr = xfm_result(hr, result_xfm)
                            # rationale for conditional is a few lines down
                            if hr:
                                yield hr
                if not keep_result(r, result_filter, **allkwargs):
                    continue
                r = xfm_result(r, result_xfm)
                # in case the result_xfm decided to not give us anything
                # exclude it from the results. There is no particular reason
                # to do so other than that it was established behavior when
                # this comment was written. This will not affect any real
                # result record
                if r:
                    yield r

                # collect if summary is desired
                if do_custom_result_summary:
                    results.append(r)

            # result summary before a potential exception
            # custom first
            if do_custom_result_summary:
                wrapped_class.custom_result_summary_renderer(results)
            elif result_renderer == 'default' and action_summary and \
                    sum(sum(s.values()) for s in action_summary.values()) > 1:
                # give a summary in default mode, when there was more than one
                # action performed
                ui.message("action summary:\n  {}".format(
                    '\n  '.join('{} ({})'.format(
                        act,
                        ', '.join('{}: {}'.format(status, action_summary[act][status])
                                  for status in sorted(action_summary[act])))
                                for act in sorted(action_summary))))

            if incomplete_results:
                raise IncompleteResultsError(
                    failed=incomplete_results,
                    msg="Command did not complete successfully")

        if return_type == 'generator':
            # hand over the generator
            lgr.log(2, "Returning generator_func from eval_func for %s", wrapped_class)
            return generator_func(*args, **kwargs)
        else:
            @wrapt.decorator
            def return_func(wrapped_, instance_, args_, kwargs_):
                results = wrapped_(*args_, **kwargs_)
                if inspect.isgenerator(results):
                    # unwind generator if there is one, this actually runs
                    # any processing
                    results = list(results)
                # render summaries
                if not result_xfm and result_renderer in ('tailored', 'default'):
                    # cannot render transformed results
                    if hasattr(wrapped_class, 'custom_result_summary_renderer'):
                        wrapped_class.custom_result_summary_renderer(results)
                if return_type == 'item-or-list' and \
                        len(results) < 2:
                    return results[0] if results else None
                else:
                    return results
            lgr.log(2, "Returning return_func from eval_func for %s", wrapped_class)
            return return_func(generator_func)(*args, **kwargs)

    return eval_func(func)


def default_result_renderer(res):
    if res.get('status', None) != 'notneeded':
        path = res.get('path', None)
        path = ' {}'.format(path) if path else ''
        ui.message('{action}({status}):{path}{type}{msg}'.format(
                action=ac.color_word(res['action'], ac.BOLD),
                status=ac.color_status(res['status']),
                path=relpath(path, res['refds']) if path and res.get('refds') else path,
                type=' ({})'.format(
                        ac.color_word(res['type'], ac.MAGENTA)
                ) if 'type' in res else '',
                msg=' [{}]'.format(
                        res['message'][0] % res['message'][1:]
                        if isinstance(res['message'], tuple) else res[
                            'message'])
                if res.get('message', None) else ''))


def _display_suppressed_message(nsimilar, ndisplayed, final=False):
    # +1 because there was the original result + nsimilar displayed.
    n_suppressed = nsimilar - ndisplayed + 1
    if n_suppressed > 0:
        ui.message('  [{} similar {} been suppressed]'
                   .format(n_suppressed,
                           single_or_plural("message has",
                                            "messages have",
                                            n_suppressed, False)),
                   cr="\n" if final else "\r")


def _process_results(
        results,
        cmd_class,
        on_failure,
        action_summary,
        incomplete_results,
        result_renderer,
        result_log_level,
        allkwargs):
    # private helper pf @eval_results
    # loop over results generated from some source and handle each
    # of them according to the requested behavior (logging, rendering, ...)

    # used to track repeated messages in the default renderer
    last_result = None
    # which result dict keys to inspect for changes to discover repetions
    # of similar messages
    repetition_keys = set(('action', 'status', 'type', 'refds'))
    # counter for detected repetitions
    result_repetitions = 0
    # how many repetitions to show, before suppression kicks in
    render_n_repetitions = 10 if sys.stdout.isatty() else float("inf")

    for res in results:
        if not res or 'action' not in res:
            # XXX Yarik has to no clue on how to track the origin of the
            # record to figure out WTF, so he just skips it
            continue

        actsum = action_summary.get(res['action'], {})
        if res['status']:
            actsum[res['status']] = actsum.get(res['status'], 0) + 1
            action_summary[res['action']] = actsum
        ## log message, if there is one and a logger was given
        msg = res.get('message', None)
        # remove logger instance from results, as it is no longer useful
        # after logging was done, it isn't serializable, and generally
        # pollutes the output
        res_lgr = res.pop('logger', None)
        if msg and res_lgr:
            if isinstance(res_lgr, logging.Logger):
                # didn't get a particular log function, go with default
                res_lgr = getattr(
                    res_lgr,
                    default_logchannels[res['status']]
                    if result_log_level is None
                    else result_log_level)
            msg = res['message']
            msgargs = None
            if isinstance(msg, tuple):
                msgargs = msg[1:]
                msg = msg[0]
            if 'path' in res:
                msg = '{} [{}({})]'.format(
                    msg, res['action'], res['path'])
            if msgargs:
                # support string expansion of logging to avoid runtime cost
                res_lgr(msg, *msgargs)
            else:
                res_lgr(msg)

        ## output rendering
        # TODO RF this in a simple callable that gets passed into this function
        if result_renderer is None or result_renderer == 'disabled':
            pass
        elif result_renderer == 'default':
            trimmed_result = {k: v for k, v in res.items() if k in repetition_keys}
            if res.get('status', None) != 'notneeded' \
                    and trimmed_result == last_result:
                # this is a similar report, suppress if too many, but count it
                result_repetitions += 1
                if result_repetitions < render_n_repetitions:
                    default_result_renderer(res)
                else:
                    _display_suppressed_message(
                        result_repetitions, render_n_repetitions)
            else:
                # this one is new, first report on any prev. suppressed results
                # by number, and then render this fresh one
                _display_suppressed_message(
                    result_repetitions, render_n_repetitions,
                    final=True)
                default_result_renderer(res)
                result_repetitions = 0
            last_result = trimmed_result
        elif result_renderer in ('json', 'json_pp'):
            ui.message(json.dumps(
                {k: v for k, v in res.items()
                 if k not in ('message', 'logger')},
                sort_keys=True,
                indent=2 if result_renderer.endswith('_pp') else None,
                default=lambda x: str(x)))
        elif result_renderer in ('tailored', 'default'):
            if hasattr(cmd_class, 'custom_result_renderer'):
                cmd_class.custom_result_renderer(res, **allkwargs)
        elif hasattr(result_renderer, '__call__'):
            try:
                result_renderer(res, **allkwargs)
            except Exception as e:
                lgr.warning('Result rendering failed for: %s [%s]',
                            res, exc_str(e))
        else:
            raise ValueError('unknown result renderer "{}"'.format(result_renderer))

        ## error handling
        # looks for error status, and report at the end via
        # an exception
        if on_failure in ('continue', 'stop') \
                and res['status'] in ('impossible', 'error'):
            incomplete_results.append(res)
            if on_failure == 'stop':
                # first fail -> that's it
                # raise will happen after the loop
                break
        yield res
    # make sure to report on any issues that we had suppressed
    _display_suppressed_message(
        result_repetitions, render_n_repetitions, final=True)


def keep_result(res, rfilter, **kwargs):
    if not rfilter:
        return True
    try:
        if not rfilter(res, **kwargs):
            # give the slightest indication which filter was employed
            raise ValueError(
                'excluded by filter {} with arguments {}'.format(rfilter, kwargs))
    except ValueError as e:
        # make sure to report the excluded result to massively improve
        # debugging experience
        lgr.debug('Not reporting result (%s): %s', exc_str(e), res)
        return False
    return True


def xfm_result(res, xfm):
    if not xfm:
        return res

    return xfm(res)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user