{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport sys\nimport re\nimport textwrap\nimport inspect\nfrom collections import OrderedDict\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import EnsureKeyChoice\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups():\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    return grps\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| CMDLINE \\>\\>.*\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        '\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        '\\|\\| PYTHON \\>\\>(.*)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        '\\|\\| REFLOW \\>\\>\\n(.*)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| PYTHON \\>\\>.*\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        '\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        '\\|\\| CMDLINE \\>\\>(.*)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        ',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        '`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        '\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        '\\|\\| REFLOW \\>\\>\\n(.*)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from inspect import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None\n    )\n\n    # return original\n    return cls\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'DownloadURL', 'Export', 'Ls', 'Move', 'SSHRun',\n        'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from inspect import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help[-1] != '.':\n                help += '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n            if defaults_idx >= 0:\n                help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from inspect import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k) for k in argnames if is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = \\\n                    lambda x, **kwargs: ui.message(args.common_output_format.format(\n                        **{k: {k_.replace(':', '#'): v_ for k_, v_ in v.items()}\n                           if isinstance(v, dict) else v\n                           for k, v in x.items()}))\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            if args.common_output_format == 'tailored' and \\\n                    hasattr(cls, 'custom_result_summary_renderer'):\n                cls.custom_result_summary_renderer(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = resolve_path(refds_path)\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from inspect import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nlgr = logging.getLogger('datalad.interface.base')\n\nimport sys\nimport re\nimport textwrap\nimport inspect\nfrom collections import OrderedDict\n\nfrom ..ui import ui\nfrom ..dochelpers import exc_str\n\nfrom datalad.interface.common_opts import eval_params\nfrom datalad.interface.common_opts import eval_defaults\nfrom datalad.support.constraints import EnsureKeyChoice\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.distribution.dataset import resolve_path\n\n\ndefault_logchannels = {\n    '': 'debug',\n    'ok': 'debug',\n    'notneeded': 'debug',\n    'impossible': 'warning',\n    'error': 'error',\n}\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups():\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    return grps\n\n\ndef dedent_docstring(text):\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| CMDLINE \\>\\>.*\\<\\< CMDLINE \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[CMD:\\s[^\\[\\]]*\\sCMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        '\\[PY:\\s([^\\[\\]]*)\\sPY\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        '\\|\\| PYTHON \\>\\>(.*)\\<\\< PYTHON \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        '\\|\\| REFLOW \\>\\>\\n(.*)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| PYTHON \\>\\>.*\\<\\< PYTHON \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[PY:\\s[^\\[\\]]*\\sPY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    docs = re.sub(\n        '\\[CMD:\\s([^\\[\\]]*)\\sCMD\\]',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        '\\|\\| CMDLINE \\>\\>(.*)\\<\\< CMDLINE \\|\\|',\n        lambda match: match.group(1),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        ',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        '`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    # clean up sphinx API refs\n    docs = re.sub(\n        '\\~datalad\\.api\\.\\S*',\n        lambda match: \"`{0}`\".format(match.group(0)[13:]),\n        docs)\n    # Remove RST paragraph markup\n    docs = re.sub(\n        r'^.. \\S+::',\n        lambda match: match.group(0)[3:-2].upper(),\n        docs,\n        flags=re.MULTILINE)\n    docs = re.sub(\n        '\\|\\| REFLOW \\>\\>\\n(.*)\\<\\< REFLOW \\|\\|',\n        lambda match: textwrap.fill(match.group(1)),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef is_api_arg(arg):\n    \"\"\"Return True if argument is our API argument or self or used for internal\n    purposes\n    \"\"\"\n    return arg != 'self' and not arg.startswith('_')\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None,\n                                     add_args=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from inspect import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if add_args:\n        add_argnames = sorted(add_args.keys())\n        args.extend(add_argnames)\n        defaults = defaults + tuple(add_args[k] for k in add_argnames)\n    if defaults is not None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if param.constraints is not None:\n                    param.constraints(defaults[defaults_idx])\n            orig_docs = param._doc\n            param._doc = alter_interface_docs_for_api(param._doc)\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            param._doc = orig_docs\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\ndef build_doc(cls, **kwargs):\n    \"\"\"Decorator to build docstrings for datalad commands\n\n    It's intended to decorate the class, the __call__-method of which is the\n    actual command. It expects that __call__-method to be decorated by\n    eval_results.\n\n    Parameters\n    ----------\n    cls: Interface\n      class defining a datalad command\n    \"\"\"\n\n    # Note, that this is a class decorator, which is executed only once when the\n    # class is imported. It builds the docstring for the class' __call__ method\n    # and returns the original class.\n    #\n    # This is because a decorator for the actual function would not be able to\n    # behave like this. To build the docstring we need to access the attribute\n    # _params of the class. From within a function decorator we cannot do this\n    # during import time, since the class is being built in this very moment and\n    # is not yet available in the module. And if we do it from within the part\n    # of a function decorator, that is executed when the function is called, we\n    # would need to actually call the command once in order to build this\n    # docstring.\n\n    lgr.debug(\"Building doc for {}\".format(cls))\n\n    cls_doc = cls.__doc__\n    if hasattr(cls, '_docs_'):\n        # expand docs\n        cls_doc = cls_doc.format(**cls._docs_)\n\n    call_doc = None\n    # suffix for update_docstring_with_parameters:\n    if cls.__call__.__doc__:\n        call_doc = cls.__call__.__doc__\n\n    # build standard doc and insert eval_doc\n    spec = getattr(cls, '_params_', dict())\n    # get docs for eval_results parameters:\n    spec.update(eval_params)\n\n    update_docstring_with_parameters(\n        cls.__call__, spec,\n        prefix=alter_interface_docs_for_api(cls_doc),\n        suffix=alter_interface_docs_for_api(call_doc),\n        add_args=eval_defaults if not hasattr(cls, '_no_eval_results') else None\n    )\n\n    # return original\n    return cls\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'Export', 'Ls', 'Move', 'SSHRun', 'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from inspect import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help[-1] != '.':\n                help += '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n            if defaults_idx >= 0:\n                help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from inspect import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k) for k in argnames if is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = \\\n                    lambda x, **kwargs: ui.message(args.common_output_format.format(\n                        **{k: {k_.replace(':', '#'): v_ for k_, v_ in v.items()}\n                           if isinstance(v, dict) else v\n                           for k, v in x.items()}))\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            if args.common_output_format == 'tailored' and \\\n                    hasattr(cls, 'custom_result_summary_renderer'):\n                cls.custom_result_summary_renderer(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = resolve_path(refds_path)\n        return refds_path\n\n\ndef get_allargs_as_kwargs(call, args, kwargs):\n    \"\"\"Generate a kwargs dict from a call signature and *args, **kwargs\n\n    Basically resolving the argnames for all positional arguments, and\n    resolvin the defaults for all kwargs that are not given in a kwargs\n    dict\n    \"\"\"\n    from inspect import getargspec\n    argspec = getargspec(call)\n    defaults = argspec.defaults\n    nargs = len(argspec.args)\n    assert (nargs >= len(defaults))\n    # map any args to their name\n    argmap = list(zip(argspec.args[:len(args)], args))\n    kwargs_ = OrderedDict(argmap)\n    # map defaults of kwargs to their names (update below)\n    for k, v in zip(argspec.args[-len(defaults):], defaults):\n        if k not in kwargs_:\n            kwargs_[k] = v\n    # update with provided kwarg args\n    kwargs_.update(kwargs)\n    # XXX we cannot assert the following, because our own highlevel\n    # API commands support more kwargs than what is discoverable\n    # from their signature...\n    #assert (nargs == len(kwargs_))\n    return kwargs_\n",
            "header": {
                "file": {
                    "path": "datalad/interface/base.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "bed8339b8",
                "new_commit_hash": "9e217a252",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/base.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/base.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 317,
                            "length": 8
                        },
                        "new_range": {
                            "start": 317,
                            "length": 7
                        },
                        "section": "class Interface(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    _OLDSTYLE_COMMANDS = (",
                                "type": "context"
                            },
                            {
                                "content": "        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',",
                                "type": "context"
                            },
                            {
                                "content": "        'CreateTestDataset', 'DownloadURL', 'Export', 'Ls', 'Move', 'SSHRun',",
                                "type": "deleted"
                            },
                            {
                                "content": "        'Test')",
                                "type": "deleted"
                            },
                            {
                                "content": "        'CreateTestDataset', 'Export', 'Ls', 'Move', 'SSHRun', 'Test')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    @classmethod",
                                "type": "context"
                            },
                            {
                                "content": "    def setup_parser(cls, parser):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/base.py",
                            "type": "class",
                            "name": "Interface",
                            "content": "class Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    # exit code to return if user-interrupted\n    # if None, would just reraise the Exception, so if in --dbg\n    # mode would fall into the debugger\n    _interrupted_exit_code = 1\n\n    _OLDSTYLE_COMMANDS = (\n        'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',\n        'CreateTestDataset', 'DownloadURL', 'Export', 'Ls', 'Move', 'SSHRun',\n        'Test')\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from inspect import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if not is_api_arg(arg):\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help[-1] != '.':\n                help += '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value constraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n            if defaults_idx >= 0:\n                help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from inspect import getargspec\n        argspec = getargspec(cls.__call__)\n        if argspec[2] is None:\n            # no **kwargs in the call receiver, pull argnames from signature\n            argnames = getargspec(cls.__call__)[0]\n        else:\n            # common options\n            # XXX define or better get from elsewhere\n            common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',\n                           'help', 'log_level', 'logger', 'pbs_runner',\n                           'result_renderer', 'subparser')\n            argnames = [name for name in dir(args)\n                        if not (name.startswith('_') or name in common_opts)]\n        kwargs = {k: getattr(args, k) for k in argnames if is_api_arg(k)}\n        # we are coming from the entry point, this is the toplevel command,\n        # let it run like generator so we can act on partial results quicker\n        # TODO remove following condition test when transition is complete and\n        # run indented code unconditionally\n        if cls.__name__ not in Interface._OLDSTYLE_COMMANDS:\n            # set all common args explicitly  to override class defaults\n            # that are tailored towards the the Python API\n            kwargs['return_type'] = 'generator'\n            kwargs['result_xfm'] = None\n            # allow commands to override the default, unless something other than\n            # default is requested\n            kwargs['result_renderer'] = \\\n                args.common_output_format if args.common_output_format != 'default' \\\n                else getattr(cls, 'result_renderer', args.common_output_format)\n            if '{' in args.common_output_format:\n                # stupid hack, could and should become more powerful\n                kwargs['result_renderer'] = \\\n                    lambda x, **kwargs: ui.message(args.common_output_format.format(\n                        **{k: {k_.replace(':', '#'): v_ for k_, v_ in v.items()}\n                           if isinstance(v, dict) else v\n                           for k, v in x.items()}))\n            if args.common_on_failure:\n                kwargs['on_failure'] = args.common_on_failure\n            # compose filter function from to be invented cmdline options\n            res_filter = cls._get_result_filter(args)\n            if res_filter is not None:\n                # Don't add result_filter if it's None because then\n                # eval_results can't distinguish between --report-{status,type}\n                # not specified via the CLI and None passed via the Python API.\n                kwargs['result_filter'] = res_filter\n        try:\n            ret = cls.__call__(**kwargs)\n            if inspect.isgenerator(ret):\n                ret = list(ret)\n            if args.common_output_format == 'tailored' and \\\n                    hasattr(cls, 'custom_result_summary_renderer'):\n                cls.custom_result_summary_renderer(ret)\n            return ret\n        except KeyboardInterrupt as exc:\n            ui.error(\"\\nInterrupted by user while doing magic: %s\" % exc_str(exc))\n            if cls._interrupted_exit_code is not None:\n                sys.exit(cls._interrupted_exit_code)\n            else:\n                raise\n\n    @classmethod\n    def _get_result_filter(cls, args):\n        from datalad import cfg\n        result_filter = None\n        if args.common_report_status or 'datalad.runtime.report-status' in cfg:\n            report_status = args.common_report_status or \\\n                            cfg.obtain('datalad.runtime.report-status')\n            if report_status == \"all\":\n                pass  # no filter\n            elif report_status == 'success':\n                result_filter = EnsureKeyChoice('status', ('ok', 'notneeded'))\n            elif report_status == 'failure':\n                result_filter = EnsureKeyChoice('status',\n                                                ('impossible', 'error'))\n            else:\n                result_filter = EnsureKeyChoice('status', (report_status,))\n        if args.common_report_type:\n            tfilt = EnsureKeyChoice('type', tuple(args.common_report_type))\n            result_filter = result_filter & tfilt if result_filter else tfilt\n        return result_filter\n\n    @classmethod\n    def get_refds_path(cls, dataset):\n        \"\"\"Return a resolved reference dataset path from a `dataset` argument\"\"\"\n        # theoretically a dataset could come in as a relative path -> resolve\n        if dataset is None:\n            return dataset\n        refds_path = dataset.path if isinstance(dataset, Dataset) \\\n            else Dataset(dataset).path\n        if refds_path:\n            refds_path = resolve_path(refds_path)\n        return refds_path",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 310,
                            "end_lineno": 468,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to DataLad downloaders\n\"\"\"\n\n# TODO:  or may be it should be converted to 'addurl' thus with option for\n# adding into annex or git, depending on the largefiles option, and\n# --download-only  to only download... I see myself using it in other projects\n# as well I think.\n\n__docformat__ = 'restructuredtext'\n\nfrom os.path import isdir, curdir\n\nfrom .base import Interface\nfrom ..interface.base import build_doc\nfrom ..ui import ui\nfrom ..utils import assure_list_from_str\nfrom ..dochelpers import exc_str\nfrom ..support.param import Parameter\nfrom ..support.constraints import EnsureStr, EnsureNone\n\nfrom logging import getLogger\nlgr = getLogger('datalad.api.download-url')\n\n\n@build_doc\nclass DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to DataLad downloaders\n\"\"\"\n\n# TODO:  or may be it should be converted to 'addurl' thus with option for\n# adding into annex or git, depending on the largefiles option, and\n# --download-only  to only download... I see myself using it in other projects\n# as well I think.\n\n__docformat__ = 'restructuredtext'\n\nfrom os.path import isdir, curdir\n\nfrom .base import Interface\nfrom ..interface.base import build_doc\nfrom ..interface.results import get_status_dict\nfrom ..interface.utils import eval_results\nfrom ..utils import assure_list_from_str\nfrom ..dochelpers import exc_str\nfrom ..support.param import Parameter\nfrom ..support.constraints import EnsureStr, EnsureNone\n\nfrom logging import getLogger\nlgr = getLogger('datalad.api.download-url')\n\n\n@build_doc\nclass DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @eval_results\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False):\n        from ..downloaders.providers import Providers\n\n        common_report = {\"action\": \"download_url\"}\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/download_url.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "0626a7003",
                "new_commit_hash": "402fc08b3",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/download_url.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/download_url.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 20,
                            "length": 7
                        },
                        "new_range": {
                            "start": 20,
                            "length": 8
                        },
                        "section": "from os.path import isdir, curdir"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from .base import Interface",
                                "type": "context"
                            },
                            {
                                "content": "from ..interface.base import build_doc",
                                "type": "context"
                            },
                            {
                                "content": "from ..ui import ui",
                                "type": "deleted"
                            },
                            {
                                "content": "from ..interface.results import get_status_dict",
                                "type": "added"
                            },
                            {
                                "content": "from ..interface.utils import eval_results",
                                "type": "added"
                            },
                            {
                                "content": "from ..utils import assure_list_from_str",
                                "type": "context"
                            },
                            {
                                "content": "from ..dochelpers import exc_str",
                                "type": "context"
                            },
                            {
                                "content": "from ..support.param import Parameter",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ..interface.utils import eval_results",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 24,
                            "end_lineno": 24,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ..interface.results import get_status_dict",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 23,
                            "end_lineno": 23,
                            "parent": null
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ..ui import ui",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 23,
                            "end_lineno": 23,
                            "parent": null
                        }
                    ]
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 42,
                            "length": 8
                        },
                        "new_range": {
                            "start": 43,
                            "length": 6
                        },
                        "section": "class DownloadURL(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    # XXX prevent common args from being added to the docstring",
                                "type": "deleted"
                            },
                            {
                                "content": "    _no_eval_results = True",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    _params_ = dict(",
                                "type": "context"
                            },
                            {
                                "content": "        urls=Parameter(",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "class",
                            "name": "DownloadURL",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 34,
                            "end_lineno": 109,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 55,
                            "length": 10
                        },
                        "new_range": {
                            "start": 54,
                            "length": 6
                        },
                        "section": "class DownloadURL(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "            args=(\"-o\", \"--overwrite\"),",
                                "type": "context"
                            },
                            {
                                "content": "            action=\"store_true\",",
                                "type": "context"
                            },
                            {
                                "content": "            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),",
                                "type": "context"
                            },
                            {
                                "content": "        stop_on_failure=Parameter(",
                                "type": "deleted"
                            },
                            {
                                "content": "            args=(\"-x\", \"--stop-on-failure\"),",
                                "type": "deleted"
                            },
                            {
                                "content": "            action=\"store_true\",",
                                "type": "deleted"
                            },
                            {
                                "content": "            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),",
                                "type": "deleted"
                            },
                            {
                                "content": "        path=Parameter(",
                                "type": "context"
                            },
                            {
                                "content": "            args=(\"-O\", \"--path\"),",
                                "type": "context"
                            },
                            {
                                "content": "            doc=\"path (filename or directory path) where to store downloaded file(s).  \"",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "class",
                            "name": "DownloadURL",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 34,
                            "end_lineno": 109,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 67,
                            "length": 43
                        },
                        "new_range": {
                            "start": 62,
                            "length": 49
                        },
                        "section": "class DownloadURL(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "            constraints=EnsureStr() | EnsureNone())",
                                "type": "context"
                            },
                            {
                                "content": "    )",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    @eval_results",
                                "type": "added"
                            },
                            {
                                "content": "    @staticmethod",
                                "type": "context"
                            },
                            {
                                "content": "    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):",
                                "type": "deleted"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "deleted"
                            },
                            {
                                "content": "        Returns",
                                "type": "deleted"
                            },
                            {
                                "content": "        -------",
                                "type": "deleted"
                            },
                            {
                                "content": "        list of str",
                                "type": "deleted"
                            },
                            {
                                "content": "          downloaded successfully files",
                                "type": "deleted"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "    def __call__(urls, path=None, overwrite=False):",
                                "type": "added"
                            },
                            {
                                "content": "        from ..downloaders.providers import Providers",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        common_report = {\"action\": \"download_url\"}",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        urls = assure_list_from_str(urls)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if len(urls) > 1 and path and not isdir(path):",
                                "type": "context"
                            },
                            {
                                "content": "            raise ValueError(",
                                "type": "deleted"
                            },
                            {
                                "content": "                \"When specifying multiple urls, --path should point to \"",
                                "type": "deleted"
                            },
                            {
                                "content": "                \"an existing directory. Got %r\" % path)",
                                "type": "deleted"
                            },
                            {
                                "content": "            yield get_status_dict(",
                                "type": "added"
                            },
                            {
                                "content": "                status=\"error\",",
                                "type": "added"
                            },
                            {
                                "content": "                message=(",
                                "type": "added"
                            },
                            {
                                "content": "                    \"When specifying multiple urls, --path should point to \"",
                                "type": "added"
                            },
                            {
                                "content": "                    \"an existing directory. Got %r\", path),",
                                "type": "added"
                            },
                            {
                                "content": "                type=\"file\",",
                                "type": "added"
                            },
                            {
                                "content": "                path=path,",
                                "type": "added"
                            },
                            {
                                "content": "                **common_report)",
                                "type": "added"
                            },
                            {
                                "content": "            return",
                                "type": "added"
                            },
                            {
                                "content": "        if not path:",
                                "type": "context"
                            },
                            {
                                "content": "            path = curdir",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress",
                                "type": "context"
                            },
                            {
                                "content": "        # in % of urls which were already downloaded",
                                "type": "context"
                            },
                            {
                                "content": "        providers = Providers.from_config_files()",
                                "type": "context"
                            },
                            {
                                "content": "        downloaded_paths, failed_urls = [], []",
                                "type": "deleted"
                            },
                            {
                                "content": "        downloaded_paths = []",
                                "type": "added"
                            },
                            {
                                "content": "        for url in urls:",
                                "type": "context"
                            },
                            {
                                "content": "            # somewhat \"ugly\"",
                                "type": "context"
                            },
                            {
                                "content": "            # providers.get_provider(url).get_downloader(url).download(url, path=path)",
                                "type": "context"
                            },
                            {
                                "content": "            # for now -- via sugaring",
                                "type": "context"
                            },
                            {
                                "content": "            try:",
                                "type": "context"
                            },
                            {
                                "content": "                downloaded_path = providers.download(url, path=path, overwrite=overwrite)",
                                "type": "context"
                            },
                            {
                                "content": "                downloaded_paths.append(downloaded_path)",
                                "type": "deleted"
                            },
                            {
                                "content": "                # ui.message(\"%s -> %s\" % (url, downloaded_path))",
                                "type": "deleted"
                            },
                            {
                                "content": "            except Exception as e:",
                                "type": "context"
                            },
                            {
                                "content": "                failed_urls.append(url)",
                                "type": "deleted"
                            },
                            {
                                "content": "                ui.error(exc_str(e))",
                                "type": "deleted"
                            },
                            {
                                "content": "                if stop_on_failure:",
                                "type": "deleted"
                            },
                            {
                                "content": "                    break",
                                "type": "deleted"
                            },
                            {
                                "content": "        if failed_urls:",
                                "type": "deleted"
                            },
                            {
                                "content": "            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))",
                                "type": "deleted"
                            },
                            {
                                "content": "        return downloaded_paths",
                                "type": "deleted"
                            },
                            {
                                "content": "                yield get_status_dict(",
                                "type": "added"
                            },
                            {
                                "content": "                    status=\"error\",",
                                "type": "added"
                            },
                            {
                                "content": "                    message=exc_str(e),",
                                "type": "added"
                            },
                            {
                                "content": "                    type=\"file\",",
                                "type": "added"
                            },
                            {
                                "content": "                    path=path,",
                                "type": "added"
                            },
                            {
                                "content": "                    **common_report)",
                                "type": "added"
                            },
                            {
                                "content": "            else:",
                                "type": "added"
                            },
                            {
                                "content": "                downloaded_paths.append(downloaded_path)",
                                "type": "added"
                            },
                            {
                                "content": "                yield get_status_dict(",
                                "type": "added"
                            },
                            {
                                "content": "                    status=\"ok\",",
                                "type": "added"
                            },
                            {
                                "content": "                    type=\"file\",",
                                "type": "added"
                            },
                            {
                                "content": "                    path=downloaded_path,",
                                "type": "added"
                            },
                            {
                                "content": "                    **common_report)",
                                "type": "added"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "method",
                            "name": "DownloadURL.__call__",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 71,
                            "end_lineno": 109,
                            "parent": {
                                "file_name": "datalad/interface/download_url.py",
                                "type": "class",
                                "name": "DownloadURL",
                                "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 34,
                                "end_lineno": 109,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "class",
                            "name": "DownloadURL",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 34,
                            "end_lineno": 109,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for install-dataset command\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nfrom os.path import join as opj\n\nfrom ...api import download_url\nfrom ...tests.utils import eq_, assert_cwd_unchanged, assert_raises, \\\n    with_tempfile\nfrom ...tests.utils import with_tree\nfrom ...tests.utils import serve_path_via_http\nfrom ...tests.utils import swallow_outputs\n\n\ndef test_download_url_exceptions():\n    assert_raises(ValueError, download_url, ['url1', 'url2'], path=__file__)\n\n    # is not in effect somehow :-/ TODO: investigate!\n    with swallow_outputs() as cmo:\n        # bogus urls can't be downloaded any ways\n        with assert_raises(RuntimeError) as cm:\n            download_url('http://example.com/bogus')\n        eq_(str(cm.exception), \"1 url(s) failed to download\")\n\n\n@assert_cwd_unchanged\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'abc'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_return(toppath, topurl, outdir):\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    with swallow_outputs() as cmo:\n        out1 = download_url(urls[0], path=outdir)\n    eq_(out1, outfiles[:1])\n\n    # can't overwrite\n    with assert_raises(RuntimeError), \\\n        swallow_outputs() as cmo:\n        out2 = download_url(urls, path=outdir)\n        eq_(out2, outfiles[1:])  # only 2nd one\n\n    with swallow_outputs() as cmo:\n        out3 = download_url(urls, path=outdir, overwrite=True)\n    eq_(out3, outfiles)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for install-dataset command\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nfrom os.path import join as opj\n\nfrom ...api import download_url\nfrom ...tests.utils import eq_, assert_cwd_unchanged, assert_in, \\\n    assert_message, assert_raises,  assert_result_count, with_tempfile\nfrom ...tests.utils import with_tree\nfrom ...tests.utils import serve_path_via_http\nfrom ...tests.utils import swallow_outputs\n\n\ndef test_download_url_exceptions():\n    res0 = download_url(['url1', 'url2'], path=__file__, on_failure='ignore')\n    assert_result_count(res0, 1, status='error')\n    assert_message('When specifying multiple urls, --path should point to '\n                   'an existing directory. Got %r',\n                   res0)\n\n    res1 = download_url('http://example.com/bogus', on_failure='ignore')\n    assert_result_count(res1, 1, status='error')\n    assert_in('http://example.com/bogus', res1[0]['message'])\n\n\n@assert_cwd_unchanged\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'abc'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_return(toppath, topurl, outdir):\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    out1 = download_url(urls[0], path=outdir)\n    assert_result_count(out1, 1)\n    eq_(out1[0]['path'], outfiles[0])\n\n    # can't overwrite\n    out2 = download_url(urls, path=outdir, on_failure='ignore')\n    assert_result_count(out2, 1, status='error')\n    assert_in('file1.txt already exists', out2[0]['message'])\n    assert_result_count(out2, 1, status='ok')  # only 2nd one\n    eq_(out2[1]['path'], outfiles[1])\n\n    out3 = download_url(urls, path=outdir, overwrite=True, on_failure='ignore')\n    assert_result_count(out3, 2, status='ok')\n    eq_([r['path'] for r in out3], outfiles)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_download_url.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "443c50de6",
                "new_commit_hash": "4e7ae7582",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_download_url.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_download_url.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 15,
                            "length": 22
                        },
                        "new_range": {
                            "start": 15,
                            "length": 23
                        },
                        "section": "__docformat__ = 'restructuredtext'"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from os.path import join as opj",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from ...api import download_url",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import eq_, assert_cwd_unchanged, assert_raises, \\",
                                "type": "deleted"
                            },
                            {
                                "content": "    with_tempfile",
                                "type": "deleted"
                            },
                            {
                                "content": "from ...tests.utils import eq_, assert_cwd_unchanged, assert_in, \\",
                                "type": "added"
                            },
                            {
                                "content": "    assert_message, assert_raises,  assert_result_count, with_tempfile",
                                "type": "added"
                            },
                            {
                                "content": "from ...tests.utils import with_tree",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import serve_path_via_http",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import swallow_outputs",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_download_url_exceptions():",
                                "type": "context"
                            },
                            {
                                "content": "    assert_raises(ValueError, download_url, ['url1', 'url2'], path=__file__)",
                                "type": "deleted"
                            },
                            {
                                "content": "    res0 = download_url(['url1', 'url2'], path=__file__, on_failure='ignore')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_result_count(res0, 1, status='error')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_message('When specifying multiple urls, --path should point to '",
                                "type": "added"
                            },
                            {
                                "content": "                   'an existing directory. Got %r',",
                                "type": "added"
                            },
                            {
                                "content": "                   res0)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # is not in effect somehow :-/ TODO: investigate!",
                                "type": "deleted"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "        # bogus urls can't be downloaded any ways",
                                "type": "deleted"
                            },
                            {
                                "content": "        with assert_raises(RuntimeError) as cm:",
                                "type": "deleted"
                            },
                            {
                                "content": "            download_url('http://example.com/bogus')",
                                "type": "deleted"
                            },
                            {
                                "content": "        eq_(str(cm.exception), \"1 url(s) failed to download\")",
                                "type": "deleted"
                            },
                            {
                                "content": "    res1 = download_url('http://example.com/bogus', on_failure='ignore')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_result_count(res1, 1, status='error')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_in('http://example.com/bogus', res1[0]['message'])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@assert_cwd_unchanged",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_exceptions",
                            "content": "def test_download_url_exceptions():\n    assert_raises(ValueError, download_url, ['url1', 'url2'], path=__file__)\n\n    # is not in effect somehow :-/ TODO: investigate!\n    with swallow_outputs() as cmo:\n        # bogus urls can't be downloaded any ways\n        with assert_raises(RuntimeError) as cm:\n            download_url('http://example.com/bogus')\n        eq_(str(cm.exception), \"1 url(s) failed to download\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 25,
                            "end_lineno": 33,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ...tests.utils import eq_, assert_cwd_unchanged, assert_in, \\\n    assert_message, assert_raises,  assert_result_count, with_tempfile",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 18,
                            "end_lineno": 19,
                            "parent": null
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ...tests.utils import eq_, assert_cwd_unchanged, assert_raises, \\\n    with_tempfile",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 18,
                            "end_lineno": 19,
                            "parent": null
                        }
                    ]
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 45,
                            "length": 16
                        },
                        "new_range": {
                            "start": 46,
                            "length": 17
                        },
                        "section": "def test_download_url_return(toppath, topurl, outdir):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    urls = [topurl + f for f in files]",
                                "type": "context"
                            },
                            {
                                "content": "    outfiles = [opj(outdir, f) for f in files]",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "        out1 = download_url(urls[0], path=outdir)",
                                "type": "deleted"
                            },
                            {
                                "content": "    eq_(out1, outfiles[:1])",
                                "type": "deleted"
                            },
                            {
                                "content": "    out1 = download_url(urls[0], path=outdir)",
                                "type": "added"
                            },
                            {
                                "content": "    assert_result_count(out1, 1)",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(out1[0]['path'], outfiles[0])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # can't overwrite",
                                "type": "context"
                            },
                            {
                                "content": "    with assert_raises(RuntimeError), \\",
                                "type": "deleted"
                            },
                            {
                                "content": "        swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "        out2 = download_url(urls, path=outdir)",
                                "type": "deleted"
                            },
                            {
                                "content": "        eq_(out2, outfiles[1:])  # only 2nd one",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "    with swallow_outputs() as cmo:",
                                "type": "deleted"
                            },
                            {
                                "content": "        out3 = download_url(urls, path=outdir, overwrite=True)",
                                "type": "deleted"
                            },
                            {
                                "content": "    eq_(out3, outfiles)",
                                "type": "deleted"
                            },
                            {
                                "content": "    out2 = download_url(urls, path=outdir, on_failure='ignore')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_result_count(out2, 1, status='error')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_in('file1.txt already exists', out2[0]['message'])",
                                "type": "added"
                            },
                            {
                                "content": "    assert_result_count(out2, 1, status='ok')  # only 2nd one",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(out2[1]['path'], outfiles[1])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    out3 = download_url(urls, path=outdir, overwrite=True, on_failure='ignore')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_result_count(out3, 2, status='ok')",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([r['path'] for r in out3], outfiles)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_return",
                            "content": "def test_download_url_return(toppath, topurl, outdir):\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    with swallow_outputs() as cmo:\n        out1 = download_url(urls[0], path=outdir)\n    eq_(out1, outfiles[:1])\n\n    # can't overwrite\n    with assert_raises(RuntimeError), \\\n        swallow_outputs() as cmo:\n        out2 = download_url(urls, path=outdir)\n        eq_(out2, outfiles[1:])  # only 2nd one\n\n    with swallow_outputs() as cmo:\n        out3 = download_url(urls, path=outdir, overwrite=True)\n    eq_(out3, outfiles)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 43,
                            "end_lineno": 60,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae^",
    "new_commit_hash": "0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae",
    "commit_message": "RF+ENH: download_url: Use the eval_results interface\n\nThis isn't strictly necessary, but since we're making major changes to\ndownload_url, we might as well convert it from the old style\ninterface.  We can drop --stop-on-failure because the eval_results\ninterface supports --on-failure.",
    "commit_date": "2018-05-17T16:42:40-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for install-dataset command\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nfrom os.path import join as opj\n\nfrom datalad.api import download_url\nfrom datalad.tests.utils import eq_, assert_cwd_unchanged, assert_in, \\\n    assert_message, assert_raises,  assert_result_count, with_tempfile\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import serve_path_via_http\nfrom datalad.tests.utils import swallow_outputs\n\n\ndef test_download_url_exceptions():\n    res0 = download_url(['url1', 'url2'], path=__file__, on_failure='ignore')\n    assert_result_count(res0, 1, status='error')\n    assert_message('When specifying multiple urls, --path should point to '\n                   'an existing directory. Got %r',\n                   res0)\n\n    res1 = download_url('http://example.com/bogus', on_failure='ignore')\n    assert_result_count(res1, 1, status='error')\n    assert_in('http://example.com/bogus', res1[0]['message'])\n\n\n@assert_cwd_unchanged\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'abc'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_return(toppath, topurl, outdir):\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    out1 = download_url(urls[0], path=outdir)\n    assert_result_count(out1, 1)\n    eq_(out1[0]['path'], outfiles[0])\n\n    # can't overwrite\n    out2 = download_url(urls, path=outdir, on_failure='ignore')\n    assert_result_count(out2, 1, status='error')\n    assert_in('file1.txt already exists', out2[0]['message'])\n    assert_result_count(out2, 1, status='ok')  # only 2nd one\n    eq_(out2[1]['path'], outfiles[1])\n\n    out3 = download_url(urls, path=outdir, overwrite=True, on_failure='ignore')\n    assert_result_count(out3, 2, status='ok')\n    eq_([r['path'] for r in out3], outfiles)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.10.0.rc3.dev39\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 160ms\nInstalled 25 packages in 4.33s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpj2NAXg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpj2NAXg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpj2NAXg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpj2NAXg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpj2NAXg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpDVciRG/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpDVciRG/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpDVciRG/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpDVciRG/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpDVciRG/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/datalad/__init__.py\", line 23, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/datalad/cmd.py\", line 35, in <module>\n    from .utils import on_windows, get_tempfile_kwargs, assure_unicode\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/datalad/utils.py\", line 24, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 199ms\nInstalled 26 packages in 3.65s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 60 packages in 1.31s\nPrepared 1 package in 1.41s\nInstalled 53 packages in 77ms\n + appdirs==1.4.4\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.10.0rc3 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae)\n + deprecated==1.2.15\n + duecredit==0.9.3\n + exifread==3.0.0\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + idna==3.10\n + importlib-resources==5.12.0\n + iso8601==2.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack==1.0.5\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.12\n + pillow==9.5.0\n + pycparser==2.21\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tqdm==4.67.1\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\nResolved 92 packages in 212ms\nInstalled 30 packages in 101ms\n + alabaster==0.7.13\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.14.0\n + build==1.1.1\n + distlib==0.3.9\n + docutils==0.19\n + filelock==3.12.2\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.9.16\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + platformdirs==4.0.0\n + pygments==2.17.2\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n + tabulate==0.9.0\n + virtualenv==20.26.6\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae\nplugins: cov-4.1.0\ncollected 2 items\n\nr2e_tests/test_1.py .E                                                   [100%]\n\n==================================== ERRORS ====================================\n__________________ ERROR at setup of test_download_url_return __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/r2e_tests/test_1.py, line 37\n  @assert_cwd_unchanged\n  @with_tree(tree=[\n      ('file1.txt', 'abc'),\n      ('file2.txt', 'abc'),\n  ])\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_return(toppath, topurl, outdir):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/r2e_tests/test_1.py:37\n==================================== PASSES ====================================\n_________________________ test_download_url_exceptions _________________________\n----------------------------- Captured stderr call -----------------------------\n[INFO] Downloading 'http://example.com/bogus' into '.' \n------------------------------ Captured log call -------------------------------\nINFO     datalad.downloaders:base.py:374 Downloading 'http://example.com/bogus' into '.'\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_download_url_exceptions\nERROR r2e_tests/test_1.py::test_download_url_return\n========================== 1 passed, 1 error in 1.13s ==========================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae\nplugins: cov-4.1.0\ncollected 2 items\n\nr2e_tests/test_1.py FE                                                   [100%]\n\n==================================== ERRORS ====================================\n__________________ ERROR at setup of test_download_url_return __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/r2e_tests/test_1.py, line 37\n  @assert_cwd_unchanged\n  @with_tree(tree=[\n      ('file1.txt', 'abc'),\n      ('file2.txt', 'abc'),\n  ])\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_return(toppath, topurl, outdir):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_0498f372313e3a25c0b1cd2b3ffdac4c0ce351ae/r2e_tests/test_1.py:37\n=================================== FAILURES ===================================\n_________________________ test_download_url_exceptions _________________________\n\n    def test_download_url_exceptions():\n>       res0 = download_url(['url1', 'url2'], path=__file__, on_failure='ignore')\nE       TypeError: __call__() got an unexpected keyword argument 'on_failure'\n\nr2e_tests/test_1.py:26: TypeError\n=========================== short test summary info ============================\nERROR r2e_tests/test_1.py::test_download_url_return\nFAILED r2e_tests/test_1.py::test_download_url_exceptions - TypeError: __call_...\n========================== 1 failed, 1 error in 0.44s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/interface/base.py datalad/interface/download_url.py datalad/interface/tests/test_download_url.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
**Issue Title:** `download_url` Function Returns Unexpected Structure After Interface Update

**Description:**
After updating the `download_url` interface to use the `eval_results` system, the function no longer returns a simple list of downloaded file paths as previously expected. Instead, it yields dictionaries containing status information and messages. This change causes existing code and tests that rely on the original return structure to fail.

**Example Buggy Code:**
```python
# Attempting to download multiple URLs to a specified directory
downloaded_files = download_url(['http://example.com/file1.dat', 'http://example.com/file2.dat'], path='/path/to/dir')

# Expecting a list of file paths
for file_path in downloaded_files:
    print(f"Downloaded: {file_path}")
```

**Expected Behavior:**
The `download_url` function should return a list of file paths that were successfully downloaded, allowing the caller to iterate over them directly.

**Actual Behavior:**
The `download_url` function now yields dictionaries with status and message information instead of returning a list of file paths. Attempting to iterate over the result as if it were a list of file paths leads to errors.

**Error Message:**
When running the above code, an error similar to the following is encountered:
```
TypeError: 'dict' object is not iterable
```

This indicates that the code is trying to iterate over a dictionary where a list was expected.
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/download_url.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to DataLad downloaders
"""

# TODO:  or may be it should be converted to 'addurl' thus with option for
# adding into annex or git, depending on the largefiles option, and
# --download-only  to only download... I see myself using it in other projects
# as well I think.

__docformat__ = 'restructuredtext'

from os.path import isdir, curdir

from .base import Interface
from ..interface.base import build_doc
from ..ui import ui
from ..utils import assure_list_from_str
from ..dochelpers import exc_str
from ..support.param import Parameter
from ..support.constraints import EnsureStr, EnsureNone

from logging import getLogger
lgr = getLogger('datalad.api.download-url')


@build_doc
class DownloadURL(Interface):
    """Download content

    It allows for a uniform download interface to various supported URL
    schemes, re-using or asking for authentication details maintained by
    datalad.

    Examples:

      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat
    """
    # XXX prevent common args from being added to the docstring
    _no_eval_results = True

    _params_ = dict(
        urls=Parameter(
            doc="URL(s) to be downloaded",
            constraints=EnsureStr(),  # TODO: EnsureURL
            metavar='url',
            nargs='+'),
        overwrite=Parameter(
            args=("-o", "--overwrite"),
            action="store_true",
            doc="""flag to overwrite it if target file exists"""),
        stop_on_failure=Parameter(
            args=("-x", "--stop-on-failure"),
            action="store_true",
            doc="""flag to stop subsequent downloads upon first failure to download"""),
        path=Parameter(
            args=("-O", "--path"),
            doc="path (filename or directory path) where to store downloaded file(s).  "
                "In case of multiple URLs provided, must point to a directory.  Otherwise current "
                "directory is used",
            constraints=EnsureStr() | EnsureNone())
    )

    @staticmethod
    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):
        """
        Returns
        -------
        list of str
          downloaded successfully files
        """

        from ..downloaders.providers import Providers

        urls = assure_list_from_str(urls)

        if len(urls) > 1 and path and not isdir(path):
            raise ValueError(
                "When specifying multiple urls, --path should point to "
                "an existing directory. Got %r" % path)
        if not path:
            path = curdir

        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress
        # in % of urls which were already downloaded
        providers = Providers.from_config_files()
        downloaded_paths, failed_urls = [], []
        for url in urls:
            # somewhat "ugly"
            # providers.get_provider(url).get_downloader(url).download(url, path=path)
            # for now -- via sugaring
            try:
                downloaded_path = providers.download(url, path=path, overwrite=overwrite)
                downloaded_paths.append(downloaded_path)
                # ui.message("%s -> %s" % (url, downloaded_path))
            except Exception as e:
                failed_urls.append(url)
                ui.error(exc_str(e))
                if stop_on_failure:
                    break
        if failed_urls:
            raise RuntimeError("%d url(s) failed to download" % len(failed_urls))
        return downloaded_paths

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
**Issue Title:** `download_url` Function Returns Unexpected Structure After Interface Update

**Description:**
After updating the `download_url` interface to use the `eval_results` system, the function no longer returns a simple list of downloaded file paths as previously expected. Instead, it yields dictionaries containing status information and messages. This change causes existing code and tests that rely on the original return structure to fail.

**Example Buggy Code:**
```python
# Attempting to download multiple URLs to a specified directory
downloaded_files = download_url(['http://example.com/file1.dat', 'http://example.com/file2.dat'], path='/path/to/dir')

# Expecting a list of file paths
for file_path in downloaded_files:
    print(f"Downloaded: {file_path}")
```

**Expected Behavior:**
The `download_url` function should return a list of file paths that were successfully downloaded, allowing the caller to iterate over them directly.

**Actual Behavior:**
The `download_url` function now yields dictionaries with status and message information instead of returning a list of file paths. Attempting to iterate over the result as if it were a list of file paths leads to errors.

**Error Message:**
When running the above code, an error similar to the following is encountered:
```
TypeError: 'dict' object is not iterable
```

This indicates that the code is trying to iterate over a dictionary where a list was expected.
{
    "test_download_url_exceptions": "PASSED",
    "test_download_url_return": "ERROR"
}
diff --git a/datalad/interface/base.py b/datalad/interface/base.py
index bed8339b8..9e217a252 100644
--- a/datalad/interface/base.py
+++ b/datalad/interface/base.py
@@ -317,8 +317,7 @@ class Interface(object):
 
     _OLDSTYLE_COMMANDS = (
         'AddArchiveContent', 'CrawlInit', 'Crawl', 'CreateSiblingGithub',
-        'CreateTestDataset', 'DownloadURL', 'Export', 'Ls', 'Move', 'SSHRun',
-        'Test')
+        'CreateTestDataset', 'Export', 'Ls', 'Move', 'SSHRun', 'Test')
 
     @classmethod
     def setup_parser(cls, parser):
diff --git a/datalad/interface/download_url.py b/datalad/interface/download_url.py
index 0626a7003..402fc08b3 100644
--- a/datalad/interface/download_url.py
+++ b/datalad/interface/download_url.py
@@ -20,7 +20,8 @@ from os.path import isdir, curdir
 
 from .base import Interface
 from ..interface.base import build_doc
-from ..ui import ui
+from ..interface.results import get_status_dict
+from ..interface.utils import eval_results
 from ..utils import assure_list_from_str
 from ..dochelpers import exc_str
 from ..support.param import Parameter
@@ -42,8 +43,6 @@ class DownloadURL(Interface):
 
       $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat
     """
-    # XXX prevent common args from being added to the docstring
-    _no_eval_results = True
 
     _params_ = dict(
         urls=Parameter(
@@ -55,10 +54,6 @@ class DownloadURL(Interface):
             args=("-o", "--overwrite"),
             action="store_true",
             doc="""flag to overwrite it if target file exists"""),
-        stop_on_failure=Parameter(
-            args=("-x", "--stop-on-failure"),
-            action="store_true",
-            doc="""flag to stop subsequent downloads upon first failure to download"""),
         path=Parameter(
             args=("-O", "--path"),
             doc="path (filename or directory path) where to store downloaded file(s).  "
@@ -67,43 +62,49 @@ class DownloadURL(Interface):
             constraints=EnsureStr() | EnsureNone())
     )
 
+    @eval_results
     @staticmethod
-    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):
-        """
-        Returns
-        -------
-        list of str
-          downloaded successfully files
-        """
-
+    def __call__(urls, path=None, overwrite=False):
         from ..downloaders.providers import Providers
 
+        common_report = {"action": "download_url"}
+
         urls = assure_list_from_str(urls)
 
         if len(urls) > 1 and path and not isdir(path):
-            raise ValueError(
-                "When specifying multiple urls, --path should point to "
-                "an existing directory. Got %r" % path)
+            yield get_status_dict(
+                status="error",
+                message=(
+                    "When specifying multiple urls, --path should point to "
+                    "an existing directory. Got %r", path),
+                type="file",
+                path=path,
+                **common_report)
+            return
         if not path:
             path = curdir
 
         # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress
         # in % of urls which were already downloaded
         providers = Providers.from_config_files()
-        downloaded_paths, failed_urls = [], []
+        downloaded_paths = []
         for url in urls:
             # somewhat "ugly"
             # providers.get_provider(url).get_downloader(url).download(url, path=path)
             # for now -- via sugaring
             try:
                 downloaded_path = providers.download(url, path=path, overwrite=overwrite)
-                downloaded_paths.append(downloaded_path)
-                # ui.message("%s -> %s" % (url, downloaded_path))
             except Exception as e:
-                failed_urls.append(url)
-                ui.error(exc_str(e))
-                if stop_on_failure:
-                    break
-        if failed_urls:
-            raise RuntimeError("%d url(s) failed to download" % len(failed_urls))
-        return downloaded_paths
+                yield get_status_dict(
+                    status="error",
+                    message=exc_str(e),
+                    type="file",
+                    path=path,
+                    **common_report)
+            else:
+                downloaded_paths.append(downloaded_path)
+                yield get_status_dict(
+                    status="ok",
+                    type="file",
+                    path=downloaded_path,
+                    **common_report)

{"datalad/interface/download_url.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to DataLad downloaders\n\"\"\"\n\n# TODO:  or may be it should be converted to 'addurl' thus with option for\n# adding into annex or git, depending on the largefiles option, and\n# --download-only  to only download... I see myself using it in other projects\n# as well I think.\n\n__docformat__ = 'restructuredtext'\n\nfrom os.path import isdir, curdir\n\nfrom .base import Interface\nfrom ..interface.base import build_doc\nfrom ..ui import ui\nfrom ..utils import assure_list_from_str\nfrom ..dochelpers import exc_str\nfrom ..support.param import Parameter\nfrom ..support.constraints import EnsureStr, EnsureNone\n\nfrom logging import getLogger\nlgr = getLogger('datalad.api.download-url')\n\n\n@build_doc\nclass DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n    # XXX prevent common args from being added to the docstring\n    _no_eval_results = True\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        stop_on_failure=Parameter(\n            args=(\"-x\", \"--stop-on-failure\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to stop subsequent downloads upon first failure to download\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone())\n    )\n\n    @staticmethod\n    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):\n        \"\"\"\n        Returns\n        -------\n        list of str\n          downloaded successfully files\n        \"\"\"\n\n        from ..downloaders.providers import Providers\n\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not isdir(path):\n            raise ValueError(\n                \"When specifying multiple urls, --path should point to \"\n                \"an existing directory. Got %r\" % path)\n        if not path:\n            path = curdir\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths, failed_urls = [], []\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n                downloaded_paths.append(downloaded_path)\n                # ui.message(\"%s -> %s\" % (url, downloaded_path))\n            except Exception as e:\n                failed_urls.append(url)\n                ui.error(exc_str(e))\n                if stop_on_failure:\n                    break\n        if failed_urls:\n            raise RuntimeError(\"%d url(s) failed to download\" % len(failed_urls))\n        return downloaded_paths\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
**Issue Title:** `download_url` Function Returns Unexpected Structure After Interface Update

**Description:**
After updating the `download_url` interface to use the `eval_results` system, the function no longer returns a simple list of downloaded file paths as previously expected. Instead, it yields dictionaries containing status information and messages. This change causes existing code and tests that rely on the original return structure to fail.

**Example Buggy Code:**
```python
# Attempting to download multiple URLs to a specified directory
downloaded_files = download_url(['http://example.com/file1.dat', 'http://example.com/file2.dat'], path='/path/to/dir')

# Expecting a list of file paths
for file_path in downloaded_files:
    print(f"Downloaded: {file_path}")
```

**Expected Behavior:**
The `download_url` function should return a list of file paths that were successfully downloaded, allowing the caller to iterate over them directly.

**Actual Behavior:**
The `download_url` function now yields dictionaries with status and message information instead of returning a list of file paths. Attempting to iterate over the result as if it were a list of file paths leads to errors.

**Error Message:**
When running the above code, an error similar to the following is encountered:
```
TypeError: 'dict' object is not iterable
```

This indicates that the code is trying to iterate over a dictionary where a list was expected.
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/download_url.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to DataLad downloaders
"""

# TODO:  or may be it should be converted to 'addurl' thus with option for
# adding into annex or git, depending on the largefiles option, and
# --download-only  to only download... I see myself using it in other projects
# as well I think.

__docformat__ = 'restructuredtext'

from os.path import isdir, curdir

from .base import Interface
from ..interface.base import build_doc
from ..ui import ui
from ..utils import assure_list_from_str
from ..dochelpers import exc_str
from ..support.param import Parameter
from ..support.constraints import EnsureStr, EnsureNone

from logging import getLogger
lgr = getLogger('datalad.api.download-url')


@build_doc
class DownloadURL(Interface):
    """Download content

    It allows for a uniform download interface to various supported URL
    schemes, re-using or asking for authentication details maintained by
    datalad.

    Examples:

      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat
    """
    # XXX prevent common args from being added to the docstring
    _no_eval_results = True

    _params_ = dict(
        urls=Parameter(
            doc="URL(s) to be downloaded",
            constraints=EnsureStr(),  # TODO: EnsureURL
            metavar='url',
            nargs='+'),
        overwrite=Parameter(
            args=("-o", "--overwrite"),
            action="store_true",
            doc="""flag to overwrite it if target file exists"""),
        stop_on_failure=Parameter(
            args=("-x", "--stop-on-failure"),
            action="store_true",
            doc="""flag to stop subsequent downloads upon first failure to download"""),
        path=Parameter(
            args=("-O", "--path"),
            doc="path (filename or directory path) where to store downloaded file(s).  "
                "In case of multiple URLs provided, must point to a directory.  Otherwise current "
                "directory is used",
            constraints=EnsureStr() | EnsureNone())
    )

    @staticmethod
    def __call__(urls, path=None, overwrite=False, stop_on_failure=False):
        """
        Returns
        -------
        list of str
          downloaded successfully files
        """

        from ..downloaders.providers import Providers

        urls = assure_list_from_str(urls)

        if len(urls) > 1 and path and not isdir(path):
            raise ValueError(
                "When specifying multiple urls, --path should point to "
                "an existing directory. Got %r" % path)
        if not path:
            path = curdir

        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress
        # in % of urls which were already downloaded
        providers = Providers.from_config_files()
        downloaded_paths, failed_urls = [], []
        for url in urls:
            # somewhat "ugly"
            # providers.get_provider(url).get_downloader(url).download(url, path=path)
            # for now -- via sugaring
            try:
                downloaded_path = providers.download(url, path=path, overwrite=overwrite)
                downloaded_paths.append(downloaded_path)
                # ui.message("%s -> %s" % (url, downloaded_path))
            except Exception as e:
                failed_urls.append(url)
                ui.error(exc_str(e))
                if stop_on_failure:
                    break
        if failed_urls:
            raise RuntimeError("%d url(s) failed to download" % len(failed_urls))
        return downloaded_paths

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user